{
  "architecture": "baseline",
  "batch_size": 32,
  "canonical_sha256_without_self": "890c06c98d3b195e1aa79852477608280a7bcd106a01b463c2d793e8ec58d94f",
  "depth": 32,
  "diagnostics": [
    {
      "activation_grad_rms_by_block": [
        0.00042041894630528986,
        0.000374794501112774,
        0.0003388339828234166,
        0.000304876797599718,
        0.0002806047268677503,
        0.0002645689237397164,
        0.00024906310136429965,
        0.00023202023294288665,
        0.00021741738601122051,
        0.00020566635066643357,
        0.0001967942516785115,
        0.0001872276043286547,
        0.00017897134239319712,
        0.00017287296941503882,
        0.00016580727242399007,
        0.00016062518989201635,
        0.0001559370430186391,
        0.00015203763905446976,
        0.00014800428471062332,
        0.00014486612053588033,
        0.00014247088984120637,
        0.00013994764594826847,
        0.00013763502647634596,
        0.00013544136891141534,
        0.00013363662583287805,
        0.0001319239236181602,
        0.00013088916603010148,
        0.00012932090612594038,
        0.00012785944272764027,
        0.00012667682312894613,
        0.0001251104404218495,
        0.00012353518104646355
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00030814765159448143,
        "first_to_last_ratio": 2.395816319369912,
        "imbalance_abs_log_ratio": 0.8737240159497589,
        "last_quartile_mean": 0.00012861906361649744,
        "mean": 0.00019174550334355445,
        "normalized": [
          2.1925882952885543,
          1.9546455826984732,
          1.7671026277801187,
          1.5900075479395406,
          1.4634227242606306,
          1.3797920635754504,
          1.2989253829752037,
          1.2100426288859099,
          1.1338851875012121,
          1.0726006455438846,
          1.0263304653664347,
          0.9764380445115058,
          0.9333796061570758,
          0.9015750899007978,
          0.8647257408008662,
          0.8376999047754503,
          0.8132500647967916,
          0.7929137132465669,
          0.7718787774931073,
          0.7555124788314888,
          0.7430207611488979,
          0.7298614231256383,
          0.7178005433052705,
          0.7063600791135226,
          0.6969478997034861,
          0.6880157360550423,
          0.6826192205174408,
          0.6744403590744623,
          0.6668184676985713,
          0.660650815377801,
          0.6524817439796045,
          0.644266378571199
        ],
        "population_cv": 0.4034880502878489
      },
      "activation_output_rms_by_block": [
        0.02817646972835064,
        0.028377581387758255,
        0.028678538277745247,
        0.028959807008504868,
        0.029224006459116936,
        0.029430421069264412,
        0.02957010455429554,
        0.029960129410028458,
        0.030406322330236435,
        0.03066711500287056,
        0.030792171135544777,
        0.031063809990882874,
        0.03129913657903671,
        0.03160789608955383,
        0.03180097043514252,
        0.032092027366161346,
        0.03248445317149162,
        0.032903194427490234,
        0.03316524624824524,
        0.033593542873859406,
        0.034045662730932236,
        0.03455357626080513,
        0.03491735830903053,
        0.03521261364221573,
        0.03560211881995201,
        0.0359802283346653,
        0.03664350509643555,
        0.03706779330968857,
        0.0375242717564106,
        0.03805854171514511,
        0.038408540189266205,
        0.039021097123622894
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.029047132236883044,
        "first_to_last_ratio": 0.7789886319526242,
        "imbalance_abs_log_ratio": 0.24975882634672159,
        "last_quartile_mean": 0.03728826204314828,
        "mean": 0.03285275783855468,
        "normalized": [
          0.8576591915605899,
          0.863780798166523,
          0.8729415782588962,
          0.8815030735264118,
          0.8895449996231614,
          0.8958280219241153,
          0.9000798258584322,
          0.9119517319446604,
          0.9255333290331198,
          0.93347155674343,
          0.9372781209682287,
          0.945546493951495,
          0.9527095634663979,
          0.96210784631481,
          0.9679848063720901,
          0.9768442431490312,
          0.9887892313676379,
          1.0015352315073034,
          1.0095117862319563,
          1.0225486407851998,
          1.0363106469855512,
          1.051770948138011,
          1.0628440534770849,
          1.0718312847663467,
          1.0836873724546434,
          1.095196589323784,
          1.1153859677933096,
          1.128300810904444,
          1.1421954875390588,
          1.1584580479414466,
          1.1691115972063533,
          1.187757122716477
        ],
        "population_cv": 0.09694557255182189
      },
      "bits_per_byte": 8.04236897991864,
      "branch_output_rms_by_sublayer": [
        0.0023512106854468584,
        0.0027520405128598213,
        0.0023248849902302027,
        0.0026936428621411324,
        0.0027838374953716993,
        0.002722234698012471,
        0.002502240240573883,
        0.002735150745138526,
        0.0026338943280279636,
        0.002727407729253173,
        0.0029348398093134165,
        0.002726956969127059,
        0.0029466808773577213,
        0.002693725284188986,
        0.0032394269946962595,
        0.0027635409496724606,
        0.003579996759071946,
        0.0027838009409606457,
        0.0033850702457129955,
        0.0026989516336470842,
        0.0034471398685127497,
        0.002707457635551691,
        0.0036693045403808355,
        0.0027280766516923904,
        0.0035536750219762325,
        0.002806172939017415,
        0.0038618813268840313,
        0.0027198586612939835,
        0.003969449549913406,
        0.002747237915173173,
        0.003774659475311637,
        0.0027643220964819193,
        0.004222253803163767,
        0.002741035772487521,
        0.004245356190949678,
        0.00275977305136621,
        0.004526515491306782,
        0.0027434700168669224,
        0.004249728750437498,
        0.0026767924427986145,
        0.004827029537409544,
        0.002736816881224513,
        0.004658437799662352,
        0.0027651384007185698,
        0.0048249042592942715,
        0.0027371037285774946,
        0.005304279271513224,
        0.0027228749822825193,
        0.005711541976779699,
        0.002716122195124626,
        0.005045780446380377,
        0.002763758646324277,
        0.005378549452871084,
        0.0027021593414247036,
        0.005505796521902084,
        0.0027199271135032177,
        0.005381250753998756,
        0.0027382206171751022,
        0.005084864795207977,
        0.002702005673199892,
        0.005992465652525425,
        0.002712319139391184,
        0.00573702622205019,
        0.002734891138970852
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.010411069784541356,
        0.009271735997530202,
        0.009345779773643629,
        0.008099773113238128,
        0.008225801589217078,
        0.008647498843153682,
        0.007992995665779355,
        0.0076659621960069,
        0.007406648616870928,
        0.006601475371863238,
        0.006568536350729326,
        0.006644454140273569,
        0.005847021988988907,
        0.006236102346072616,
        0.006448411665711761,
        0.005769055287848533,
        0.005701484582581849,
        0.005836584711958408,
        0.005498613632501928,
        0.004978853123018873,
        0.005187902378445178,
        0.005078997897134883,
        0.005158829428828408,
        0.005043411661395455,
        0.00572819320620205,
        0.004439465860302503,
        0.004811331873512697,
        0.004686191415610403,
        0.004548827932369003,
        0.00413026790671143,
        0.004686908144290253,
        0.004087931241888514
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00870757712038879,
        "first_to_last_ratio": 1.8766776125890325,
        "imbalance_abs_log_ratio": 0.6295029861071718,
        "last_quartile_mean": 0.004639889697610857,
        "mean": 0.006274566179006908,
        "normalized": [
          1.6592493389222875,
          1.4776696480708191,
          1.489470268862924,
          1.2908897415629936,
          1.3109753494573861,
          1.3781827454599171,
          1.2738722387727572,
          1.2217517478188766,
          1.1804240174646141,
          1.0521006844983298,
          1.0468510751716935,
          1.0589503641708666,
          0.9318607569319367,
          0.9938698817038568,
          1.027706375507919,
          0.9194349255810409,
          0.9086659411861104,
          0.930197330850717,
          0.8763336740154054,
          0.7934975870804967,
          0.8268145128188418,
          0.8094580170542961,
          0.8221810531042816,
          0.8037865116905484,
          0.9129225898305317,
          0.707533514453289,
          0.766799127820212,
          0.7468550465352011,
          0.7249629380893642,
          0.6582555333514927,
          0.7469692741422425,
          0.6515081880187487
        ],
        "population_cv": 0.26225105130829346
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.027978211641311646,
        0.028061864897608757,
        0.02817646972835064,
        0.028305010870099068,
        0.028377581387758255,
        0.02853298746049404,
        0.028678538277745247,
        0.028806988149881363,
        0.028959807008504868,
        0.029115403071045876,
        0.029224006459116936,
        0.02931177243590355,
        0.029430421069264412,
        0.02947254478931427,
        0.02957010455429554,
        0.02984699048101902,
        0.029960129410028458,
        0.0302706528455019,
        0.030406322330236435,
        0.030570803210139275,
        0.03066711500287056,
        0.030674004927277565,
        0.030792171135544777,
        0.03094458393752575,
        0.031063809990882874,
        0.031205367296934128,
        0.03129913657903671,
        0.03147619590163231,
        0.03160789608955383,
        0.03167128562927246,
        0.03180097043514252,
        0.03195846453309059,
        0.032092027366161346,
        0.032371606677770615,
        0.03248445317149162,
        0.032751601189374924,
        0.032903194427490234,
        0.03311120718717575,
        0.03316524624824524,
        0.03350966051220894,
        0.033593542873859406,
        0.03394605591893196,
        0.034045662730932236,
        0.03444153815507889,
        0.03455357626080513,
        0.03483813256025314,
        0.03491735830903053,
        0.03510763868689537,
        0.03521261364221573,
        0.03553535044193268,
        0.03560211881995201,
        0.03590916842222214,
        0.0359802283346653,
        0.036534491926431656,
        0.03664350509643555,
        0.036974843591451645,
        0.03706779330968857,
        0.037423327565193176,
        0.0375242717564106,
        0.037971291691064835,
        0.03805854171514511,
        0.03832196444272995,
        0.038408540189266205,
        0.03889527916908264
      ],
      "loss_nats": 5.574545383453369,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 0,
      "stream_state_rms_by_sublayer": [
        0.028061864897608757,
        0.02817646972835064,
        0.028305010870099068,
        0.028377581387758255,
        0.02853298746049404,
        0.028678538277745247,
        0.028806988149881363,
        0.028959807008504868,
        0.029115403071045876,
        0.029224006459116936,
        0.02931177243590355,
        0.029430421069264412,
        0.02947254478931427,
        0.02957010455429554,
        0.02984699048101902,
        0.029960129410028458,
        0.0302706528455019,
        0.030406322330236435,
        0.030570803210139275,
        0.03066711500287056,
        0.030674004927277565,
        0.030792171135544777,
        0.03094458393752575,
        0.031063809990882874,
        0.031205367296934128,
        0.03129913657903671,
        0.03147619590163231,
        0.03160789608955383,
        0.03167128562927246,
        0.03180097043514252,
        0.03195846453309059,
        0.032092027366161346,
        0.032371606677770615,
        0.03248445317149162,
        0.032751601189374924,
        0.032903194427490234,
        0.03311120718717575,
        0.03316524624824524,
        0.03350966051220894,
        0.033593542873859406,
        0.03394605591893196,
        0.034045662730932236,
        0.03444153815507889,
        0.03455357626080513,
        0.03483813256025314,
        0.03491735830903053,
        0.03510763868689537,
        0.03521261364221573,
        0.03553535044193268,
        0.03560211881995201,
        0.03590916842222214,
        0.0359802283346653,
        0.036534491926431656,
        0.03664350509643555,
        0.036974843591451645,
        0.03706779330968857,
        0.037423327565193176,
        0.0375242717564106,
        0.037971291691064835,
        0.03805854171514511,
        0.03832196444272995,
        0.038408540189266205,
        0.03889527916908264,
        0.039021097123622894
      ]
    },
    {
      "activation_grad_rms_by_block": [
        7.223192369565368e-05,
        6.862939335405827e-05,
        6.46814878564328e-05,
        6.090753959142603e-05,
        5.703999340767041e-05,
        5.302598947309889e-05,
        4.9269601731793955e-05,
        4.545194315141998e-05,
        4.1738676372915506e-05,
        3.8814047002233565e-05,
        3.584829028113745e-05,
        3.317802838864736e-05,
        3.07806913042441e-05,
        2.884516288759187e-05,
        2.6981158953276463e-05,
        2.530650635890197e-05,
        2.3913053155411035e-05,
        2.265583498228807e-05,
        2.164491161238402e-05,
        2.069839865725953e-05,
        1.9972700101789087e-05,
        1.928487472468987e-05,
        1.8700015061767772e-05,
        1.8183285646955483e-05,
        1.7775813830667175e-05,
        1.7374133676639758e-05,
        1.7024083717842586e-05,
        1.673021324677393e-05,
        1.646951204747893e-05,
        1.624442847969476e-05,
        1.604618228157051e-05,
        1.58860220835777e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 5.890473403269425e-05,
        "first_to_last_ratio": 3.528539860533838,
        "imbalance_abs_log_ratio": 1.2608841481311845,
        "last_quartile_mean": 1.669379867053067e-05,
        "mean": 3.222918428491539e-05,
        "normalized": [
          2.2411961487173366,
          2.1294176342582674,
          2.0069228958644927,
          1.8898256640002307,
          1.7698242966194933,
          1.64527867054697,
          1.528726302727252,
          1.4102728368677142,
          1.2950584167422121,
          1.2043136636380887,
          1.1122928202038285,
          1.029440524939878,
          0.9550564802426835,
          0.8950013327235407,
          0.8371654310191393,
          0.7852046808006393,
          0.7419689230733446,
          0.7029602357293278,
          0.6715935290523237,
          0.6422253344757239,
          0.6197085202413003,
          0.5983668266067783,
          0.5802199303728628,
          0.5641869644049918,
          0.5515440190333021,
          0.5390807760769664,
          0.5282195033961989,
          0.5191013554322059,
          0.5110123762948401,
          0.5040285331483811,
          0.49787739397055714,
          0.49290797877912856
        ],
        "population_cv": 0.5370990514498413
      },
      "activation_output_rms_by_block": [
        0.029305610805749893,
        0.03049159236252308,
        0.0324031226336956,
        0.03488599509000778,
        0.037802305072546005,
        0.04105908423662186,
        0.04460485652089119,
        0.04878050833940506,
        0.053197599947452545,
        0.057943083345890045,
        0.06266378611326218,
        0.06765592098236084,
        0.07297883927822113,
        0.07823853194713593,
        0.08444638550281525,
        0.09052036702632904,
        0.09696295112371445,
        0.10418187081813812,
        0.11156722903251648,
        0.1197449341416359,
        0.12838369607925415,
        0.13761533796787262,
        0.14720149338245392,
        0.15857642889022827,
        0.16927987337112427,
        0.1816006898880005,
        0.19378916919231415,
        0.20805081725120544,
        0.22281049191951752,
        0.2388531118631363,
        0.25510889291763306,
        0.2763713300228119
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.03741663438268006,
        "first_to_last_ratio": 0.17145265067740043,
        "imbalance_abs_log_ratio": 1.763448139840426,
        "last_quartile_mean": 0.2182330470532179,
        "mean": 0.11303362209582701,
        "normalized": [
          0.2592645467992289,
          0.2697568369230271,
          0.286668002253569,
          0.30863378916082446,
          0.3344341654423688,
          0.36324664710658405,
          0.39461582928906175,
          0.4315575085973109,
          0.47063518766436574,
          0.5126181243379725,
          0.554381828622084,
          0.5985468724076087,
          0.6456383324277757,
          0.6921704400555022,
          0.747090856127956,
          0.8008269163451932,
          0.8578239759627604,
          0.9216892185390237,
          0.9870269302520681,
          1.0593744756769734,
          1.135800956377508,
          1.2174726016583441,
          1.3022806015864932,
          1.4029138052020595,
          1.497606377928977,
          1.6066077200821192,
          1.714438284814271,
          1.8406100184494245,
          1.9711877562467606,
          2.1131156149330748,
          2.256929294023315,
          2.445036484706394
        ],
        "population_cv": 0.6334590447760624
      },
      "bits_per_byte": 5.11848367743279,
      "branch_output_rms_by_sublayer": [
        0.003032666863873601,
        0.004902590066194534,
        0.002639081096276641,
        0.004850045777857304,
        0.0032534960191696882,
        0.005442064255475998,
        0.0027798404917120934,
        0.005710980389267206,
        0.0031097980681806803,
        0.006003146059811115,
        0.0030368315055966377,
        0.0059938253834843636,
        0.0034559990745037794,
        0.0061976369470357895,
        0.0037613045424222946,
        0.0062723769806325436,
        0.004607585724443197,
        0.0063410610891878605,
        0.003909118939191103,
        0.006816320586949587,
        0.004674337804317474,
        0.006481682416051626,
        0.004910735413432121,
        0.0065277558751404285,
        0.00493645528331399,
        0.006758649833500385,
        0.004751654341816902,
        0.006488988175988197,
        0.006120393518358469,
        0.006713794078677893,
        0.005455858074128628,
        0.00696563022211194,
        0.005903614219278097,
        0.007100075017660856,
        0.006780208554118872,
        0.007398899644613266,
        0.007146476302295923,
        0.007193166296929121,
        0.007645847275853157,
        0.00761415995657444,
        0.00879710540175438,
        0.007752604782581329,
        0.00870877131819725,
        0.007873632945120335,
        0.008980454877018929,
        0.008168715983629227,
        0.011330032721161842,
        0.008200661279261112,
        0.010545967146754265,
        0.007979212328791618,
        0.011436101980507374,
        0.008451862260699272,
        0.01130830217152834,
        0.008476903662085533,
        0.013203900307416916,
        0.008634857833385468,
        0.01393965259194374,
        0.009264753200113773,
        0.014520598575472832,
        0.009397287853062153,
        0.015960348770022392,
        0.00835501030087471,
        0.02132205106317997,
        0.009183173067867756
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00028524855645157025,
        0.00022469127596424853,
        0.00022670239393809648,
        0.0001993403227560908,
        0.0002100203582478305,
        0.00019183315773228503,
        0.00020704106188439305,
        0.00020121077118127333,
        0.00020796371696885172,
        0.00017825031863398375,
        0.00018469615204762195,
        0.0001848551977445252,
        0.00018214824531137669,
        0.0001649843875278976,
        0.0001966104199015976,
        0.00016919176207710993,
        0.0001681445657707544,
        0.00017333427923610245,
        0.0001624042419733422,
        0.00015992743704390023,
        0.00016485453322609178,
        0.00015449118786750607,
        0.00015016403044324942,
        0.00015560032515501767,
        0.00015156764536984935,
        0.00014581309890539414,
        0.000137484036700603,
        0.0001500871171184727,
        0.00014447415143049674,
        0.00014218802742214665,
        0.00014221421491139387,
        0.00016697872359292516
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.0002182609872694735,
        "first_to_last_ratio": 1.4787241905811908,
        "imbalance_abs_log_ratio": 0.3911796826204884,
        "last_quartile_mean": 0.0001476008769314102,
        "mean": 0.00017764111607924995,
        "normalized": [
          1.6057575112527105,
          1.2648607536557492,
          1.2761819951466593,
          1.122151938446279,
          1.1822733547459552,
          1.0798916487706802,
          1.1655019201299495,
          1.1326813049942133,
          1.170695847666675,
          1.0034294006262718,
          1.0397151072008834,
          1.0406104274984227,
          1.0253721060281789,
          0.9287511313219521,
          1.1067844215405909,
          0.9524358201038854,
          0.9465408092557868,
          0.9757554054027329,
          0.9142266472863736,
          0.9002839014620511,
          0.9280201389443322,
          0.869681475084765,
          0.8453224892837178,
          0.8759251719945322,
          0.8532238972323919,
          0.8208296712138615,
          0.7739426532271284,
          0.8448895189980412,
          0.8132922975221759,
          0.8004229570293466,
          0.8005703749797919,
          0.9399779019539145
        ],
        "population_cv": 0.17622384112301237
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.028563687577843666,
        0.028535494580864906,
        0.029305610805749893,
        0.029406530782580376,
        0.03049159236252308,
        0.030669530853629112,
        0.0324031226336956,
        0.03261565789580345,
        0.03488599509000778,
        0.035065844655036926,
        0.037802305072546005,
        0.03803478553891182,
        0.04105908423662186,
        0.041278183460235596,
        0.04460485652089119,
        0.04512108862400055,
        0.04878050833940506,
        0.04953810200095177,
        0.053197599947452545,
        0.05369982123374939,
        0.057943083345890045,
        0.0585285909473896,
        0.06266378611326218,
        0.06350461393594742,
        0.06765592098236084,
        0.06854932755231857,
        0.07297883927822113,
        0.07389408349990845,
        0.07823853194713593,
        0.07984820008277893,
        0.08444638550281525,
        0.08570227771997452,
        0.09052036702632904,
        0.09210614860057831,
        0.09696295112371445,
        0.09899988770484924,
        0.10418187081813812,
        0.10663887858390808,
        0.11156722903251648,
        0.11457803845405579,
        0.1197449341416359,
        0.12300471216440201,
        0.12838369607925415,
        0.13199855387210846,
        0.13761533796787262,
        0.14150823652744293,
        0.14720149338245392,
        0.15258266031742096,
        0.15857642889022827,
        0.16335271298885345,
        0.16927987337112427,
        0.17534348368644714,
        0.1816006898880005,
        0.1876170039176941,
        0.19378916919231415,
        0.2015211284160614,
        0.20805081725120544,
        0.21581925451755524,
        0.22281049191951752,
        0.23164939880371094,
        0.2388531118631363,
        0.24887791275978088,
        0.25510889291763306,
        0.26933422684669495
      ],
      "loss_nats": 3.5478625297546387,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 100,
      "stream_state_rms_by_sublayer": [
        0.028535494580864906,
        0.029305610805749893,
        0.029406530782580376,
        0.03049159236252308,
        0.030669530853629112,
        0.0324031226336956,
        0.03261565789580345,
        0.03488599509000778,
        0.035065844655036926,
        0.037802305072546005,
        0.03803478553891182,
        0.04105908423662186,
        0.041278183460235596,
        0.04460485652089119,
        0.04512108862400055,
        0.04878050833940506,
        0.04953810200095177,
        0.053197599947452545,
        0.05369982123374939,
        0.057943083345890045,
        0.0585285909473896,
        0.06266378611326218,
        0.06350461393594742,
        0.06765592098236084,
        0.06854932755231857,
        0.07297883927822113,
        0.07389408349990845,
        0.07823853194713593,
        0.07984820008277893,
        0.08444638550281525,
        0.08570227771997452,
        0.09052036702632904,
        0.09210614860057831,
        0.09696295112371445,
        0.09899988770484924,
        0.10418187081813812,
        0.10663887858390808,
        0.11156722903251648,
        0.11457803845405579,
        0.1197449341416359,
        0.12300471216440201,
        0.12838369607925415,
        0.13199855387210846,
        0.13761533796787262,
        0.14150823652744293,
        0.14720149338245392,
        0.15258266031742096,
        0.15857642889022827,
        0.16335271298885345,
        0.16927987337112427,
        0.17534348368644714,
        0.1816006898880005,
        0.1876170039176941,
        0.19378916919231415,
        0.2015211284160614,
        0.20805081725120544,
        0.21581925451755524,
        0.22281049191951752,
        0.23164939880371094,
        0.2388531118631363,
        0.24887791275978088,
        0.25510889291763306,
        0.26933422684669495,
        0.2763713300228119
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.00014093925710767508,
        0.00014282441406976432,
        0.00014458753867074847,
        0.0001413008722011,
        0.00013349838263820857,
        0.00012304198753554374,
        0.00011454652121756226,
        0.00010784466576296836,
        0.0001000517513602972,
        9.189492993755266e-05,
        8.563220035284758e-05,
        8.04013543529436e-05,
        7.555109914392233e-05,
        7.068048580549657e-05,
        6.581390334758908e-05,
        6.151734851300716e-05,
        5.7112174545181915e-05,
        5.2947088988730684e-05,
        4.8879119276534766e-05,
        4.527245255303569e-05,
        4.1345181671204045e-05,
        3.832591755781323e-05,
        3.484322223812342e-05,
        3.232451854273677e-05,
        3.010137334058527e-05,
        2.8228545488673262e-05,
        2.6716850697994232e-05,
        2.5478970201220363e-05,
        2.4533494070055895e-05,
        2.4085495169856586e-05,
        2.3973674615263008e-05,
        2.415898779872805e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00013107295490044635,
        "first_to_last_ratio": 5.058842318548806,
        "imbalance_abs_log_ratio": 1.621137666322135,
        "last_quartile_mean": 2.5909673922797083e-05,
        "mean": 6.995168058665513e-05,
        "normalized": [
          2.01480873548251,
          2.041758151797876,
          2.06696304446375,
          2.019978233776078,
          1.9084371028489115,
          1.7589568471213655,
          1.6375092100276802,
          1.5417022844700914,
          1.430298036032952,
          1.3136915248764403,
          1.2241621592889058,
          1.14938417031087,
          1.0800469482692516,
          1.010418694915251,
          0.9408480653450457,
          0.8794263125215463,
          0.8164517859500485,
          0.756909462999114,
          0.6987554675828556,
          0.6471960669615769,
          0.591053443240527,
          0.5478913049177658,
          0.49810414769034944,
          0.46209781197027316,
          0.4303166569857621,
          0.4035434924784137,
          0.38193293533380923,
          0.3642367129358298,
          0.3507205811826659,
          0.34431617607842635,
          0.34271763614835193,
          0.34536679599570513
        ],
        "population_cv": 0.5909500348424317
      },
      "activation_output_rms_by_block": [
        0.04959565028548241,
        0.05870900675654411,
        0.06652694195508957,
        0.07286322116851807,
        0.07801168411970139,
        0.08261121064424515,
        0.08867742121219635,
        0.09177010506391525,
        0.09494514763355255,
        0.0981190949678421,
        0.10163456201553345,
        0.10214314609766006,
        0.10418473184108734,
        0.10623370856046677,
        0.10849270969629288,
        0.11100933700799942,
        0.11405495554208755,
        0.11736668646335602,
        0.12177617102861404,
        0.12738163769245148,
        0.13297238945960999,
        0.14011575281620026,
        0.15017743408679962,
        0.16121913492679596,
        0.17427310347557068,
        0.189283087849617,
        0.2044389545917511,
        0.22237628698349,
        0.24115440249443054,
        0.26191622018814087,
        0.2883519232273102,
        0.3199674189090729
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.07359565515071154,
        "first_to_last_ratio": 0.309589437408787,
        "imbalance_abs_log_ratio": 1.1725082547452423,
        "last_quartile_mean": 0.2377201747149229,
        "mean": 0.13694853871129453,
        "normalized": [
          0.3621480795062482,
          0.4286939262660582,
          0.48578059014807806,
          0.5320481828734462,
          0.5696423258970309,
          0.6032281280372068,
          0.6475236760221296,
          0.6701064934864231,
          0.6932906953736058,
          0.7164668998381211,
          0.7421368856647125,
          0.745850573206854,
          0.7607582586968847,
          0.775719913188861,
          0.7922151686847111,
          0.8105916138471667,
          0.8328307597537082,
          0.8570130617514684,
          0.8892111750482725,
          0.9301423650893323,
          0.9709661067646236,
          1.0231270383365143,
          1.0965975650415178,
          1.177224207310945,
          1.2725444544024032,
          1.3821475543353592,
          1.4928158892059098,
          1.6237945222058072,
          1.7609125644108963,
          1.9125156255978355,
          2.105549470923482,
          2.336406229084387
        ],
        "population_cv": 0.48939845088217687
      },
      "bits_per_byte": 3.103317553853433,
      "branch_output_rms_by_sublayer": [
        0.011270907707512379,
        0.01699911244213581,
        0.012035840190947056,
        0.01337367296218872,
        0.010826056823134422,
        0.010907532647252083,
        0.01394603680819273,
        0.008780849166214466,
        0.015218054875731468,
        0.009724137373268604,
        0.015542514622211456,
        0.009259181097149849,
        0.025958625599741936,
        0.009840037673711777,
        0.03120059333741665,
        0.009063811041414738,
        0.014079542830586433,
        0.008434661664068699,
        0.01716585084795952,
        0.009029604494571686,
        0.021518073976039886,
        0.01014146488159895,
        0.030795104801654816,
        0.00910214427858591,
        0.018123170360922813,
        0.009148034267127514,
        0.016974516212940216,
        0.00891866721212864,
        0.01868187077343464,
        0.00946173444390297,
        0.017949005588889122,
        0.00928061455488205,
        0.021634839475154877,
        0.01075725443661213,
        0.02261096052825451,
        0.010938199236989021,
        0.017853887751698494,
        0.012016859836876392,
        0.016204316169023514,
        0.012549139559268951,
        0.019116709008812904,
        0.014130656607449055,
        0.021943919360637665,
        0.015834644436836243,
        0.015276721678674221,
        0.01847725547850132,
        0.01698230765759945,
        0.019000399857759476,
        0.017433995380997658,
        0.019877459853887558,
        0.017341874539852142,
        0.022254064679145813,
        0.018439622595906258,
        0.023488705977797508,
        0.013915229588747025,
        0.02547476813197136,
        0.008195439353585243,
        0.027127297595143318,
        0.007334497291594744,
        0.02754027582705021,
        0.014406879432499409,
        0.029805582016706467,
        0.02094510942697525,
        0.03139834478497505
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0005239687790313407,
        0.0004396407496139251,
        0.0004479555677531567,
        0.000484081397596248,
        0.0004298002646761996,
        0.0003778969963663625,
        0.0003311985651517251,
        0.0003381159607123189,
        0.00027120323619484496,
        0.00025249768611013433,
        0.0002492173720055997,
        0.0002512649978189958,
        0.00021872457711399188,
        0.0001974354513181695,
        0.0001883229322305711,
        0.00018548264077727127,
        0.00019625685908145198,
        0.00017137558371316787,
        0.00015305597823747372,
        0.00014260825992090635,
        0.00013921721401447933,
        0.00013266258711253505,
        0.00012717821667718618,
        0.00011256530678442586,
        0.0001087245205863228,
        0.00010436458537579927,
        0.00010507936295752309,
        9.703474953173527e-05,
        9.385369360633854e-05,
        9.122967598255955e-05,
        0.00010232602359400015,
        0.0001188300325189452
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.0004215822851126596,
        "first_to_last_ratio": 4.105774523524949,
        "imbalance_abs_log_ratio": 1.4123944031885163,
        "last_quartile_mean": 0.00010268033051915298,
        "mean": 0.00022447405700517828,
        "normalized": [
          2.3342063934775927,
          1.9585370152764836,
          1.995578347580821,
          2.156513781835738,
          1.9146990543601428,
          1.6834773755509973,
          1.475442505786247,
          1.506258519239571,
          1.2081718476206305,
          1.1248412822347198,
          1.1102279494144423,
          1.11934983120655,
          0.9743868847567816,
          0.8795468570054622,
          0.8389518804225415,
          0.8262987859349487,
          0.874296396206376,
          0.7634538529733728,
          0.6818426159328645,
          0.6352995166725061,
          0.6201928894227087,
          0.5909929587519093,
          0.5665608684314569,
          0.5014624330589309,
          0.48435227677029374,
          0.4649293854629791,
          0.4681136179362547,
          0.4322760092026885,
          0.41810485745430026,
          0.4064152321194739,
          0.45584788264257925,
          0.5293708952576376
        ],
        "population_cv": 0.5698198347315865
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.044032808393239975,
        0.044310975819826126,
        0.04959565028548241,
        0.052685707807540894,
        0.05870900675654411,
        0.061390094459056854,
        0.06652694195508957,
        0.0693279355764389,
        0.07286322116851807,
        0.07403621077537537,
        0.07801168411970139,
        0.07902701944112778,
        0.08261121064424515,
        0.08570802956819534,
        0.08867742121219635,
        0.08839071542024612,
        0.09177010506391525,
        0.0920921340584755,
        0.09494514763355255,
        0.09543459117412567,
        0.0981190949678421,
        0.09975367039442062,
        0.10163456201553345,
        0.09967570006847382,
        0.10214314609766006,
        0.10169660300016403,
        0.10418473184108734,
        0.1038530021905899,
        0.10623370856046677,
        0.10576826333999634,
        0.10849270969629288,
        0.10825389623641968,
        0.11100933700799942,
        0.11050844192504883,
        0.11405495554208755,
        0.11365467309951782,
        0.11736668646335602,
        0.117034412920475,
        0.12177617102861404,
        0.12205220758914948,
        0.12738163769245148,
        0.12672542035579681,
        0.13297238945960999,
        0.13195016980171204,
        0.14011575281620026,
        0.13969174027442932,
        0.15017743408679962,
        0.15003392100334167,
        0.16121913492679596,
        0.16189023852348328,
        0.17427310347557068,
        0.1747252345085144,
        0.189283087849617,
        0.18889831006526947,
        0.2044389545917511,
        0.20492415130138397,
        0.22237628698349,
        0.22264817357063293,
        0.24115440249443054,
        0.2428991198539734,
        0.26191622018814087,
        0.267524778842926,
        0.2883519232273102,
        0.2990366220474243
      ],
      "loss_nats": 2.1510558128356934,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 500,
      "stream_state_rms_by_sublayer": [
        0.044310975819826126,
        0.04959565028548241,
        0.052685707807540894,
        0.05870900675654411,
        0.061390094459056854,
        0.06652694195508957,
        0.0693279355764389,
        0.07286322116851807,
        0.07403621077537537,
        0.07801168411970139,
        0.07902701944112778,
        0.08261121064424515,
        0.08570802956819534,
        0.08867742121219635,
        0.08839071542024612,
        0.09177010506391525,
        0.0920921340584755,
        0.09494514763355255,
        0.09543459117412567,
        0.0981190949678421,
        0.09975367039442062,
        0.10163456201553345,
        0.09967570006847382,
        0.10214314609766006,
        0.10169660300016403,
        0.10418473184108734,
        0.1038530021905899,
        0.10623370856046677,
        0.10576826333999634,
        0.10849270969629288,
        0.10825389623641968,
        0.11100933700799942,
        0.11050844192504883,
        0.11405495554208755,
        0.11365467309951782,
        0.11736668646335602,
        0.117034412920475,
        0.12177617102861404,
        0.12205220758914948,
        0.12738163769245148,
        0.12672542035579681,
        0.13297238945960999,
        0.13195016980171204,
        0.14011575281620026,
        0.13969174027442932,
        0.15017743408679962,
        0.15003392100334167,
        0.16121913492679596,
        0.16189023852348328,
        0.17427310347557068,
        0.1747252345085144,
        0.189283087849617,
        0.18889831006526947,
        0.2044389545917511,
        0.20492415130138397,
        0.22237628698349,
        0.22264817357063293,
        0.24115440249443054,
        0.2428991198539734,
        0.26191622018814087,
        0.267524778842926,
        0.2883519232273102,
        0.2990366220474243,
        0.3199674189090729
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.00010639085667207837,
        0.00010674773511709645,
        0.000114928770926781,
        0.00011333057773299515,
        0.00011114918743260205,
        0.00010892991122091189,
        0.00010597817890811712,
        0.00010435423610033467,
        0.00010266973549732938,
        0.0001015385496430099,
        0.00010008511890191585,
        9.485893679084256e-05,
        9.186805982608348e-05,
        8.913136116461828e-05,
        8.655320561956614e-05,
        8.490345499012619e-05,
        8.653065015096217e-05,
        8.836571214487776e-05,
        8.356331090908498e-05,
        8.248459926107898e-05,
        7.508730777772143e-05,
        7.080488285282627e-05,
        6.500465678982437e-05,
        6.0319798649288714e-05,
        5.538603363675065e-05,
        5.071960185887292e-05,
        4.5680528273805976e-05,
        4.0155850001610816e-05,
        3.449051291681826e-05,
        2.892977681767661e-05,
        2.4221473722718656e-05,
        2.1527655917452648e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00010897618176386459,
        "first_to_last_ratio": 2.8953050536910427,
        "imbalance_abs_log_ratio": 1.0630904780785062,
        "last_quartile_mean": 3.763892914321332e-05,
        "mean": 7.927156963205562e-05,
        "normalized": [
          1.3421060938480058,
          1.3466080665814153,
          1.4498107134781197,
          1.429649725103549,
          1.4021317850587363,
          1.3741359194288378,
          1.3369002203440912,
          1.3164144041136308,
          1.29516465958575,
          1.2808949048733107,
          1.262560075023968,
          1.1966325030668217,
          1.1589029995557718,
          1.1243799205481557,
          1.0918568412522767,
          1.0710454628842516,
          1.0915723071032915,
          1.1147213629682524,
          1.054139728744489,
          1.0405319289618822,
          0.9472161094607345,
          0.8931939052231708,
          0.8200248473891447,
          0.760926003222431,
          0.6986872329364575,
          0.6398208348124227,
          0.5762536112989216,
          0.5065605511281904,
          0.4350930992903045,
          0.3649451745683369,
          0.305550575510795,
          0.2715684326344833
        ],
        "population_cv": 0.35248901334980626
      },
      "activation_output_rms_by_block": [
        0.09771591424942017,
        0.13586144149303436,
        0.1991395801305771,
        0.2241763025522232,
        0.22696681320667267,
        0.22763192653656006,
        0.22912801802158356,
        0.2290782928466797,
        0.22818906605243683,
        0.22590669989585876,
        0.22214065492153168,
        0.21565376222133636,
        0.20698806643486023,
        0.20169880986213684,
        0.1979832947254181,
        0.1976672261953354,
        0.2011667788028717,
        0.20515663921833038,
        0.205980584025383,
        0.20982402563095093,
        0.2095191776752472,
        0.2096652388572693,
        0.2105090171098709,
        0.2148621529340744,
        0.2245095670223236,
        0.2386746108531952,
        0.25724881887435913,
        0.2833212912082672,
        0.3200184106826782,
        0.3660464584827423,
        0.43435168266296387,
        0.5434449911117554
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.19621228612959385,
        "first_to_last_ratio": 0.5884274155428802,
        "imbalance_abs_log_ratio": 0.5303016979635413,
        "last_quartile_mean": 0.3334519788622856,
        "mean": 0.23750704107806087,
        "normalized": [
          0.4114232311003534,
          0.5720312159014314,
          0.8384575852011324,
          0.9438722386278381,
          0.9556214088494162,
          0.9584218030055994,
          0.9647209488255828,
          0.9645115858750019,
          0.960767584054228,
          0.951157906184392,
          0.9353013448073788,
          0.9079889221071891,
          0.8715028636428086,
          0.8492329698800171,
          0.8335891594066359,
          0.8322583839961553,
          0.8469929055145556,
          0.8637918197587332,
          0.8672609582032722,
          0.8834433904719,
          0.8821598581845203,
          0.8827748344031583,
          0.8863274796164187,
          0.9046559291834054,
          0.9452754158498172,
          1.0049159375226715,
          1.0831208106786603,
          1.1928963870807885,
          1.3474059936500937,
          1.5412025547589332,
          1.8287949725254948,
          2.288121601132417
        ],
        "population_cv": 0.33570692978684213
      },
      "bits_per_byte": 2.0753701983577404,
      "branch_output_rms_by_sublayer": [
        0.017155639827251434,
        0.059614092111587524,
        0.05783598870038986,
        0.017162656411528587,
        0.08590122312307358,
        0.003484122920781374,
        0.037979111075401306,
        0.0025395466946065426,
        0.02590934932231903,
        0.004171412438154221,
        0.027761045843362808,
        0.005880473647266626,
        0.021641533821821213,
        0.0071118478663265705,
        0.01924360916018486,
        0.00746425986289978,
        0.022704657167196274,
        0.00924856960773468,
        0.018269814550876617,
        0.010196910239756107,
        0.022024810314178467,
        0.012587273493409157,
        0.03360125049948692,
        0.015558412298560143,
        0.026457328349351883,
        0.018742306157946587,
        0.024426763877272606,
        0.01840486004948616,
        0.02595876343548298,
        0.019045589491724968,
        0.023261789232492447,
        0.01923760399222374,
        0.0244763121008873,
        0.01806296780705452,
        0.0326010137796402,
        0.017410961911082268,
        0.037907715886831284,
        0.019881773740053177,
        0.034735601395368576,
        0.020177578553557396,
        0.04502493515610695,
        0.02574281580746174,
        0.034673795104026794,
        0.027684781700372696,
        0.0429018959403038,
        0.03708433732390404,
        0.03208737075328827,
        0.0411839634180069,
        0.027529660612344742,
        0.04565756395459175,
        0.02247321978211403,
        0.048749130219221115,
        0.01957971602678299,
        0.05322333425283432,
        0.02326148748397827,
        0.06035253033041954,
        0.014175337739288807,
        0.07119441032409668,
        0.011726614087820053,
        0.07984230667352676,
        0.013617267832159996,
        0.10812961310148239,
        0.01758233644068241,
        0.16684016585350037
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00037331716215226093,
        0.0002052857320094383,
        0.00022573696371192266,
        0.00015002318346068796,
        0.00011232796492243865,
        0.00011469567077281986,
        0.0001328622706326584,
        0.00011987742098506258,
        0.00012268315232766492,
        0.00013855849890850403,
        0.00016581239426236876,
        0.00020970093409220108,
        0.00019415593885306704,
        0.0001873084275669645,
        0.00019997003245353254,
        0.00019081336552079618,
        0.00019544152058884818,
        0.0002331889851483562,
        0.00028258169221688257,
        0.00026717547757069314,
        0.00031847213451059053,
        0.00028331189819324093,
        0.0003090653888381074,
        0.000275741819401602,
        0.000255221776165695,
        0.00023469865916502512,
        0.00021558263955965886,
        0.0002009260839676739,
        0.00018496297858040776,
        0.0001671377375581853,
        0.00015254274522576858,
        0.00014770686125523138
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00017926579608091117,
        "first_to_last_ratio": 0.9200315924660546,
        "imbalance_abs_log_ratio": 0.08334726989162704,
        "last_quartile_mean": 0.00019484743518470574,
        "mean": 0.0002052152347055736,
        "normalized": [
          1.8191493564689118,
          1.0003435286077362,
          1.1000010015620527,
          0.7310528561679606,
          0.5473665982138111,
          0.5589042691561181,
          0.6474288852057137,
          0.584154588508271,
          0.5978267281358565,
          0.6751862214678978,
          0.8079926156567426,
          1.021858510617222,
          0.946108797096017,
          0.9127413363617942,
          0.9744404831368078,
          0.9298206626547972,
          0.9523733502010805,
          1.1363141994936055,
          1.377001530233894,
          1.301928085183418,
          1.5518932352537513,
          1.3805597747151384,
          1.5060547979370522,
          1.3436712912528852,
          1.2436785043365166,
          1.1436707391717382,
          1.0505196647264496,
          0.9790992576937408,
          0.9013121216160088,
          0.8144509241625318,
          0.7433305107421713,
          0.7197655742623075
        ],
        "population_cv": 0.31225216605899336
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.05658925697207451,
        0.0616849884390831,
        0.09771591424942017,
        0.1262788027524948,
        0.13586144149303436,
        0.19808362424373627,
        0.1991395801305771,
        0.22383953630924225,
        0.2241763025522232,
        0.22702015936374664,
        0.22696681320667267,
        0.227730393409729,
        0.22763192653656006,
        0.22866052389144897,
        0.22912801802158356,
        0.22953395545482635,
        0.2290782928466797,
        0.22848571836948395,
        0.22818906605243683,
        0.22744755446910858,
        0.22590669989585876,
        0.22389201819896698,
        0.22214065492153168,
        0.21693368256092072,
        0.21565376222133636,
        0.209661602973938,
        0.20698806643486023,
        0.20248256623744965,
        0.20169880986213684,
        0.19645103812217712,
        0.1979832947254181,
        0.1954326629638672,
        0.1976672261953354,
        0.19818583130836487,
        0.2011667788028717,
        0.20395968854427338,
        0.20515663921833038,
        0.2042165845632553,
        0.205980584025383,
        0.20741206407546997,
        0.20982402563095093,
        0.20696893334388733,
        0.2095191776752472,
        0.20577766001224518,
        0.2096652388572693,
        0.20420841872692108,
        0.2105090171098709,
        0.20596569776535034,
        0.2148621529340744,
        0.21317537128925323,
        0.2245095670223236,
        0.22439339756965637,
        0.2386746108531952,
        0.23865793645381927,
        0.25724881887435913,
        0.25777244567871094,
        0.2833212912082672,
        0.28324905037879944,
        0.3200184106826782,
        0.3202260434627533,
        0.3660464584827423,
        0.366729199886322,
        0.43435168266296387,
        0.4362008571624756
      ],
      "loss_nats": 1.4385370016098022,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 2000,
      "stream_state_rms_by_sublayer": [
        0.0616849884390831,
        0.09771591424942017,
        0.1262788027524948,
        0.13586144149303436,
        0.19808362424373627,
        0.1991395801305771,
        0.22383953630924225,
        0.2241763025522232,
        0.22702015936374664,
        0.22696681320667267,
        0.227730393409729,
        0.22763192653656006,
        0.22866052389144897,
        0.22912801802158356,
        0.22953395545482635,
        0.2290782928466797,
        0.22848571836948395,
        0.22818906605243683,
        0.22744755446910858,
        0.22590669989585876,
        0.22389201819896698,
        0.22214065492153168,
        0.21693368256092072,
        0.21565376222133636,
        0.209661602973938,
        0.20698806643486023,
        0.20248256623744965,
        0.20169880986213684,
        0.19645103812217712,
        0.1979832947254181,
        0.1954326629638672,
        0.1976672261953354,
        0.19818583130836487,
        0.2011667788028717,
        0.20395968854427338,
        0.20515663921833038,
        0.2042165845632553,
        0.205980584025383,
        0.20741206407546997,
        0.20982402563095093,
        0.20696893334388733,
        0.2095191776752472,
        0.20577766001224518,
        0.2096652388572693,
        0.20420841872692108,
        0.2105090171098709,
        0.20596569776535034,
        0.2148621529340744,
        0.21317537128925323,
        0.2245095670223236,
        0.22439339756965637,
        0.2386746108531952,
        0.23865793645381927,
        0.25724881887435913,
        0.25777244567871094,
        0.2833212912082672,
        0.28324905037879944,
        0.3200184106826782,
        0.3202260434627533,
        0.3660464584827423,
        0.366729199886322,
        0.43435168266296387,
        0.4362008571624756,
        0.5434449911117554
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.0001095466359402053,
        0.00011157916014781222,
        0.0001209566617035307,
        0.00011857408389914781,
        0.00011687575170071796,
        0.00011450610327301547,
        0.00011203528265468776,
        0.00011082263517891988,
        0.00010936566832242534,
        0.00010819115414051339,
        0.00010705165914259851,
        0.00010176753858104348,
        9.876617696136236e-05,
        9.541869076201692e-05,
        9.238059283234179e-05,
        9.015990508487448e-05,
        9.102027252083644e-05,
        9.192643483402207e-05,
        8.731557318242267e-05,
        8.749608969083056e-05,
        7.95004380051978e-05,
        7.466241368092597e-05,
        6.735928764101118e-05,
        6.187916733324528e-05,
        5.64747242606245e-05,
        5.17304906679783e-05,
        4.670304406317882e-05,
        4.1035571484826505e-05,
        3.507353176246397e-05,
        2.8758000553352758e-05,
        2.3154751033871435e-05,
        1.9675369912874885e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00011436203931225464,
        "first_to_last_ratio": 3.02339634825199,
        "imbalance_abs_log_ratio": 1.1063808181034647,
        "last_quartile_mean": 3.78256854673964e-05,
        "mean": 8.318008940477739e-05,
        "normalized": [
          1.3169814642434559,
          1.341416689333394,
          1.45415402374626,
          1.4255104165869963,
          1.4050928838507029,
          1.376604715051423,
          1.3469002432720767,
          1.3323216646188754,
          1.3148058520377592,
          1.300685715953312,
          1.286986584273256,
          1.223460317358093,
          1.1873776244786027,
          1.1471337845969738,
          1.1106094438392846,
          1.0839120963928204,
          1.0942555264386233,
          1.1051495074341955,
          1.0497172316985723,
          1.0518874206187772,
          0.955762834280288,
          0.8975995844101339,
          0.8098006160250947,
          0.743918019036071,
          0.6789452219244779,
          0.6219095343387212,
          0.5614690293960716,
          0.49333406321717244,
          0.4216577790844446,
          0.34573178219860223,
          0.2783689125554313,
          0.23653941771003728
        ],
        "population_cv": 0.36398106608216985
      },
      "activation_output_rms_by_block": [
        0.11107808351516724,
        0.1828140765428543,
        0.2578570544719696,
        0.27254435420036316,
        0.2737041115760803,
        0.2742544412612915,
        0.27536308765411377,
        0.2744285464286804,
        0.2742423117160797,
        0.27115195989608765,
        0.26583269238471985,
        0.255132257938385,
        0.23856717348098755,
        0.22743643820285797,
        0.22118839621543884,
        0.22011704742908478,
        0.22253327071666718,
        0.2256278693675995,
        0.22584785521030426,
        0.22925184667110443,
        0.23080435395240784,
        0.23398378491401672,
        0.24050307273864746,
        0.24922524392604828,
        0.2630935311317444,
        0.27895617485046387,
        0.2995731830596924,
        0.3278428912162781,
        0.36820048093795776,
        0.42407870292663574,
        0.5123529434204102,
        0.6704596281051636
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.24025546945631504,
        "first_to_last_ratio": 0.6112286812568156,
        "imbalance_abs_log_ratio": 0.4922841160907453,
        "last_quartile_mean": 0.39306969195604324,
        "mean": 0.2780639645643532,
        "normalized": [
          0.39946953820209985,
          0.6574533195240588,
          0.9273299935716515,
          0.9801498537480837,
          0.9843206832100536,
          0.9862998310153919,
          0.9902868503134846,
          0.9869259645298935,
          0.9862562095945767,
          0.975142393301143,
          0.9560127390156568,
          0.9175308218672072,
          0.8579578941656613,
          0.817928488357655,
          0.7954586872196038,
          0.7916058011110692,
          0.8002952524442108,
          0.8114243415938267,
          0.8122154755440653,
          0.824457232458282,
          0.8300405063777765,
          0.8414746775282531,
          0.8649199586701105,
          0.896287457875072,
          0.9461619075450383,
          1.0032086512450777,
          1.077353491413524,
          1.179019696888499,
          1.3241574884211362,
          1.5251120496359385,
          1.842572244926166,
          2.411170498685733
        ],
        "population_cv": 0.35267348500337414
      },
      "bits_per_byte": 1.9134731260894795,
      "branch_output_rms_by_sublayer": [
        0.024391189217567444,
        0.06930837035179138,
        0.10445628315210342,
        0.019052622839808464,
        0.09584712982177734,
        0.0026203703600913286,
        0.031347230076789856,
        0.0018832145724445581,
        0.020275620743632317,
        0.002480348339304328,
        0.017658807337284088,
        0.005030750762671232,
        0.020567260682582855,
        0.0064065041951835155,
        0.016946211457252502,
        0.006859246641397476,
        0.017460251227021217,
        0.009737920947372913,
        0.022223282605409622,
        0.009925507940351963,
        0.02365109510719776,
        0.012353206984698772,
        0.04038562253117561,
        0.01845649816095829,
        0.03250120207667351,
        0.023668669164180756,
        0.029116366058588028,
        0.023308098316192627,
        0.029388509690761566,
        0.023618459701538086,
        0.026225870475172997,
        0.024062534794211388,
        0.02648484893143177,
        0.02197168581187725,
        0.03580358996987343,
        0.021145176142454147,
        0.04329143092036247,
        0.025955181568861008,
        0.04141230136156082,
        0.02680986560881138,
        0.05275771766901016,
        0.035374872386455536,
        0.044423554092645645,
        0.038993529975414276,
        0.05167620629072189,
        0.05052104964852333,
        0.03969942033290863,
        0.05720135569572449,
        0.0389031358063221,
        0.06300201267004013,
        0.026857485994696617,
        0.06565802544355392,
        0.030672859400510788,
        0.0703272894024849,
        0.02773989364504814,
        0.07888873666524887,
        0.019402364268898964,
        0.09176591783761978,
        0.018159963190555573,
        0.10585808008909225,
        0.02488313429057598,
        0.1433597356081009,
        0.024982891976833344,
        0.2408120483160019
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00037295757895409754,
        0.00022454763424927058,
        0.00021936409959100382,
        0.00010566073392567048,
        9.515815728211845e-05,
        0.00010551160558417538,
        0.00011265453132986426,
        0.00010566938037237213,
        0.00012588745811984293,
        0.0001281493774985002,
        0.0001672472035341268,
        0.0002455532725992925,
        0.00022493633083964273,
        0.0002220131528127045,
        0.0002449305149794571,
        0.0002379224969510024,
        0.00024418235837095776,
        0.00027770634611442576,
        0.000335912973129177,
        0.00033241348484404164,
        0.00039774019686300875,
        0.00036753844348542204,
        0.0003869475308616486,
        0.00034605021914329437,
        0.00032032390763334626,
        0.00028824702486555996,
        0.00027123288540817104,
        0.0002526057589974623,
        0.00022790421762734515,
        0.00020876185354851858,
        0.00018495058171630387,
        0.00016871013519521616
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00016769046516107157,
        "first_to_last_ratio": 0.6977158937201501,
        "imbalance_abs_log_ratio": 0.359943288132801,
        "last_quartile_mean": 0.00024034204562399042,
        "mean": 0.00023591848270084503,
        "normalized": [
          1.5808747779504164,
          0.9518017905108639,
          0.9298300713012261,
          0.4478696739485771,
          0.40335185354164543,
          0.44723755585512437,
          0.47751464845047825,
          0.44790632409401143,
          0.5336057466912151,
          0.5431934625529073,
          0.7089194607367985,
          1.040839487386263,
          0.9534493790589178,
          0.9410587516122122,
          1.0381997721223044,
          1.0084945199172823,
          1.035028521612661,
          1.1771284001795213,
          1.4238518715599293,
          1.4090184077080408,
          1.6859221554394308,
          1.5579044052749225,
          1.640174718113608,
          1.466821145938367,
          1.357773685072106,
          1.2218077259807991,
          1.1496890040281678,
          1.0707332299935883,
          0.9660295158660277,
          0.8848897770050418,
          0.7839596948867686,
          0.7151204656107762
        ],
        "population_cv": 0.3768493091252366
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.060260046273469925,
        0.06848448514938354,
        0.11107808351516724,
        0.17387914657592773,
        0.1828140765428543,
        0.25734034180641174,
        0.2578570544719696,
        0.2722952663898468,
        0.27254435420036316,
        0.27370211482048035,
        0.2737041115760803,
        0.27443060278892517,
        0.2742544412612915,
        0.2751515209674835,
        0.27536308765411377,
        0.2749122083187103,
        0.2744285464286804,
        0.2747279405593872,
        0.2742423117160797,
        0.2733822464942932,
        0.27115195989608765,
        0.26904159784317017,
        0.26583269238471985,
        0.2588832378387451,
        0.255132257938385,
        0.24577246606349945,
        0.23856717348098755,
        0.2310730367898941,
        0.22743643820285797,
        0.22001685202121735,
        0.22118839621543884,
        0.21798942983150482,
        0.22011704742908478,
        0.21957813203334808,
        0.22253327071666718,
        0.22450163960456848,
        0.2256278693675995,
        0.22364993393421173,
        0.22584785521030426,
        0.22638610005378723,
        0.22925184667110443,
        0.22674965858459473,
        0.23080435395240784,
        0.2277289777994156,
        0.23398378491401672,
        0.2319381833076477,
        0.24050307273864746,
        0.23799650371074677,
        0.24922524392604828,
        0.24935781955718994,
        0.2630935311317444,
        0.2632073760032654,
        0.27895617485046387,
        0.2799976170063019,
        0.2995731830596924,
        0.30053073167800903,
        0.3278428912162781,
        0.32797256112098694,
        0.36820048093795776,
        0.3684364855289459,
        0.42407870292663574,
        0.4252319037914276,
        0.5123529434204102,
        0.5140777826309204
      ],
      "loss_nats": 1.3263185024261475,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 4000,
      "stream_state_rms_by_sublayer": [
        0.06848448514938354,
        0.11107808351516724,
        0.17387914657592773,
        0.1828140765428543,
        0.25734034180641174,
        0.2578570544719696,
        0.2722952663898468,
        0.27254435420036316,
        0.27370211482048035,
        0.2737041115760803,
        0.27443060278892517,
        0.2742544412612915,
        0.2751515209674835,
        0.27536308765411377,
        0.2749122083187103,
        0.2744285464286804,
        0.2747279405593872,
        0.2742423117160797,
        0.2733822464942932,
        0.27115195989608765,
        0.26904159784317017,
        0.26583269238471985,
        0.2588832378387451,
        0.255132257938385,
        0.24577246606349945,
        0.23856717348098755,
        0.2310730367898941,
        0.22743643820285797,
        0.22001685202121735,
        0.22118839621543884,
        0.21798942983150482,
        0.22011704742908478,
        0.21957813203334808,
        0.22253327071666718,
        0.22450163960456848,
        0.2256278693675995,
        0.22364993393421173,
        0.22584785521030426,
        0.22638610005378723,
        0.22925184667110443,
        0.22674965858459473,
        0.23080435395240784,
        0.2277289777994156,
        0.23398378491401672,
        0.2319381833076477,
        0.24050307273864746,
        0.23799650371074677,
        0.24922524392604828,
        0.24935781955718994,
        0.2630935311317444,
        0.2632073760032654,
        0.27895617485046387,
        0.2799976170063019,
        0.2995731830596924,
        0.30053073167800903,
        0.3278428912162781,
        0.32797256112098694,
        0.36820048093795776,
        0.3684364855289459,
        0.42407870292663574,
        0.4252319037914276,
        0.5123529434204102,
        0.5140777826309204,
        0.6704596281051636
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.00018927091150544584,
        0.0001941375230671838,
        0.00020724961359519511,
        0.00020481550018303096,
        0.00020258830045349896,
        0.00020023793331347406,
        0.00019735096429940313,
        0.0001957487838808447,
        0.00019380047160666436,
        0.00019302338478155434,
        0.0001922047813422978,
        0.00018312338215764612,
        0.00017868881695903838,
        0.00017335887241642922,
        0.00016814567788969725,
        0.0001643431605771184,
        0.00016659151879139245,
        0.0001678116386756301,
        0.0001584039127919823,
        0.0001587578735779971,
        0.00014523210120387375,
        0.00013619904348161072,
        0.00012270425213500857,
        0.00011189868382643908,
        0.00010116446355823427,
        9.155859152087942e-05,
        8.147361222654581e-05,
        7.049509440548718e-05,
        5.894774221815169e-05,
        4.627152156899683e-05,
        3.385082527529448e-05,
        2.4755347112659365e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00019892494128725957,
        "first_to_last_ratio": 3.129490087873211,
        "imbalance_abs_log_ratio": 1.140870080057971,
        "last_quartile_mean": 6.356464973578113e-05,
        "mean": 0.00014731888438745955,
        "normalized": [
          1.2847701928535473,
          1.3178047327360134,
          1.4068095509745602,
          1.3902867988353147,
          1.3751685759490058,
          1.3592142931712237,
          1.3396175590113453,
          1.3287419647165597,
          1.3155168287655152,
          1.310241966494184,
          1.304685289611514,
          1.2430407881451104,
          1.212938977253409,
          1.1767593349436627,
          1.141372191276318,
          1.1155607189155992,
          1.1308225655119981,
          1.1391047344227307,
          1.0752451286242999,
          1.077647813028859,
          0.9858349240678649,
          0.9245185642554635,
          0.8329159743857911,
          0.759567819778877,
          0.6867039753855606,
          0.6214993542855888,
          0.553042555034996,
          0.4785204198266929,
          0.40013703920751104,
          0.314090903969238,
          0.2297792670372409,
          0.16803919752440544
        ],
        "population_cv": 0.3716245052237534
      },
      "activation_output_rms_by_block": [
        0.10094190388917923,
        0.17939583957195282,
        0.23316815495491028,
        0.2387540638446808,
        0.23957562446594238,
        0.2399372160434723,
        0.24013783037662506,
        0.23953619599342346,
        0.2386971116065979,
        0.235844686627388,
        0.23058047890663147,
        0.2186841070652008,
        0.2011982500553131,
        0.1896955370903015,
        0.18346232175827026,
        0.1819538027048111,
        0.18335868418216705,
        0.18572981655597687,
        0.1859624832868576,
        0.18881060183048248,
        0.18956796824932098,
        0.19203881919384003,
        0.19704653322696686,
        0.20452122390270233,
        0.21571911871433258,
        0.22943545877933502,
        0.2464856505393982,
        0.26912161707878113,
        0.30166757106781006,
        0.3485357463359833,
        0.4315659701824188,
        0.6013737916946411
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.2139308536425233,
        "first_to_last_ratio": 0.6473178416327895,
        "imbalance_abs_log_ratio": 0.4349178506515776,
        "last_quartile_mean": 0.3304881155490875,
        "mean": 0.2363282556179911,
        "normalized": [
          0.42712583658360814,
          0.7590960255803447,
          0.9866283417748029,
          1.0102645712860119,
          1.0137409250513,
          1.015270964599827,
          1.0161198446147375,
          1.0135740873093813,
          1.0100235834365736,
          0.9979538249062154,
          0.9756788425644265,
          0.9253405035862031,
          0.8513508024217668,
          0.8026781926446058,
          0.7763029489576774,
          0.7699197974825631,
          0.7758644166466246,
          0.7858976323855102,
          0.7868821390001439,
          0.7989336752676848,
          0.8021383975166516,
          0.8125935626768981,
          0.8337832169568393,
          0.8654116426657729,
          0.9127944441100949,
          0.9708338013978413,
          1.0429800274827308,
          1.138761915603517,
          1.2764769505828182,
          1.4747950702067896,
          1.8261293769290816,
          2.5446546377709565
        ],
        "population_cv": 0.36483989965797836
      },
      "bits_per_byte": 1.8402207275901343,
      "branch_output_rms_by_sublayer": [
        0.033478349447250366,
        0.052373696118593216,
        0.10464497655630112,
        0.018782487139105797,
        0.07005563378334045,
        0.002347440691664815,
        0.020065777003765106,
        0.0017286264337599277,
        0.017071161419153214,
        0.002082002116367221,
        0.015527697280049324,
        0.004463158082216978,
        0.01603785529732704,
        0.005855648312717676,
        0.013071421533823013,
        0.006003717426210642,
        0.014386727474629879,
        0.008227710612118244,
        0.014956407248973846,
        0.008421593345701694,
        0.015932755544781685,
        0.010948682203888893,
        0.037222135812044144,
        0.016641829162836075,
        0.029781410470604897,
        0.021082323044538498,
        0.025465231388807297,
        0.02131747640669346,
        0.024801449850201607,
        0.021334858611226082,
        0.024093298241496086,
        0.021644918248057365,
        0.023488737642765045,
        0.020262837409973145,
        0.031102469190955162,
        0.020060395821928978,
        0.03728024289011955,
        0.024732086807489395,
        0.03451274335384369,
        0.025743624195456505,
        0.0439009852707386,
        0.03343674913048744,
        0.0384296216070652,
        0.037262238562107086,
        0.044400904327631,
        0.047452766448259354,
        0.03565425053238869,
        0.054450251162052155,
        0.032797131687402725,
        0.05981304869055748,
        0.022211335599422455,
        0.0634065493941307,
        0.02717488259077072,
        0.068015918135643,
        0.023526398465037346,
        0.07457657158374786,
        0.01840120367705822,
        0.08500036597251892,
        0.017178243026137352,
        0.09797899425029755,
        0.022575926035642624,
        0.13843585550785065,
        0.029057970270514488,
        0.24571193754673004
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0004888726102119165,
        0.00034371112095624357,
        0.00027329239576199496,
        0.00013109360072049493,
        0.0001309003833444521,
        0.00015153657998531148,
        0.00016751171475037753,
        0.00015920301567702626,
        0.00019282855273444342,
        0.00018189099839941241,
        0.00022228035276414997,
        0.00039117902483509027,
        0.00035836801505216564,
        0.00037533451853890784,
        0.0004202645380768536,
        0.0004297743688669142,
        0.0004147505595850994,
        0.0004900318658719625,
        0.0005824864404877667,
        0.0005699482810090311,
        0.0006776225188364896,
        0.0006405760434101032,
        0.0006565350519436726,
        0.0006182198906110644,
        0.0005596621809681081,
        0.0005013865499612654,
        0.000471541721969693,
        0.0004314515507572245,
        0.0003811454676239625,
        0.0003426357383233983,
        0.00029149764497300414,
        0.0002412795793653285
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00023076517767597717,
        "first_to_last_ratio": 0.5732227450358324,
        "imbalance_abs_log_ratio": 0.5564809030131627,
        "last_quartile_mean": 0.00040257505424274806,
        "mean": 0.000384025402386654,
        "normalized": [
          1.2730215427772602,
          0.8950218366288691,
          0.7116518700677824,
          0.3413670030830513,
          0.34086386611648084,
          0.3946004067531388,
          0.4361995683340739,
          0.4145637624167724,
          0.5021244728500928,
          0.4736431425342962,
          0.5788167954065396,
          1.0186279928462485,
          0.9331883068801387,
          0.9773689870676945,
          1.0943665066554955,
          1.1191300528453012,
          1.0800081375020862,
          1.2760402380324216,
          1.5167914332429844,
          1.4841421360850018,
          1.7645252491766907,
          1.6680564343635333,
          1.7096136033278435,
          1.6098411375105153,
          1.4573571891075452,
          1.305607771895378,
          1.227892006724398,
          1.1234974251073624,
          0.9925006659851322,
          0.8922215462674452,
          0.7590582371931511,
          0.6282906752152749
        ],
        "population_cv": 0.4280332727708177
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.061826109886169434,
        0.07441755384206772,
        0.10094190388917923,
        0.17219661176204681,
        0.17939583957195282,
        0.2327817976474762,
        0.23316815495491028,
        0.2385595589876175,
        0.2387540638446808,
        0.23968438804149628,
        0.23957562446594238,
        0.24012914299964905,
        0.2399372160434723,
        0.2401275336742401,
        0.24013783037662506,
        0.24013836681842804,
        0.23953619599342346,
        0.239403635263443,
        0.2386971116065979,
        0.23807503283023834,
        0.235844686627388,
        0.23385408520698547,
        0.23058047890663147,
        0.2232486605644226,
        0.2186841070652008,
        0.20881249010562897,
        0.2011982500553131,
        0.19461779296398163,
        0.1896955370903015,
        0.18348202109336853,
        0.18346232175827026,
        0.18089424073696136,
        0.1819538027048111,
        0.1813284158706665,
        0.18335868418216705,
        0.18508672714233398,
        0.18572981655597687,
        0.1841643899679184,
        0.1859624832868576,
        0.18672171235084534,
        0.18881060183048248,
        0.18648506700992584,
        0.18956796824932098,
        0.1869000494480133,
        0.19203881919384003,
        0.18984925746917725,
        0.19704653322696686,
        0.19547933340072632,
        0.20452122390270233,
        0.20447643101215363,
        0.21571911871433258,
        0.21594053506851196,
        0.22943545877933502,
        0.23029562830924988,
        0.2464856505393982,
        0.24702563881874084,
        0.26912161707878113,
        0.2691715657711029,
        0.30166757106781006,
        0.30165502429008484,
        0.3485357463359833,
        0.3492375612258911,
        0.4315659701824188,
        0.4323083460330963
      ],
      "loss_nats": 1.2755438089370728,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 8000,
      "stream_state_rms_by_sublayer": [
        0.07441755384206772,
        0.10094190388917923,
        0.17219661176204681,
        0.17939583957195282,
        0.2327817976474762,
        0.23316815495491028,
        0.2385595589876175,
        0.2387540638446808,
        0.23968438804149628,
        0.23957562446594238,
        0.24012914299964905,
        0.2399372160434723,
        0.2401275336742401,
        0.24013783037662506,
        0.24013836681842804,
        0.23953619599342346,
        0.239403635263443,
        0.2386971116065979,
        0.23807503283023834,
        0.235844686627388,
        0.23385408520698547,
        0.23058047890663147,
        0.2232486605644226,
        0.2186841070652008,
        0.20881249010562897,
        0.2011982500553131,
        0.19461779296398163,
        0.1896955370903015,
        0.18348202109336853,
        0.18346232175827026,
        0.18089424073696136,
        0.1819538027048111,
        0.1813284158706665,
        0.18335868418216705,
        0.18508672714233398,
        0.18572981655597687,
        0.1841643899679184,
        0.1859624832868576,
        0.18672171235084534,
        0.18881060183048248,
        0.18648506700992584,
        0.18956796824932098,
        0.1869000494480133,
        0.19203881919384003,
        0.18984925746917725,
        0.19704653322696686,
        0.19547933340072632,
        0.20452122390270233,
        0.20447643101215363,
        0.21571911871433258,
        0.21594053506851196,
        0.22943545877933502,
        0.23029562830924988,
        0.2464856505393982,
        0.24702563881874084,
        0.26912161707878113,
        0.2691715657711029,
        0.30166757106781006,
        0.30165502429008484,
        0.3485357463359833,
        0.3492375612258911,
        0.4315659701824188,
        0.4323083460330963,
        0.6013737916946411
      ]
    }
  ],
  "environment": {
    "autocast": "cuda-bfloat16-forward-fp32-cross-entropy",
    "compile": false,
    "compute_capability": [
      12,
      0
    ],
    "cublas_workspace_config": ":4096:8",
    "cuda": "12.8",
    "deterministic_algorithms": true,
    "gpu": "NVIDIA GeForce RTX 5090",
    "python": "3.10.14",
    "torch": "2.11.0+cu128"
  },
  "evaluations": [
    {
      "bits_per_byte": 8.037726566232575,
      "cross_entropy_nats": 5.57132750749588,
      "step": 0
    },
    {
      "bits_per_byte": 5.132850850111892,
      "cross_entropy_nats": 3.5578210949897766,
      "step": 100
    },
    {
      "bits_per_byte": 3.1258965111502706,
      "cross_entropy_nats": 2.1667063534259796,
      "step": 500
    },
    {
      "bits_per_byte": 1.9712387018569424,
      "cross_entropy_nats": 1.3663585484027863,
      "step": 2000
    },
    {
      "bits_per_byte": 1.7961544728728112,
      "cross_entropy_nats": 1.2449994087219238,
      "step": 4000
    },
    {
      "bits_per_byte": 1.7269767303333703,
      "cross_entropy_nats": 1.1970490515232086,
      "step": 8000
    }
  ],
  "gradient_gate": null,
  "hashes": {
    "final_mixer_parameters": null,
    "final_model_state": "33b1b8bda9295f48a681da23e6805bc7ab56359cd02b7b13a2812bd5c192bc4d",
    "final_optimizer_state": "4b960a87623e39ce5348146d7169b13bd4e5894a99361177367c7518bf6f603e",
    "final_public_parameters": "33b1b8bda9295f48a681da23e6805bc7ab56359cd02b7b13a2812bd5c192bc4d",
    "initial_mixer_parameters": null,
    "initial_public_parameter_elements": 18985152,
    "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670",
    "initial_public_parameter_tensors": 227,
    "initial_public_parameters": "bbf2cfc5c0019fb54bf6927274ab9b185b564a77b4b8ef2dc6f5d81575f835c2"
  },
  "manifest": {
    "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716",
    "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371",
    "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
    "input_gate_tensor_hashes": {
      "0": "add6249175ac3bb3fa911069fbe8f25d17d9ab05df34b66ee89996983522ce83",
      "1": "619b5ee4f70e975312ead9adf5df4f9d8584ce2839369c749442e429923405cf",
      "7999": "fe22994f7750c2f8323fdb5ac51c9be1e208733abaf6a337f5b1ae1df1cd2730"
    },
    "path": "experiments/k3/attnres_gradient/manifest.json",
    "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338"
  },
  "model": {
    "attnres_aggregation_groups": 8,
    "context": 256,
    "d_ff": 768,
    "d_head": 32,
    "d_model": 192,
    "heads": 6,
    "layers": 32,
    "parameters": {
      "core": 18985152,
      "embedding": 98304,
      "mixer": 0,
      "total": 18985152
    },
    "sublayers": 64,
    "sublayers_per_attnres_group": 8,
    "transformer_blocks_per_attnres_group": 4,
    "vocabulary": 256
  },
  "optimizer": {
    "betas": [
      0.9,
      0.95
    ],
    "epsilon": 1e-08,
    "grad_clip": 1.0,
    "min_lr": 3e-05,
    "name": "AdamW",
    "peak_lr": 0.0003,
    "warmup_steps": 400,
    "weight_decay_ndim_ge_2": 0.1
  },
  "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
  "run_kind": "formal",
  "schema_version": 1,
  "seed": 2026073002,
  "steps": 8000,
  "target_bytes_seen": 65536000,
  "timing": {
    "mean_ms": 41.92772524377444,
    "measured_steps": 7980,
    "median_ms": 41.88774450449273,
    "p95_ms": 42.070325661916286,
    "peak_allocated_bytes": 5938562048,
    "peak_reserved_bytes": 6325010432,
    "warmup_steps_excluded": 20
  },
  "training_history": [
    {
      "bits_per_byte": 8.02169460149312,
      "learning_rate": 7.499999999999999e-07,
      "loss_nats": 5.560214996337891,
      "step": 1,
      "unclipped_grad_norm": 27.98912239074707
    },
    {
      "bits_per_byte": 7.204295458472364,
      "learning_rate": 7.499999999999999e-06,
      "loss_nats": 4.9936370849609375,
      "step": 10,
      "unclipped_grad_norm": 12.742182731628418
    },
    {
      "bits_per_byte": 6.681083646642096,
      "learning_rate": 1.4999999999999999e-05,
      "loss_nats": 4.630974292755127,
      "step": 20,
      "unclipped_grad_norm": 3.4196884632110596
    },
    {
      "bits_per_byte": 6.539581824957005,
      "learning_rate": 2.2499999999999998e-05,
      "loss_nats": 4.53289270401001,
      "step": 30,
      "unclipped_grad_norm": 3.0530965328216553
    },
    {
      "bits_per_byte": 6.404133792582285,
      "learning_rate": 2.9999999999999997e-05,
      "loss_nats": 4.43900728225708,
      "step": 40,
      "unclipped_grad_norm": 3.002453088760376
    },
    {
      "bits_per_byte": 6.077271133589068,
      "learning_rate": 3.75e-05,
      "loss_nats": 4.2124433517456055,
      "step": 50,
      "unclipped_grad_norm": 2.940347909927368
    },
    {
      "bits_per_byte": 5.8763623767123425,
      "learning_rate": 4.4999999999999996e-05,
      "loss_nats": 4.073184013366699,
      "step": 60,
      "unclipped_grad_norm": 2.7971668243408203
    },
    {
      "bits_per_byte": 5.7093882046079605,
      "learning_rate": 5.2499999999999995e-05,
      "loss_nats": 3.957446336746216,
      "step": 70,
      "unclipped_grad_norm": 2.5766468048095703
    },
    {
      "bits_per_byte": 5.512541333979312,
      "learning_rate": 5.9999999999999995e-05,
      "loss_nats": 3.82100248336792,
      "step": 80,
      "unclipped_grad_norm": 2.5652287006378174
    },
    {
      "bits_per_byte": 5.341233887003791,
      "learning_rate": 6.749999999999999e-05,
      "loss_nats": 3.702261209487915,
      "step": 90,
      "unclipped_grad_norm": 2.3708243370056152
    },
    {
      "bits_per_byte": 5.171287854692499,
      "learning_rate": 7.5e-05,
      "loss_nats": 3.584463596343994,
      "step": 100,
      "unclipped_grad_norm": 2.273510217666626
    },
    {
      "bits_per_byte": 5.001223154352113,
      "learning_rate": 8.249999999999999e-05,
      "loss_nats": 3.466583728790283,
      "step": 110,
      "unclipped_grad_norm": 2.179899215698242
    },
    {
      "bits_per_byte": 4.73609126012899,
      "learning_rate": 8.999999999999999e-05,
      "loss_nats": 3.282808303833008,
      "step": 120,
      "unclipped_grad_norm": 1.9098186492919922
    },
    {
      "bits_per_byte": 4.5869558165027025,
      "learning_rate": 9.75e-05,
      "loss_nats": 3.1794354915618896,
      "step": 130,
      "unclipped_grad_norm": 1.6915351152420044
    },
    {
      "bits_per_byte": 4.392154507523416,
      "learning_rate": 0.00010499999999999999,
      "loss_nats": 3.0444095134735107,
      "step": 140,
      "unclipped_grad_norm": 1.478805422782898
    },
    {
      "bits_per_byte": 4.261904134823071,
      "learning_rate": 0.0001125,
      "loss_nats": 2.9541268348693848,
      "step": 150,
      "unclipped_grad_norm": 1.2380529642105103
    },
    {
      "bits_per_byte": 4.155084334510974,
      "learning_rate": 0.00011999999999999999,
      "loss_nats": 2.880084991455078,
      "step": 160,
      "unclipped_grad_norm": 1.0523717403411865
    },
    {
      "bits_per_byte": 4.00081108117281,
      "learning_rate": 0.00012749999999999998,
      "loss_nats": 2.77315092086792,
      "step": 170,
      "unclipped_grad_norm": 0.9717193841934204
    },
    {
      "bits_per_byte": 3.86890313968345,
      "learning_rate": 0.00013499999999999997,
      "loss_nats": 2.6817193031311035,
      "step": 180,
      "unclipped_grad_norm": 1.2074095010757446
    },
    {
      "bits_per_byte": 3.870022058809963,
      "learning_rate": 0.0001425,
      "loss_nats": 2.682494878768921,
      "step": 190,
      "unclipped_grad_norm": 0.712932288646698
    },
    {
      "bits_per_byte": 3.7502069054333704,
      "learning_rate": 0.00015,
      "loss_nats": 2.599445343017578,
      "step": 200,
      "unclipped_grad_norm": 0.6916731595993042
    },
    {
      "bits_per_byte": 3.7020534830182927,
      "learning_rate": 0.0001575,
      "loss_nats": 2.566067934036255,
      "step": 210,
      "unclipped_grad_norm": 0.8496056795120239
    },
    {
      "bits_per_byte": 3.664806168425029,
      "learning_rate": 0.00016499999999999997,
      "loss_nats": 2.540250062942505,
      "step": 220,
      "unclipped_grad_norm": 0.7601449489593506
    },
    {
      "bits_per_byte": 3.680203087226173,
      "learning_rate": 0.0001725,
      "loss_nats": 2.550922393798828,
      "step": 230,
      "unclipped_grad_norm": 0.7172863483428955
    },
    {
      "bits_per_byte": 3.6667052008558563,
      "learning_rate": 0.00017999999999999998,
      "loss_nats": 2.5415663719177246,
      "step": 240,
      "unclipped_grad_norm": 0.8026229739189148
    },
    {
      "bits_per_byte": 3.532286312663899,
      "learning_rate": 0.0001875,
      "loss_nats": 2.448394298553467,
      "step": 250,
      "unclipped_grad_norm": 0.9514944553375244
    },
    {
      "bits_per_byte": 3.530420644867339,
      "learning_rate": 0.000195,
      "loss_nats": 2.44710111618042,
      "step": 260,
      "unclipped_grad_norm": 0.7874091267585754
    },
    {
      "bits_per_byte": 3.4939940314839926,
      "learning_rate": 0.00020249999999999996,
      "loss_nats": 2.4218521118164062,
      "step": 270,
      "unclipped_grad_norm": 0.9165833592414856
    },
    {
      "bits_per_byte": 3.5060328170443893,
      "learning_rate": 0.00020999999999999998,
      "loss_nats": 2.430196762084961,
      "step": 280,
      "unclipped_grad_norm": 0.971961259841919
    },
    {
      "bits_per_byte": 3.3964946589527543,
      "learning_rate": 0.00021749999999999997,
      "loss_nats": 2.3542706966400146,
      "step": 290,
      "unclipped_grad_norm": 0.7937634587287903
    },
    {
      "bits_per_byte": 3.488020042123622,
      "learning_rate": 0.000225,
      "loss_nats": 2.4177112579345703,
      "step": 300,
      "unclipped_grad_norm": 1.0019733905792236
    },
    {
      "bits_per_byte": 3.3886120061331084,
      "learning_rate": 0.00023249999999999996,
      "loss_nats": 2.348806858062744,
      "step": 310,
      "unclipped_grad_norm": 0.8179588913917542
    },
    {
      "bits_per_byte": 3.3666969448642643,
      "learning_rate": 0.00023999999999999998,
      "loss_nats": 2.3336164951324463,
      "step": 320,
      "unclipped_grad_norm": 0.8202242255210876
    },
    {
      "bits_per_byte": 3.351965942887261,
      "learning_rate": 0.0002475,
      "loss_nats": 2.3234057426452637,
      "step": 330,
      "unclipped_grad_norm": 0.8768095374107361
    },
    {
      "bits_per_byte": 3.357094121570688,
      "learning_rate": 0.00025499999999999996,
      "loss_nats": 2.326960325241089,
      "step": 340,
      "unclipped_grad_norm": 0.9328333139419556
    },
    {
      "bits_per_byte": 3.3293237389705688,
      "learning_rate": 0.0002625,
      "loss_nats": 2.307711362838745,
      "step": 350,
      "unclipped_grad_norm": 0.7516238689422607
    },
    {
      "bits_per_byte": 3.3205963073698848,
      "learning_rate": 0.00026999999999999995,
      "loss_nats": 2.301661968231201,
      "step": 360,
      "unclipped_grad_norm": 0.9922879338264465
    },
    {
      "bits_per_byte": 3.3203087523776422,
      "learning_rate": 0.00027749999999999997,
      "loss_nats": 2.3014626502990723,
      "step": 370,
      "unclipped_grad_norm": 0.7593456506729126
    },
    {
      "bits_per_byte": 3.3612334000116544,
      "learning_rate": 0.000285,
      "loss_nats": 2.329829454421997,
      "step": 380,
      "unclipped_grad_norm": 0.8890619874000549
    },
    {
      "bits_per_byte": 3.310503677486652,
      "learning_rate": 0.0002925,
      "loss_nats": 2.294666290283203,
      "step": 390,
      "unclipped_grad_norm": 0.8439806699752808
    },
    {
      "bits_per_byte": 3.2443041155165395,
      "learning_rate": 0.0003,
      "loss_nats": 2.2487802505493164,
      "step": 400,
      "unclipped_grad_norm": 1.1809823513031006
    },
    {
      "bits_per_byte": 3.208035726171987,
      "learning_rate": 0.0002999988466112388,
      "loss_nats": 2.2236409187316895,
      "step": 410,
      "unclipped_grad_norm": 0.9897300601005554
    },
    {
      "bits_per_byte": 3.232116049015194,
      "learning_rate": 0.00029999538646466355,
      "loss_nats": 2.2403321266174316,
      "step": 420,
      "unclipped_grad_norm": 0.898377537727356
    },
    {
      "bits_per_byte": 3.2113598068478635,
      "learning_rate": 0.0002999896196193985,
      "loss_nats": 2.225944995880127,
      "step": 430,
      "unclipped_grad_norm": 1.0354490280151367
    },
    {
      "bits_per_byte": 3.2315632967753207,
      "learning_rate": 0.00029998154617398326,
      "loss_nats": 2.2399489879608154,
      "step": 440,
      "unclipped_grad_norm": 0.9574596285820007
    },
    {
      "bits_per_byte": 3.1852710706078806,
      "learning_rate": 0.0002999711662663707,
      "loss_nats": 2.2078616619110107,
      "step": 450,
      "unclipped_grad_norm": 0.916765570640564
    },
    {
      "bits_per_byte": 3.155730642565081,
      "learning_rate": 0.0002999584800739247,
      "loss_nats": 2.1873857975006104,
      "step": 460,
      "unclipped_grad_norm": 0.9241403937339783
    },
    {
      "bits_per_byte": 3.18185446326584,
      "learning_rate": 0.0002999434878134173,
      "loss_nats": 2.205493450164795,
      "step": 470,
      "unclipped_grad_norm": 0.9457137584686279
    },
    {
      "bits_per_byte": 3.13667255709237,
      "learning_rate": 0.000299926189741025,
      "loss_nats": 2.17417573928833,
      "step": 480,
      "unclipped_grad_norm": 0.7014725208282471
    },
    {
      "bits_per_byte": 3.105025341576501,
      "learning_rate": 0.000299906586152324,
      "loss_nats": 2.1522395610809326,
      "step": 490,
      "unclipped_grad_norm": 0.840235710144043
    },
    {
      "bits_per_byte": 3.1575567543519423,
      "learning_rate": 0.0002998846773822856,
      "loss_nats": 2.1886515617370605,
      "step": 500,
      "unclipped_grad_norm": 1.1171972751617432
    },
    {
      "bits_per_byte": 3.1160109053830136,
      "learning_rate": 0.0002998604638052702,
      "loss_nats": 2.1598541736602783,
      "step": 510,
      "unclipped_grad_norm": 0.8018220067024231
    },
    {
      "bits_per_byte": 3.0148496782496044,
      "learning_rate": 0.0002998339458350211,
      "loss_nats": 2.0897345542907715,
      "step": 520,
      "unclipped_grad_norm": 1.302595853805542
    },
    {
      "bits_per_byte": 3.0793170309602265,
      "learning_rate": 0.0002998051239246572,
      "loss_nats": 2.1344199180603027,
      "step": 530,
      "unclipped_grad_norm": 0.6869481205940247
    },
    {
      "bits_per_byte": 3.040265962324714,
      "learning_rate": 0.00029977399856666537,
      "loss_nats": 2.107351779937744,
      "step": 540,
      "unclipped_grad_norm": 1.3514107465744019
    },
    {
      "bits_per_byte": 2.996858229177544,
      "learning_rate": 0.0002997405702928922,
      "loss_nats": 2.077263832092285,
      "step": 550,
      "unclipped_grad_norm": 0.9677014350891113
    },
    {
      "bits_per_byte": 2.9987245849047075,
      "learning_rate": 0.0002997048396745345,
      "loss_nats": 2.0785574913024902,
      "step": 560,
      "unclipped_grad_norm": 1.1580114364624023
    },
    {
      "bits_per_byte": 2.9750642436949075,
      "learning_rate": 0.0002996668073221302,
      "loss_nats": 2.062157392501831,
      "step": 570,
      "unclipped_grad_norm": 1.1581705808639526
    },
    {
      "bits_per_byte": 2.870633970333898,
      "learning_rate": 0.000299626473885547,
      "loss_nats": 1.989771842956543,
      "step": 580,
      "unclipped_grad_norm": 0.9525179862976074
    },
    {
      "bits_per_byte": 2.9300787417087224,
      "learning_rate": 0.00029958384005397226,
      "loss_nats": 2.030975818634033,
      "step": 590,
      "unclipped_grad_norm": 0.8967422842979431
    },
    {
      "bits_per_byte": 2.847721925638002,
      "learning_rate": 0.0002995389065559004,
      "loss_nats": 1.9738904237747192,
      "step": 600,
      "unclipped_grad_norm": 0.7550538182258606
    },
    {
      "bits_per_byte": 2.905717055248704,
      "learning_rate": 0.000299491674159121,
      "loss_nats": 2.014089584350586,
      "step": 610,
      "unclipped_grad_norm": 0.6918516755104065
    },
    {
      "bits_per_byte": 2.79248952533076,
      "learning_rate": 0.00029944214367070535,
      "loss_nats": 1.9356062412261963,
      "step": 620,
      "unclipped_grad_norm": 0.7874937653541565
    },
    {
      "bits_per_byte": 2.8091388217954867,
      "learning_rate": 0.00029939031593699284,
      "loss_nats": 1.9471466541290283,
      "step": 630,
      "unclipped_grad_norm": 0.9318027496337891
    },
    {
      "bits_per_byte": 2.8177121569409977,
      "learning_rate": 0.00029933619184357654,
      "loss_nats": 1.9530892372131348,
      "step": 640,
      "unclipped_grad_norm": 0.8821531534194946
    },
    {
      "bits_per_byte": 2.7350290897815905,
      "learning_rate": 0.00029927977231528774,
      "loss_nats": 1.895777702331543,
      "step": 650,
      "unclipped_grad_norm": 1.1635478734970093
    },
    {
      "bits_per_byte": 2.7242683072997553,
      "learning_rate": 0.00029922105831618057,
      "loss_nats": 1.8883188962936401,
      "step": 660,
      "unclipped_grad_norm": 0.8677663207054138
    },
    {
      "bits_per_byte": 2.7265658235326318,
      "learning_rate": 0.0002991600508495153,
      "loss_nats": 1.889911413192749,
      "step": 670,
      "unclipped_grad_norm": 0.8130120635032654
    },
    {
      "bits_per_byte": 2.7610239234942022,
      "learning_rate": 0.00029909675095774114,
      "loss_nats": 1.9137959480285645,
      "step": 680,
      "unclipped_grad_norm": 0.9226085543632507
    },
    {
      "bits_per_byte": 2.640049951016096,
      "learning_rate": 0.0002990311597224787,
      "loss_nats": 1.8299431800842285,
      "step": 690,
      "unclipped_grad_norm": 0.8291181921958923
    },
    {
      "bits_per_byte": 2.583675413924474,
      "learning_rate": 0.00029896327826450103,
      "loss_nats": 1.7908673286437988,
      "step": 700,
      "unclipped_grad_norm": 0.9929485321044922
    },
    {
      "bits_per_byte": 2.6777724536736933,
      "learning_rate": 0.0002988931077437152,
      "loss_nats": 1.8560904264450073,
      "step": 710,
      "unclipped_grad_norm": 1.0362995862960815
    },
    {
      "bits_per_byte": 2.7123457820113424,
      "learning_rate": 0.00029882064935914164,
      "loss_nats": 1.8800548315048218,
      "step": 720,
      "unclipped_grad_norm": 0.9176086187362671
    },
    {
      "bits_per_byte": 2.545722970462673,
      "learning_rate": 0.0002987459043488943,
      "loss_nats": 1.7645606994628906,
      "step": 730,
      "unclipped_grad_norm": 1.4210379123687744
    },
    {
      "bits_per_byte": 2.557563288043974,
      "learning_rate": 0.0002986688739901591,
      "loss_nats": 1.7727677822113037,
      "step": 740,
      "unclipped_grad_norm": 1.1384987831115723
    },
    {
      "bits_per_byte": 2.56630310239552,
      "learning_rate": 0.0002985895595991723,
      "loss_nats": 1.7788257598876953,
      "step": 750,
      "unclipped_grad_norm": 0.9141139984130859
    },
    {
      "bits_per_byte": 2.5326694872621855,
      "learning_rate": 0.00029850796253119824,
      "loss_nats": 1.7555127143859863,
      "step": 760,
      "unclipped_grad_norm": 0.9497804045677185
    },
    {
      "bits_per_byte": 2.5270525338850063,
      "learning_rate": 0.00029842408418050544,
      "loss_nats": 1.7516193389892578,
      "step": 770,
      "unclipped_grad_norm": 1.0209811925888062
    },
    {
      "bits_per_byte": 2.4536180057583974,
      "learning_rate": 0.0002983379259803436,
      "loss_nats": 1.7007184028625488,
      "step": 780,
      "unclipped_grad_norm": 0.8395339250564575
    },
    {
      "bits_per_byte": 2.5403350979764427,
      "learning_rate": 0.0002982494894029186,
      "loss_nats": 1.7608261108398438,
      "step": 790,
      "unclipped_grad_norm": 1.0305956602096558
    },
    {
      "bits_per_byte": 2.476898265309791,
      "learning_rate": 0.0002981587759593675,
      "loss_nats": 1.7168550491333008,
      "step": 800,
      "unclipped_grad_norm": 0.7494933605194092
    },
    {
      "bits_per_byte": 2.5328731147208075,
      "learning_rate": 0.00029806578719973283,
      "loss_nats": 1.7556538581848145,
      "step": 810,
      "unclipped_grad_norm": 2.0184309482574463
    },
    {
      "bits_per_byte": 2.4645244575455134,
      "learning_rate": 0.00029797052471293596,
      "loss_nats": 1.7082781791687012,
      "step": 820,
      "unclipped_grad_norm": 1.0523004531860352
    },
    {
      "bits_per_byte": 2.4679443325579205,
      "learning_rate": 0.0002978729901267499,
      "loss_nats": 1.7106486558914185,
      "step": 830,
      "unclipped_grad_norm": 0.9168965816497803
    },
    {
      "bits_per_byte": 2.457752984633092,
      "learning_rate": 0.0002977731851077716,
      "loss_nats": 1.7035845518112183,
      "step": 840,
      "unclipped_grad_norm": 1.0069817304611206
    },
    {
      "bits_per_byte": 2.4003085434704525,
      "learning_rate": 0.0002976711113613935,
      "loss_nats": 1.6637670993804932,
      "step": 850,
      "unclipped_grad_norm": 0.9418690204620361
    },
    {
      "bits_per_byte": 2.440524794565561,
      "learning_rate": 0.0002975667706317743,
      "loss_nats": 1.6916428804397583,
      "step": 860,
      "unclipped_grad_norm": 1.019945502281189
    },
    {
      "bits_per_byte": 2.404373525406246,
      "learning_rate": 0.00029746016470180915,
      "loss_nats": 1.6665847301483154,
      "step": 870,
      "unclipped_grad_norm": 1.4541757106781006
    },
    {
      "bits_per_byte": 2.4247352393724975,
      "learning_rate": 0.00029735129539309926,
      "loss_nats": 1.6806983947753906,
      "step": 880,
      "unclipped_grad_norm": 0.9109940528869629
    },
    {
      "bits_per_byte": 2.4066381929528076,
      "learning_rate": 0.0002972401645659207,
      "loss_nats": 1.6681544780731201,
      "step": 890,
      "unclipped_grad_norm": 0.9647485613822937
    },
    {
      "bits_per_byte": 2.3372799976169487,
      "learning_rate": 0.00029712677411919283,
      "loss_nats": 1.6200790405273438,
      "step": 900,
      "unclipped_grad_norm": 0.8496110439300537
    },
    {
      "bits_per_byte": 2.3871222896634996,
      "learning_rate": 0.0002970111259904454,
      "loss_nats": 1.6546270847320557,
      "step": 910,
      "unclipped_grad_norm": 1.1169326305389404
    },
    {
      "bits_per_byte": 2.3333503660273847,
      "learning_rate": 0.0002968932221557859,
      "loss_nats": 1.617355227470398,
      "step": 920,
      "unclipped_grad_norm": 0.8540288805961609
    },
    {
      "bits_per_byte": 2.3443775496354062,
      "learning_rate": 0.0002967730646298655,
      "loss_nats": 1.624998688697815,
      "step": 930,
      "unclipped_grad_norm": 1.0937522649765015
    },
    {
      "bits_per_byte": 2.285418285283967,
      "learning_rate": 0.0002966506554658449,
      "loss_nats": 1.5841312408447266,
      "step": 940,
      "unclipped_grad_norm": 0.9093461632728577
    },
    {
      "bits_per_byte": 2.261280348252721,
      "learning_rate": 0.00029652599675535876,
      "loss_nats": 1.5674000978469849,
      "step": 950,
      "unclipped_grad_norm": 0.8109434843063354
    },
    {
      "bits_per_byte": 2.2762792991997682,
      "learning_rate": 0.0002963990906284808,
      "loss_nats": 1.5777965784072876,
      "step": 960,
      "unclipped_grad_norm": 0.8615142107009888
    },
    {
      "bits_per_byte": 2.320710157136921,
      "learning_rate": 0.0002962699392536863,
      "loss_nats": 1.6085937023162842,
      "step": 970,
      "unclipped_grad_norm": 1.2632875442504883
    },
    {
      "bits_per_byte": 2.270013111315582,
      "learning_rate": 0.0002961385448378161,
      "loss_nats": 1.5734531879425049,
      "step": 980,
      "unclipped_grad_norm": 0.9684879183769226
    },
    {
      "bits_per_byte": 2.2007698003905,
      "learning_rate": 0.00029600490962603817,
      "loss_nats": 1.5254573822021484,
      "step": 990,
      "unclipped_grad_norm": 0.8230604529380798
    },
    {
      "bits_per_byte": 2.245051377386703,
      "learning_rate": 0.00029586903590180956,
      "loss_nats": 1.556151032447815,
      "step": 1000,
      "unclipped_grad_norm": 0.7837699055671692
    },
    {
      "bits_per_byte": 2.2494322914521314,
      "learning_rate": 0.00029573092598683735,
      "loss_nats": 1.559187650680542,
      "step": 1010,
      "unclipped_grad_norm": 0.8822826743125916
    },
    {
      "bits_per_byte": 2.262538057378481,
      "learning_rate": 0.00029559058224103895,
      "loss_nats": 1.5682718753814697,
      "step": 1020,
      "unclipped_grad_norm": 0.9820390939712524
    },
    {
      "bits_per_byte": 2.1631948588473424,
      "learning_rate": 0.0002954480070625017,
      "loss_nats": 1.4994124174118042,
      "step": 1030,
      "unclipped_grad_norm": 0.7792394161224365
    },
    {
      "bits_per_byte": 2.258897528625017,
      "learning_rate": 0.00029530320288744205,
      "loss_nats": 1.5657484531402588,
      "step": 1040,
      "unclipped_grad_norm": 0.8651536703109741
    },
    {
      "bits_per_byte": 2.2449380408197843,
      "learning_rate": 0.0002951561721901638,
      "loss_nats": 1.556072473526001,
      "step": 1050,
      "unclipped_grad_norm": 0.8636346459388733
    },
    {
      "bits_per_byte": 2.190915194496061,
      "learning_rate": 0.00029500691748301594,
      "loss_nats": 1.5186266899108887,
      "step": 1060,
      "unclipped_grad_norm": 0.9602811932563782
    },
    {
      "bits_per_byte": 2.2937991718431627,
      "learning_rate": 0.00029485544131634963,
      "loss_nats": 1.5899404287338257,
      "step": 1070,
      "unclipped_grad_norm": 0.82843017578125
    },
    {
      "bits_per_byte": 2.212529630091197,
      "learning_rate": 0.0002947017462784745,
      "loss_nats": 1.5336086750030518,
      "step": 1080,
      "unclipped_grad_norm": 0.9238869547843933
    },
    {
      "bits_per_byte": 2.140786207370522,
      "learning_rate": 0.00029454583499561476,
      "loss_nats": 1.4838799238204956,
      "step": 1090,
      "unclipped_grad_norm": 0.8506197333335876
    },
    {
      "bits_per_byte": 2.2365177702335415,
      "learning_rate": 0.00029438771013186413,
      "loss_nats": 1.5502359867095947,
      "step": 1100,
      "unclipped_grad_norm": 0.905985951423645
    },
    {
      "bits_per_byte": 2.2504714106286445,
      "learning_rate": 0.0002942273743891401,
      "loss_nats": 1.5599079132080078,
      "step": 1110,
      "unclipped_grad_norm": 0.8354071378707886
    },
    {
      "bits_per_byte": 2.1399247462723503,
      "learning_rate": 0.00029406483050713824,
      "loss_nats": 1.4832828044891357,
      "step": 1120,
      "unclipped_grad_norm": 0.8565837144851685
    },
    {
      "bits_per_byte": 2.1407069233684743,
      "learning_rate": 0.00029390008126328504,
      "loss_nats": 1.4838249683380127,
      "step": 1130,
      "unclipped_grad_norm": 0.7768547534942627
    },
    {
      "bits_per_byte": 2.1434902905900377,
      "learning_rate": 0.00029373312947269045,
      "loss_nats": 1.4857542514801025,
      "step": 1140,
      "unclipped_grad_norm": 0.8102613687515259
    },
    {
      "bits_per_byte": 2.1476347285105297,
      "learning_rate": 0.00029356397798809995,
      "loss_nats": 1.4886269569396973,
      "step": 1150,
      "unclipped_grad_norm": 0.7917956113815308
    },
    {
      "bits_per_byte": 2.0881920209275155,
      "learning_rate": 0.0002933926296998457,
      "loss_nats": 1.4474244117736816,
      "step": 1160,
      "unclipped_grad_norm": 0.8217911720275879
    },
    {
      "bits_per_byte": 2.1987029128924305,
      "learning_rate": 0.0002932190875357972,
      "loss_nats": 1.5240247249603271,
      "step": 1170,
      "unclipped_grad_norm": 0.7990338802337646
    },
    {
      "bits_per_byte": 2.091903062567833,
      "learning_rate": 0.00029304335446131126,
      "loss_nats": 1.4499967098236084,
      "step": 1180,
      "unclipped_grad_norm": 0.8429117202758789
    },
    {
      "bits_per_byte": 2.129434492500322,
      "learning_rate": 0.00029286543347918116,
      "loss_nats": 1.4760115146636963,
      "step": 1190,
      "unclipped_grad_norm": 0.878530740737915
    },
    {
      "bits_per_byte": 2.113836010032367,
      "learning_rate": 0.0002926853276295856,
      "loss_nats": 1.4651994705200195,
      "step": 1200,
      "unclipped_grad_norm": 0.7968210577964783
    },
    {
      "bits_per_byte": 2.2311123555169234,
      "learning_rate": 0.0002925030399900368,
      "loss_nats": 1.5464892387390137,
      "step": 1210,
      "unclipped_grad_norm": 0.8802653551101685
    },
    {
      "bits_per_byte": 2.085374257175779,
      "learning_rate": 0.00029231857367532726,
      "loss_nats": 1.4454712867736816,
      "step": 1220,
      "unclipped_grad_norm": 0.8048909306526184
    },
    {
      "bits_per_byte": 2.0275156817898625,
      "learning_rate": 0.0002921319318374776,
      "loss_nats": 1.4053667783737183,
      "step": 1230,
      "unclipped_grad_norm": 0.7767927646636963
    },
    {
      "bits_per_byte": 2.034634215691725,
      "learning_rate": 0.00029194311766568174,
      "loss_nats": 1.4103009700775146,
      "step": 1240,
      "unclipped_grad_norm": 0.769820511341095
    },
    {
      "bits_per_byte": 2.2082001388383765,
      "learning_rate": 0.000291752134386253,
      "loss_nats": 1.5306077003479004,
      "step": 1250,
      "unclipped_grad_norm": 0.8053375482559204
    },
    {
      "bits_per_byte": 2.1275602255712194,
      "learning_rate": 0.00029155898526256856,
      "loss_nats": 1.4747123718261719,
      "step": 1260,
      "unclipped_grad_norm": 0.7535898685455322
    },
    {
      "bits_per_byte": 1.9931237951488738,
      "learning_rate": 0.0002913636735950141,
      "loss_nats": 1.3815281391143799,
      "step": 1270,
      "unclipped_grad_norm": 0.7098326683044434
    },
    {
      "bits_per_byte": 2.1454299109264685,
      "learning_rate": 0.0002911662027209272,
      "loss_nats": 1.4870986938476562,
      "step": 1280,
      "unclipped_grad_norm": 0.7225908637046814
    },
    {
      "bits_per_byte": 2.0451564582368134,
      "learning_rate": 0.00029096657601454013,
      "loss_nats": 1.4175944328308105,
      "step": 1290,
      "unclipped_grad_norm": 0.7847483158111572
    },
    {
      "bits_per_byte": 2.0739233083160316,
      "learning_rate": 0.00029076479688692266,
      "loss_nats": 1.4375340938568115,
      "step": 1300,
      "unclipped_grad_norm": 0.7665325403213501
    },
    {
      "bits_per_byte": 2.153918458625104,
      "learning_rate": 0.00029056086878592323,
      "loss_nats": 1.4929825067520142,
      "step": 1310,
      "unclipped_grad_norm": 0.8358681797981262
    },
    {
      "bits_per_byte": 2.0810513012636918,
      "learning_rate": 0.00029035479519611045,
      "loss_nats": 1.4424748420715332,
      "step": 1320,
      "unclipped_grad_norm": 0.7432150840759277
    },
    {
      "bits_per_byte": 1.9944121171864873,
      "learning_rate": 0.0002901465796387134,
      "loss_nats": 1.3824211359024048,
      "step": 1330,
      "unclipped_grad_norm": 0.9147225022315979
    },
    {
      "bits_per_byte": 2.0106892431947467,
      "learning_rate": 0.00028993622567156154,
      "loss_nats": 1.393703579902649,
      "step": 1340,
      "unclipped_grad_norm": 0.7953330278396606
    },
    {
      "bits_per_byte": 2.178856458948164,
      "learning_rate": 0.0002897237368890237,
      "loss_nats": 1.510268211364746,
      "step": 1350,
      "unclipped_grad_norm": 0.9230644702911377
    },
    {
      "bits_per_byte": 2.000966204028218,
      "learning_rate": 0.0002895091169219469,
      "loss_nats": 1.3869640827178955,
      "step": 1360,
      "unclipped_grad_norm": 0.766094446182251
    },
    {
      "bits_per_byte": 2.099968876910648,
      "learning_rate": 0.00028929236943759425,
      "loss_nats": 1.4555875062942505,
      "step": 1370,
      "unclipped_grad_norm": 0.802431583404541
    },
    {
      "bits_per_byte": 2.0735674762113967,
      "learning_rate": 0.00028907349813958225,
      "loss_nats": 1.437287449836731,
      "step": 1380,
      "unclipped_grad_norm": 0.7674750685691833
    },
    {
      "bits_per_byte": 2.0708314042188203,
      "learning_rate": 0.0002888525067678175,
      "loss_nats": 1.4353909492492676,
      "step": 1390,
      "unclipped_grad_norm": 0.7390276789665222
    },
    {
      "bits_per_byte": 2.082916281129648,
      "learning_rate": 0.00028862939909843273,
      "loss_nats": 1.4437675476074219,
      "step": 1400,
      "unclipped_grad_norm": 0.8315935730934143
    },
    {
      "bits_per_byte": 2.026164414102033,
      "learning_rate": 0.0002884041789437225,
      "loss_nats": 1.4044301509857178,
      "step": 1410,
      "unclipped_grad_norm": 0.8864192366600037
    },
    {
      "bits_per_byte": 2.0547052789780182,
      "learning_rate": 0.0002881768501520778,
      "loss_nats": 1.424213171005249,
      "step": 1420,
      "unclipped_grad_norm": 0.6849789023399353
    },
    {
      "bits_per_byte": 2.0500001776157135,
      "learning_rate": 0.0002879474166079205,
      "loss_nats": 1.4209518432617188,
      "step": 1430,
      "unclipped_grad_norm": 0.6987867951393127
    },
    {
      "bits_per_byte": 2.022067959341132,
      "learning_rate": 0.00028771588223163694,
      "loss_nats": 1.4015907049179077,
      "step": 1440,
      "unclipped_grad_norm": 0.7956437468528748
    },
    {
      "bits_per_byte": 2.068799257216225,
      "learning_rate": 0.00028748225097951054,
      "loss_nats": 1.4339823722839355,
      "step": 1450,
      "unclipped_grad_norm": 0.9165704250335693
    },
    {
      "bits_per_byte": 2.0134160281241766,
      "learning_rate": 0.000287246526843655,
      "loss_nats": 1.3955936431884766,
      "step": 1460,
      "unclipped_grad_norm": 0.7257574200630188
    },
    {
      "bits_per_byte": 2.0111035494006755,
      "learning_rate": 0.00028700871385194533,
      "loss_nats": 1.3939907550811768,
      "step": 1470,
      "unclipped_grad_norm": 0.77181476354599
    },
    {
      "bits_per_byte": 1.915849582359102,
      "learning_rate": 0.00028676881606794945,
      "loss_nats": 1.3279657363891602,
      "step": 1480,
      "unclipped_grad_norm": 0.6826913356781006
    },
    {
      "bits_per_byte": 2.0189380470780796,
      "learning_rate": 0.00028652683759085854,
      "loss_nats": 1.399421215057373,
      "step": 1490,
      "unclipped_grad_norm": 0.850294291973114
    },
    {
      "bits_per_byte": 1.925917102775309,
      "learning_rate": 0.0002862827825554171,
      "loss_nats": 1.3349440097808838,
      "step": 1500,
      "unclipped_grad_norm": 0.6999683380126953
    },
    {
      "bits_per_byte": 1.9985412486510523,
      "learning_rate": 0.00028603665513185245,
      "loss_nats": 1.3852832317352295,
      "step": 1510,
      "unclipped_grad_norm": 0.7689449787139893
    },
    {
      "bits_per_byte": 1.9951115706275462,
      "learning_rate": 0.000285788459525803,
      "loss_nats": 1.3829059600830078,
      "step": 1520,
      "unclipped_grad_norm": 0.785367488861084
    },
    {
      "bits_per_byte": 1.9521630311581806,
      "learning_rate": 0.00028553819997824707,
      "loss_nats": 1.3531363010406494,
      "step": 1530,
      "unclipped_grad_norm": 0.6442116498947144
    },
    {
      "bits_per_byte": 2.0327179849958115,
      "learning_rate": 0.00028528588076542964,
      "loss_nats": 1.4089727401733398,
      "step": 1540,
      "unclipped_grad_norm": 0.7405364513397217
    },
    {
      "bits_per_byte": 1.996995812550399,
      "learning_rate": 0.00028503150619878996,
      "loss_nats": 1.3842120170593262,
      "step": 1550,
      "unclipped_grad_norm": 0.7386215329170227
    },
    {
      "bits_per_byte": 2.0870359535484155,
      "learning_rate": 0.0002847750806248876,
      "loss_nats": 1.4466230869293213,
      "step": 1560,
      "unclipped_grad_norm": 0.7484163641929626
    },
    {
      "bits_per_byte": 2.0235489019477115,
      "learning_rate": 0.000284516608425328,
      "loss_nats": 1.4026172161102295,
      "step": 1570,
      "unclipped_grad_norm": 0.7196536064147949
    },
    {
      "bits_per_byte": 2.00750653225788,
      "learning_rate": 0.000284256094016688,
      "loss_nats": 1.3914974927902222,
      "step": 1580,
      "unclipped_grad_norm": 0.7144435048103333
    },
    {
      "bits_per_byte": 2.0128109931584413,
      "learning_rate": 0.0002839935418504399,
      "loss_nats": 1.395174264907837,
      "step": 1590,
      "unclipped_grad_norm": 0.8628523349761963
    },
    {
      "bits_per_byte": 2.0153315708894866,
      "learning_rate": 0.000283728956412876,
      "loss_nats": 1.3969213962554932,
      "step": 1600,
      "unclipped_grad_norm": 0.715620756149292
    },
    {
      "bits_per_byte": 2.048095125792106,
      "learning_rate": 0.00028346234222503136,
      "loss_nats": 1.4196313619613647,
      "step": 1610,
      "unclipped_grad_norm": 0.7313905358314514
    },
    {
      "bits_per_byte": 2.0567931483594926,
      "learning_rate": 0.00028319370384260675,
      "loss_nats": 1.4256603717803955,
      "step": 1620,
      "unclipped_grad_norm": 0.761407732963562
    },
    {
      "bits_per_byte": 1.907960738164025,
      "learning_rate": 0.00028292304585589105,
      "loss_nats": 1.3224976062774658,
      "step": 1630,
      "unclipped_grad_norm": 0.7100295424461365
    },
    {
      "bits_per_byte": 2.0356341228238413,
      "learning_rate": 0.0002826503728896824,
      "loss_nats": 1.410994052886963,
      "step": 1640,
      "unclipped_grad_norm": 0.7265889048576355
    },
    {
      "bits_per_byte": 1.9881094689803178,
      "learning_rate": 0.0002823756896032095,
      "loss_nats": 1.3780524730682373,
      "step": 1650,
      "unclipped_grad_norm": 0.7565467953681946
    },
    {
      "bits_per_byte": 1.9576370669524927,
      "learning_rate": 0.0002820990006900518,
      "loss_nats": 1.3569306135177612,
      "step": 1660,
      "unclipped_grad_norm": 0.6934499144554138
    },
    {
      "bits_per_byte": 2.105078653450433,
      "learning_rate": 0.00028182031087805947,
      "loss_nats": 1.4591293334960938,
      "step": 1670,
      "unclipped_grad_norm": 0.8928083181381226
    },
    {
      "bits_per_byte": 2.0211131116635417,
      "learning_rate": 0.0002815396249292725,
      "loss_nats": 1.4009288549423218,
      "step": 1680,
      "unclipped_grad_norm": 0.6971679925918579
    },
    {
      "bits_per_byte": 1.9058530907777012,
      "learning_rate": 0.0002812569476398391,
      "loss_nats": 1.321036696434021,
      "step": 1690,
      "unclipped_grad_norm": 0.6719903349876404
    },
    {
      "bits_per_byte": 1.9394077658742894,
      "learning_rate": 0.00028097228383993446,
      "loss_nats": 1.3442950248718262,
      "step": 1700,
      "unclipped_grad_norm": 0.6790670156478882
    },
    {
      "bits_per_byte": 2.021125322431753,
      "learning_rate": 0.0002806856383936774,
      "loss_nats": 1.4009373188018799,
      "step": 1710,
      "unclipped_grad_norm": 0.6948477029800415
    },
    {
      "bits_per_byte": 2.0066268409987162,
      "learning_rate": 0.00028039701619904774,
      "loss_nats": 1.39088773727417,
      "step": 1720,
      "unclipped_grad_norm": 0.6923770904541016
    },
    {
      "bits_per_byte": 1.9082544845316984,
      "learning_rate": 0.0002801064221878024,
      "loss_nats": 1.3227012157440186,
      "step": 1730,
      "unclipped_grad_norm": 0.875411868095398
    },
    {
      "bits_per_byte": 1.950284636645457,
      "learning_rate": 0.00027981386132539134,
      "loss_nats": 1.3518342971801758,
      "step": 1740,
      "unclipped_grad_norm": 0.7188331484794617
    },
    {
      "bits_per_byte": 2.0503372636114046,
      "learning_rate": 0.0002795193386108724,
      "loss_nats": 1.4211854934692383,
      "step": 1750,
      "unclipped_grad_norm": 0.6843723058700562
    },
    {
      "bits_per_byte": 1.9245816574913592,
      "learning_rate": 0.00027922285907682625,
      "loss_nats": 1.334018349647522,
      "step": 1760,
      "unclipped_grad_norm": 0.7317860722541809
    },
    {
      "bits_per_byte": 1.9854330749675908,
      "learning_rate": 0.00027892442778927005,
      "loss_nats": 1.376197338104248,
      "step": 1770,
      "unclipped_grad_norm": 0.7310082912445068
    },
    {
      "bits_per_byte": 1.9060752923626159,
      "learning_rate": 0.00027862404984757116,
      "loss_nats": 1.3211907148361206,
      "step": 1780,
      "unclipped_grad_norm": 0.6923527717590332
    },
    {
      "bits_per_byte": 1.8947768921141837,
      "learning_rate": 0.0002783217303843597,
      "loss_nats": 1.313359260559082,
      "step": 1790,
      "unclipped_grad_norm": 0.7431268692016602
    },
    {
      "bits_per_byte": 1.9836379200578849,
      "learning_rate": 0.00027801747456544134,
      "loss_nats": 1.374953031539917,
      "step": 1800,
      "unclipped_grad_norm": 0.800605058670044
    },
    {
      "bits_per_byte": 1.9736849400871523,
      "learning_rate": 0.00027771128758970843,
      "loss_nats": 1.3680541515350342,
      "step": 1810,
      "unclipped_grad_norm": 0.6639943718910217
    },
    {
      "bits_per_byte": 2.0327128255162856,
      "learning_rate": 0.00027740317468905165,
      "loss_nats": 1.4089691638946533,
      "step": 1820,
      "unclipped_grad_norm": 0.7219430804252625
    },
    {
      "bits_per_byte": 1.8641528013882975,
      "learning_rate": 0.0002770931411282704,
      "loss_nats": 1.2921322584152222,
      "step": 1830,
      "unclipped_grad_norm": 0.6714999079704285
    },
    {
      "bits_per_byte": 1.9760018903595762,
      "learning_rate": 0.0002767811922049827,
      "loss_nats": 1.3696601390838623,
      "step": 1840,
      "unclipped_grad_norm": 0.6850439310073853
    },
    {
      "bits_per_byte": 1.8495459708852175,
      "learning_rate": 0.00027646733324953503,
      "loss_nats": 1.2820075750350952,
      "step": 1850,
      "unclipped_grad_norm": 0.7066381573677063
    },
    {
      "bits_per_byte": 2.002332434206672,
      "learning_rate": 0.00027615156962491097,
      "loss_nats": 1.387911081314087,
      "step": 1860,
      "unclipped_grad_norm": 0.7545463442802429
    },
    {
      "bits_per_byte": 2.025226076758926,
      "learning_rate": 0.0002758339067266396,
      "loss_nats": 1.4037797451019287,
      "step": 1870,
      "unclipped_grad_norm": 0.7131425142288208
    },
    {
      "bits_per_byte": 1.8761874593650736,
      "learning_rate": 0.0002755143499827035,
      "loss_nats": 1.3004740476608276,
      "step": 1880,
      "unclipped_grad_norm": 0.627357006072998
    },
    {
      "bits_per_byte": 1.9627021280030534,
      "learning_rate": 0.0002751929048534456,
      "loss_nats": 1.3604414463043213,
      "step": 1890,
      "unclipped_grad_norm": 0.6769031882286072
    },
    {
      "bits_per_byte": 1.87336023656754,
      "learning_rate": 0.0002748695768314763,
      "loss_nats": 1.2985143661499023,
      "step": 1900,
      "unclipped_grad_norm": 0.6752462983131409
    },
    {
      "bits_per_byte": 1.8594791728511009,
      "learning_rate": 0.00027454437144157947,
      "loss_nats": 1.2888927459716797,
      "step": 1910,
      "unclipped_grad_norm": 0.6314391493797302
    },
    {
      "bits_per_byte": 1.9380384400081196,
      "learning_rate": 0.00027421729424061787,
      "loss_nats": 1.3433458805084229,
      "step": 1920,
      "unclipped_grad_norm": 0.6901462078094482
    },
    {
      "bits_per_byte": 1.8913296718602894,
      "learning_rate": 0.0002738883508174385,
      "loss_nats": 1.3109698295593262,
      "step": 1930,
      "unclipped_grad_norm": 0.6903179287910461
    },
    {
      "bits_per_byte": 1.8624152606666307,
      "learning_rate": 0.0002735575467927768,
      "loss_nats": 1.2909278869628906,
      "step": 1940,
      "unclipped_grad_norm": 0.6572144627571106
    },
    {
      "bits_per_byte": 1.930714558821127,
      "learning_rate": 0.0002732248878191609,
      "loss_nats": 1.3382693529129028,
      "step": 1950,
      "unclipped_grad_norm": 0.7911640405654907
    },
    {
      "bits_per_byte": 1.901616298173696,
      "learning_rate": 0.0002728903795808148,
      "loss_nats": 1.3180999755859375,
      "step": 1960,
      "unclipped_grad_norm": 0.7476807236671448
    },
    {
      "bits_per_byte": 1.8545080143279113,
      "learning_rate": 0.0002725540277935614,
      "loss_nats": 1.2854470014572144,
      "step": 1970,
      "unclipped_grad_norm": 0.6583976745605469
    },
    {
      "bits_per_byte": 1.8296589290353475,
      "learning_rate": 0.00027221583820472456,
      "loss_nats": 1.2682229280471802,
      "step": 1980,
      "unclipped_grad_norm": 0.6357024908065796
    },
    {
      "bits_per_byte": 1.9398576724889465,
      "learning_rate": 0.0002718758165930314,
      "loss_nats": 1.344606876373291,
      "step": 1990,
      "unclipped_grad_norm": 0.7643471956253052
    },
    {
      "bits_per_byte": 1.866766077768158,
      "learning_rate": 0.0002715339687685131,
      "loss_nats": 1.2939436435699463,
      "step": 2000,
      "unclipped_grad_norm": 0.682447612285614
    },
    {
      "bits_per_byte": 1.9452981716663407,
      "learning_rate": 0.00027119030057240576,
      "loss_nats": 1.3483779430389404,
      "step": 2010,
      "unclipped_grad_norm": 0.6716283559799194
    },
    {
      "bits_per_byte": 1.922642553102881,
      "learning_rate": 0.0002708448178770506,
      "loss_nats": 1.332674264907837,
      "step": 2020,
      "unclipped_grad_norm": 0.7301247119903564
    },
    {
      "bits_per_byte": 1.9056240098867518,
      "learning_rate": 0.00027049752658579376,
      "loss_nats": 1.3208779096603394,
      "step": 2030,
      "unclipped_grad_norm": 0.7755060791969299
    },
    {
      "bits_per_byte": 1.9283054258478407,
      "learning_rate": 0.00027014843263288514,
      "loss_nats": 1.3365994691848755,
      "step": 2040,
      "unclipped_grad_norm": 0.648145854473114
    },
    {
      "bits_per_byte": 1.8707481640662356,
      "learning_rate": 0.00026979754198337724,
      "loss_nats": 1.296703815460205,
      "step": 2050,
      "unclipped_grad_norm": 0.6962859630584717
    },
    {
      "bits_per_byte": 1.774047478965604,
      "learning_rate": 0.00026944486063302305,
      "loss_nats": 1.2296760082244873,
      "step": 2060,
      "unclipped_grad_norm": 0.605278730392456
    },
    {
      "bits_per_byte": 1.8770535639948187,
      "learning_rate": 0.0002690903946081738,
      "loss_nats": 1.3010743856430054,
      "step": 2070,
      "unclipped_grad_norm": 0.6994346976280212
    },
    {
      "bits_per_byte": 1.9817110264376179,
      "learning_rate": 0.00026873414996567596,
      "loss_nats": 1.37361741065979,
      "step": 2080,
      "unclipped_grad_norm": 0.7077776789665222
    },
    {
      "bits_per_byte": 1.8423689628820616,
      "learning_rate": 0.00026837613279276736,
      "loss_nats": 1.2770328521728516,
      "step": 2090,
      "unclipped_grad_norm": 0.6188594698905945
    },
    {
      "bits_per_byte": 1.9159211271418608,
      "learning_rate": 0.0002680163492069739,
      "loss_nats": 1.3280153274536133,
      "step": 2100,
      "unclipped_grad_norm": 0.6360442638397217
    },
    {
      "bits_per_byte": 1.8300450300865345,
      "learning_rate": 0.00026765480535600414,
      "loss_nats": 1.2684905529022217,
      "step": 2110,
      "unclipped_grad_norm": 0.6518478393554688
    },
    {
      "bits_per_byte": 1.8775524856649717,
      "learning_rate": 0.0002672915074176452,
      "loss_nats": 1.3014202117919922,
      "step": 2120,
      "unclipped_grad_norm": 0.677630603313446
    },
    {
      "bits_per_byte": 1.88719968048247,
      "learning_rate": 0.00026692646159965624,
      "loss_nats": 1.3081071376800537,
      "step": 2130,
      "unclipped_grad_norm": 0.6904116868972778
    },
    {
      "bits_per_byte": 1.7861939256654409,
      "learning_rate": 0.0002665596741396633,
      "loss_nats": 1.2380952835083008,
      "step": 2140,
      "unclipped_grad_norm": 0.5910601615905762
    },
    {
      "bits_per_byte": 1.8339614190119824,
      "learning_rate": 0.000266191151305052,
      "loss_nats": 1.271205186843872,
      "step": 2150,
      "unclipped_grad_norm": 0.7349988222122192
    },
    {
      "bits_per_byte": 1.8980882460738961,
      "learning_rate": 0.00026582089939286075,
      "loss_nats": 1.3156545162200928,
      "step": 2160,
      "unclipped_grad_norm": 0.6920630931854248
    },
    {
      "bits_per_byte": 1.8727663804741106,
      "learning_rate": 0.0002654489247296732,
      "loss_nats": 1.2981027364730835,
      "step": 2170,
      "unclipped_grad_norm": 0.7049499750137329
    },
    {
      "bits_per_byte": 1.7856067768953952,
      "learning_rate": 0.00026507523367151007,
      "loss_nats": 1.2376883029937744,
      "step": 2180,
      "unclipped_grad_norm": 0.652675211429596
    },
    {
      "bits_per_byte": 1.9184699100276477,
      "learning_rate": 0.00026469983260372037,
      "loss_nats": 1.3297820091247559,
      "step": 2190,
      "unclipped_grad_norm": 0.6759907007217407
    },
    {
      "bits_per_byte": 1.9474407315307896,
      "learning_rate": 0.0002643227279408727,
      "loss_nats": 1.349863052368164,
      "step": 2200,
      "unclipped_grad_norm": 0.6637232899665833
    },
    {
      "bits_per_byte": 1.9672947527116937,
      "learning_rate": 0.0002639439261266453,
      "loss_nats": 1.3636248111724854,
      "step": 2210,
      "unclipped_grad_norm": 0.7057329416275024
    },
    {
      "bits_per_byte": 1.8580620358079913,
      "learning_rate": 0.000263563433633716,
      "loss_nats": 1.2879104614257812,
      "step": 2220,
      "unclipped_grad_norm": 0.6423102021217346
    },
    {
      "bits_per_byte": 1.8639622446111417,
      "learning_rate": 0.00026318125696365156,
      "loss_nats": 1.2920001745224,
      "step": 2230,
      "unclipped_grad_norm": 0.664786159992218
    },
    {
      "bits_per_byte": 1.8281711071227333,
      "learning_rate": 0.000262797402646797,
      "loss_nats": 1.2671916484832764,
      "step": 2240,
      "unclipped_grad_norm": 0.6536247730255127
    },
    {
      "bits_per_byte": 1.8379201156695426,
      "learning_rate": 0.00026241187724216334,
      "loss_nats": 1.273949146270752,
      "step": 2250,
      "unclipped_grad_norm": 0.6435070633888245
    },
    {
      "bits_per_byte": 1.8619839281782669,
      "learning_rate": 0.0002620246873373162,
      "loss_nats": 1.2906289100646973,
      "step": 2260,
      "unclipped_grad_norm": 0.6457217335700989
    },
    {
      "bits_per_byte": 1.9082623957336382,
      "learning_rate": 0.00026163583954826264,
      "loss_nats": 1.322706699371338,
      "step": 2270,
      "unclipped_grad_norm": 0.6787397861480713
    },
    {
      "bits_per_byte": 1.7849188462919439,
      "learning_rate": 0.0002612453405193386,
      "loss_nats": 1.2372114658355713,
      "step": 2280,
      "unclipped_grad_norm": 0.6086795330047607
    },
    {
      "bits_per_byte": 1.8290736720744614,
      "learning_rate": 0.000260853196923095,
      "loss_nats": 1.2678172588348389,
      "step": 2290,
      "unclipped_grad_norm": 0.630083441734314
    },
    {
      "bits_per_byte": 1.8820873242029221,
      "learning_rate": 0.0002604594154601839,
      "loss_nats": 1.3045635223388672,
      "step": 2300,
      "unclipped_grad_norm": 0.6462850570678711
    },
    {
      "bits_per_byte": 1.834706619838171,
      "learning_rate": 0.0002600640028592441,
      "loss_nats": 1.2717217206954956,
      "step": 2310,
      "unclipped_grad_norm": 0.6394227743148804
    },
    {
      "bits_per_byte": 1.8546760413778043,
      "learning_rate": 0.0002596669658767859,
      "loss_nats": 1.2855634689331055,
      "step": 2320,
      "unclipped_grad_norm": 0.6594879031181335
    },
    {
      "bits_per_byte": 1.7978681080060082,
      "learning_rate": 0.000259268311297076,
      "loss_nats": 1.2461872100830078,
      "step": 2330,
      "unclipped_grad_norm": 0.6738982796669006
    },
    {
      "bits_per_byte": 1.8462244699491042,
      "learning_rate": 0.00025886804593202114,
      "loss_nats": 1.279705286026001,
      "step": 2340,
      "unclipped_grad_norm": 0.687351405620575
    },
    {
      "bits_per_byte": 1.7549572327371814,
      "learning_rate": 0.00025846617662105193,
      "loss_nats": 1.216443657875061,
      "step": 2350,
      "unclipped_grad_norm": 0.6065738201141357
    },
    {
      "bits_per_byte": 1.8467686230564342,
      "learning_rate": 0.00025806271023100637,
      "loss_nats": 1.2800824642181396,
      "step": 2360,
      "unclipped_grad_norm": 0.676578938961029
    },
    {
      "bits_per_byte": 1.8782005162934228,
      "learning_rate": 0.00025765765365601165,
      "loss_nats": 1.3018693923950195,
      "step": 2370,
      "unclipped_grad_norm": 0.6810930967330933
    },
    {
      "bits_per_byte": 1.7780921669485956,
      "learning_rate": 0.00025725101381736707,
      "loss_nats": 1.2324795722961426,
      "step": 2380,
      "unclipped_grad_norm": 0.5974810719490051
    },
    {
      "bits_per_byte": 1.7797094917973095,
      "learning_rate": 0.00025684279766342567,
      "loss_nats": 1.2336006164550781,
      "step": 2390,
      "unclipped_grad_norm": 0.6242655515670776
    },
    {
      "bits_per_byte": 1.8773909939558115,
      "learning_rate": 0.00025643301216947504,
      "loss_nats": 1.301308274269104,
      "step": 2400,
      "unclipped_grad_norm": 0.7230958342552185
    },
    {
      "bits_per_byte": 1.843124310684651,
      "learning_rate": 0.00025602166433761866,
      "loss_nats": 1.2775564193725586,
      "step": 2410,
      "unclipped_grad_norm": 0.6579370498657227
    },
    {
      "bits_per_byte": 1.9371601246101633,
      "learning_rate": 0.0002556087611966561,
      "loss_nats": 1.342737078666687,
      "step": 2420,
      "unclipped_grad_norm": 0.6557210087776184
    },
    {
      "bits_per_byte": 1.8373321069862427,
      "learning_rate": 0.00025519430980196277,
      "loss_nats": 1.2735415697097778,
      "step": 2430,
      "unclipped_grad_norm": 0.6670473217964172
    },
    {
      "bits_per_byte": 1.7749123797167932,
      "learning_rate": 0.0002547783172353695,
      "loss_nats": 1.2302755117416382,
      "step": 2440,
      "unclipped_grad_norm": 0.6538338661193848
    },
    {
      "bits_per_byte": 1.8761522029216466,
      "learning_rate": 0.0002543607906050415,
      "loss_nats": 1.3004496097564697,
      "step": 2450,
      "unclipped_grad_norm": 0.6531840562820435
    },
    {
      "bits_per_byte": 1.8790119304401935,
      "learning_rate": 0.0002539417370453568,
      "loss_nats": 1.3024318218231201,
      "step": 2460,
      "unclipped_grad_norm": 0.6944200992584229
    },
    {
      "bits_per_byte": 1.8171404838616947,
      "learning_rate": 0.00025352116371678464,
      "loss_nats": 1.2595458030700684,
      "step": 2470,
      "unclipped_grad_norm": 0.6366217136383057
    },
    {
      "bits_per_byte": 1.8241991678010567,
      "learning_rate": 0.0002530990778057627,
      "loss_nats": 1.264438509941101,
      "step": 2480,
      "unclipped_grad_norm": 0.6316180229187012
    },
    {
      "bits_per_byte": 1.7491533342185142,
      "learning_rate": 0.00025267548652457475,
      "loss_nats": 1.2124207019805908,
      "step": 2490,
      "unclipped_grad_norm": 0.6618465185165405
    },
    {
      "bits_per_byte": 1.827854315079844,
      "learning_rate": 0.00025225039711122696,
      "loss_nats": 1.2669720649719238,
      "step": 2500,
      "unclipped_grad_norm": 0.6852855682373047
    },
    {
      "bits_per_byte": 1.8052024801347035,
      "learning_rate": 0.00025182381682932464,
      "loss_nats": 1.2512710094451904,
      "step": 2510,
      "unclipped_grad_norm": 0.663596510887146
    },
    {
      "bits_per_byte": 1.7913451500240836,
      "learning_rate": 0.0002513957529679477,
      "loss_nats": 1.2416658401489258,
      "step": 2520,
      "unclipped_grad_norm": 0.6331848502159119
    },
    {
      "bits_per_byte": 1.7932974970766782,
      "learning_rate": 0.00025096621284152656,
      "loss_nats": 1.2430191040039062,
      "step": 2530,
      "unclipped_grad_norm": 0.6864507794380188
    },
    {
      "bits_per_byte": 1.8207522915124645,
      "learning_rate": 0.0002505352037897168,
      "loss_nats": 1.2620493173599243,
      "step": 2540,
      "unclipped_grad_norm": 0.6478230953216553
    },
    {
      "bits_per_byte": 1.8469901367107453,
      "learning_rate": 0.0002501027331772739,
      "loss_nats": 1.280236005783081,
      "step": 2550,
      "unclipped_grad_norm": 0.6545907258987427
    },
    {
      "bits_per_byte": 1.7922375679994107,
      "learning_rate": 0.00024966880839392746,
      "loss_nats": 1.2422844171524048,
      "step": 2560,
      "unclipped_grad_norm": 0.6228907704353333
    },
    {
      "bits_per_byte": 1.783868376260474,
      "learning_rate": 0.0002492334368542548,
      "loss_nats": 1.2364833354949951,
      "step": 2570,
      "unclipped_grad_norm": 0.6730257868766785
    },
    {
      "bits_per_byte": 1.8186578867902572,
      "learning_rate": 0.00024879662599755416,
      "loss_nats": 1.260597586631775,
      "step": 2580,
      "unclipped_grad_norm": 0.6890478730201721
    },
    {
      "bits_per_byte": 1.7885676302126492,
      "learning_rate": 0.00024835838328771803,
      "loss_nats": 1.2397406101226807,
      "step": 2590,
      "unclipped_grad_norm": 0.6227992177009583
    },
    {
      "bits_per_byte": 1.8280013602463319,
      "learning_rate": 0.0002479187162131051,
      "loss_nats": 1.2670739889144897,
      "step": 2600,
      "unclipped_grad_norm": 0.6276137828826904
    },
    {
      "bits_per_byte": 1.7336515060003441,
      "learning_rate": 0.00024747763228641264,
      "loss_nats": 1.2016756534576416,
      "step": 2610,
      "unclipped_grad_norm": 0.6445866823196411
    },
    {
      "bits_per_byte": 1.7935037042750628,
      "learning_rate": 0.00024703513904454795,
      "loss_nats": 1.2431620359420776,
      "step": 2620,
      "unclipped_grad_norm": 0.6606776714324951
    },
    {
      "bits_per_byte": 1.8032969123631437,
      "learning_rate": 0.00024659124404849956,
      "loss_nats": 1.2499501705169678,
      "step": 2630,
      "unclipped_grad_norm": 0.6334376931190491
    },
    {
      "bits_per_byte": 1.810602563389145,
      "learning_rate": 0.00024614595488320833,
      "loss_nats": 1.2550140619277954,
      "step": 2640,
      "unclipped_grad_norm": 0.6691069006919861
    },
    {
      "bits_per_byte": 1.7587798911179091,
      "learning_rate": 0.00024569927915743745,
      "loss_nats": 1.2190933227539062,
      "step": 2650,
      "unclipped_grad_norm": 0.6500447988510132
    },
    {
      "bits_per_byte": 1.7476075541525593,
      "learning_rate": 0.0002452512245036427,
      "loss_nats": 1.2113492488861084,
      "step": 2660,
      "unclipped_grad_norm": 0.5952810049057007
    },
    {
      "bits_per_byte": 1.7865554331975544,
      "learning_rate": 0.00024480179857784177,
      "loss_nats": 1.2383458614349365,
      "step": 2670,
      "unclipped_grad_norm": 0.6339060664176941
    },
    {
      "bits_per_byte": 1.7817725956770598,
      "learning_rate": 0.00024435100905948387,
      "loss_nats": 1.2350306510925293,
      "step": 2680,
      "unclipped_grad_norm": 0.6211492419242859
    },
    {
      "bits_per_byte": 1.8492028654967463,
      "learning_rate": 0.00024389886365131793,
      "loss_nats": 1.2817697525024414,
      "step": 2690,
      "unclipped_grad_norm": 0.6305774450302124
    },
    {
      "bits_per_byte": 1.8342178451444189,
      "learning_rate": 0.00024344537007926163,
      "loss_nats": 1.2713829278945923,
      "step": 2700,
      "unclipped_grad_norm": 0.6566506624221802
    },
    {
      "bits_per_byte": 1.7120219359319322,
      "learning_rate": 0.00024299053609226878,
      "loss_nats": 1.186683177947998,
      "step": 2710,
      "unclipped_grad_norm": 0.6094632744789124
    },
    {
      "bits_per_byte": 1.8346758349436665,
      "learning_rate": 0.00024253436946219733,
      "loss_nats": 1.271700382232666,
      "step": 2720,
      "unclipped_grad_norm": 0.6520996689796448
    },
    {
      "bits_per_byte": 1.7481449999365055,
      "learning_rate": 0.0002420768779836764,
      "loss_nats": 1.2117217779159546,
      "step": 2730,
      "unclipped_grad_norm": 0.598540186882019
    },
    {
      "bits_per_byte": 1.7678683143027543,
      "learning_rate": 0.00024161806947397305,
      "loss_nats": 1.2253929376602173,
      "step": 2740,
      "unclipped_grad_norm": 0.6463729739189148
    },
    {
      "bits_per_byte": 1.7756850977671197,
      "learning_rate": 0.00024115795177285886,
      "loss_nats": 1.2308111190795898,
      "step": 2750,
      "unclipped_grad_norm": 0.6158115863800049
    },
    {
      "bits_per_byte": 1.8688916113501715,
      "learning_rate": 0.0002406965327424758,
      "loss_nats": 1.2954169511795044,
      "step": 2760,
      "unclipped_grad_norm": 0.6701047420501709
    },
    {
      "bits_per_byte": 1.7889463360098492,
      "learning_rate": 0.00024023382026720197,
      "loss_nats": 1.2400031089782715,
      "step": 2770,
      "unclipped_grad_norm": 0.6446090936660767
    },
    {
      "bits_per_byte": 1.7765037351852266,
      "learning_rate": 0.00023976982225351698,
      "loss_nats": 1.2313785552978516,
      "step": 2780,
      "unclipped_grad_norm": 0.637730598449707
    },
    {
      "bits_per_byte": 1.7761525466121648,
      "learning_rate": 0.0002393045466298666,
      "loss_nats": 1.2311351299285889,
      "step": 2790,
      "unclipped_grad_norm": 0.7148120403289795
    },
    {
      "bits_per_byte": 1.7209247898364963,
      "learning_rate": 0.00023883800134652763,
      "loss_nats": 1.1928541660308838,
      "step": 2800,
      "unclipped_grad_norm": 0.6417195200920105
    },
    {
      "bits_per_byte": 1.792574482012451,
      "learning_rate": 0.0002383701943754715,
      "loss_nats": 1.2425179481506348,
      "step": 2810,
      "unclipped_grad_norm": 0.6152215003967285
    },
    {
      "bits_per_byte": 1.7960922151531988,
      "learning_rate": 0.00023790113371022884,
      "loss_nats": 1.2449562549591064,
      "step": 2820,
      "unclipped_grad_norm": 0.6487079858779907
    },
    {
      "bits_per_byte": 1.72477857707703,
      "learning_rate": 0.0002374308273657521,
      "loss_nats": 1.1955254077911377,
      "step": 2830,
      "unclipped_grad_norm": 0.6001560091972351
    },
    {
      "bits_per_byte": 1.807270399528678,
      "learning_rate": 0.00023695928337827911,
      "loss_nats": 1.252704381942749,
      "step": 2840,
      "unclipped_grad_norm": 0.6889218688011169
    },
    {
      "bits_per_byte": 1.7224509638802528,
      "learning_rate": 0.00023648650980519564,
      "loss_nats": 1.1939120292663574,
      "step": 2850,
      "unclipped_grad_norm": 0.6565119028091431
    },
    {
      "bits_per_byte": 1.7068036383394527,
      "learning_rate": 0.00023601251472489757,
      "loss_nats": 1.1830661296844482,
      "step": 2860,
      "unclipped_grad_norm": 0.637535035610199
    },
    {
      "bits_per_byte": 1.8267986855688483,
      "learning_rate": 0.00023553730623665305,
      "loss_nats": 1.2662403583526611,
      "step": 2870,
      "unclipped_grad_norm": 0.635680615901947
    },
    {
      "bits_per_byte": 1.7536737262137922,
      "learning_rate": 0.00023506089246046403,
      "loss_nats": 1.2155539989471436,
      "step": 2880,
      "unclipped_grad_norm": 0.6707039475440979
    },
    {
      "bits_per_byte": 1.7805597740231751,
      "learning_rate": 0.00023458328153692736,
      "loss_nats": 1.2341899871826172,
      "step": 2890,
      "unclipped_grad_norm": 0.6349734663963318
    },
    {
      "bits_per_byte": 1.713862494261466,
      "learning_rate": 0.0002341044816270961,
      "loss_nats": 1.1879589557647705,
      "step": 2900,
      "unclipped_grad_norm": 0.6683149337768555
    },
    {
      "bits_per_byte": 1.7497669683167927,
      "learning_rate": 0.00023362450091233953,
      "loss_nats": 1.212846040725708,
      "step": 2910,
      "unclipped_grad_norm": 0.6716704368591309
    },
    {
      "bits_per_byte": 1.8262012178397509,
      "learning_rate": 0.00023314334759420388,
      "loss_nats": 1.2658262252807617,
      "step": 2920,
      "unclipped_grad_norm": 0.6724333763122559
    },
    {
      "bits_per_byte": 1.761767401746047,
      "learning_rate": 0.0002326610298942717,
      "loss_nats": 1.2211641073226929,
      "step": 2930,
      "unclipped_grad_norm": 0.6612268686294556
    },
    {
      "bits_per_byte": 1.8132794733498245,
      "learning_rate": 0.00023217755605402175,
      "loss_nats": 1.2568695545196533,
      "step": 2940,
      "unclipped_grad_norm": 0.6505576968193054
    },
    {
      "bits_per_byte": 1.7731756989083804,
      "learning_rate": 0.00023169293433468802,
      "loss_nats": 1.2290717363357544,
      "step": 2950,
      "unclipped_grad_norm": 0.6778621077537537
    },
    {
      "bits_per_byte": 1.7513619354208945,
      "learning_rate": 0.00023120717301711857,
      "loss_nats": 1.213951587677002,
      "step": 2960,
      "unclipped_grad_norm": 0.6551554203033447
    },
    {
      "bits_per_byte": 1.8079858473562673,
      "learning_rate": 0.00023072028040163408,
      "loss_nats": 1.2532002925872803,
      "step": 2970,
      "unclipped_grad_norm": 0.6640533208847046
    },
    {
      "bits_per_byte": 1.7386847502604952,
      "learning_rate": 0.00023023226480788593,
      "loss_nats": 1.2051644325256348,
      "step": 2980,
      "unclipped_grad_norm": 0.643166184425354
    },
    {
      "bits_per_byte": 1.8672367942835695,
      "learning_rate": 0.00022974313457471416,
      "loss_nats": 1.2942699193954468,
      "step": 2990,
      "unclipped_grad_norm": 0.6637417078018188
    },
    {
      "bits_per_byte": 1.7982774267150616,
      "learning_rate": 0.0002292528980600049,
      "loss_nats": 1.2464709281921387,
      "step": 3000,
      "unclipped_grad_norm": 0.6690325736999512
    },
    {
      "bits_per_byte": 1.762204753627191,
      "learning_rate": 0.00022876156364054758,
      "loss_nats": 1.2214672565460205,
      "step": 3010,
      "unclipped_grad_norm": 0.6516304612159729
    },
    {
      "bits_per_byte": 1.7294503138050674,
      "learning_rate": 0.0002282691397118918,
      "loss_nats": 1.1987636089324951,
      "step": 3020,
      "unclipped_grad_norm": 0.6919088363647461
    },
    {
      "bits_per_byte": 1.7609028449601596,
      "learning_rate": 0.00022777563468820376,
      "loss_nats": 1.220564842224121,
      "step": 3030,
      "unclipped_grad_norm": 0.6205207109451294
    },
    {
      "bits_per_byte": 1.7350170482481948,
      "learning_rate": 0.00022728105700212282,
      "loss_nats": 1.2026221752166748,
      "step": 3040,
      "unclipped_grad_norm": 0.7004191279411316
    },
    {
      "bits_per_byte": 1.7472788953067604,
      "learning_rate": 0.00022678541510461698,
      "loss_nats": 1.2111214399337769,
      "step": 3050,
      "unclipped_grad_norm": 0.6576176881790161
    },
    {
      "bits_per_byte": 1.768669753455775,
      "learning_rate": 0.00022628871746483876,
      "loss_nats": 1.225948452949524,
      "step": 3060,
      "unclipped_grad_norm": 0.7110167145729065
    },
    {
      "bits_per_byte": 1.7354397816040157,
      "learning_rate": 0.0002257909725699804,
      "loss_nats": 1.2029151916503906,
      "step": 3070,
      "unclipped_grad_norm": 0.6717135906219482
    },
    {
      "bits_per_byte": 1.748681069859245,
      "learning_rate": 0.00022529218892512886,
      "loss_nats": 1.2120933532714844,
      "step": 3080,
      "unclipped_grad_norm": 0.6585274934768677
    },
    {
      "bits_per_byte": 1.7073046238014162,
      "learning_rate": 0.00022479237505312034,
      "loss_nats": 1.1834133863449097,
      "step": 3090,
      "unclipped_grad_norm": 0.6292386651039124
    },
    {
      "bits_per_byte": 1.8085337840819162,
      "learning_rate": 0.00022429153949439496,
      "loss_nats": 1.253580093383789,
      "step": 3100,
      "unclipped_grad_norm": 0.6514226198196411
    },
    {
      "bits_per_byte": 1.7739243393875863,
      "learning_rate": 0.00022378969080685053,
      "loss_nats": 1.229590654373169,
      "step": 3110,
      "unclipped_grad_norm": 0.6451593041419983
    },
    {
      "bits_per_byte": 1.6671654209338944,
      "learning_rate": 0.0002232868375656965,
      "loss_nats": 1.1555910110473633,
      "step": 3120,
      "unclipped_grad_norm": 0.6327124834060669
    },
    {
      "bits_per_byte": 1.7555804978639082,
      "learning_rate": 0.00022278298836330733,
      "loss_nats": 1.216875672340393,
      "step": 3130,
      "unclipped_grad_norm": 0.6219494342803955
    },
    {
      "bits_per_byte": 1.7189500850392896,
      "learning_rate": 0.00022227815180907572,
      "loss_nats": 1.1914854049682617,
      "step": 3140,
      "unclipped_grad_norm": 0.6722466349601746
    },
    {
      "bits_per_byte": 1.6998178750440565,
      "learning_rate": 0.00022177233652926546,
      "loss_nats": 1.178223967552185,
      "step": 3150,
      "unclipped_grad_norm": 0.6060081720352173
    },
    {
      "bits_per_byte": 1.7570095017099274,
      "learning_rate": 0.00022126555116686416,
      "loss_nats": 1.2178661823272705,
      "step": 3160,
      "unclipped_grad_norm": 0.6486148834228516
    },
    {
      "bits_per_byte": 1.8187225522669817,
      "learning_rate": 0.00022075780438143532,
      "loss_nats": 1.260642409324646,
      "step": 3170,
      "unclipped_grad_norm": 0.6782575845718384
    },
    {
      "bits_per_byte": 1.680190870962291,
      "learning_rate": 0.00022024910484897065,
      "loss_nats": 1.1646195650100708,
      "step": 3180,
      "unclipped_grad_norm": 0.6483905911445618
    },
    {
      "bits_per_byte": 1.7650935462137336,
      "learning_rate": 0.00021973946126174158,
      "loss_nats": 1.223469614982605,
      "step": 3190,
      "unclipped_grad_norm": 0.7115707993507385
    },
    {
      "bits_per_byte": 1.6804184040093824,
      "learning_rate": 0.00021922888232815086,
      "loss_nats": 1.1647772789001465,
      "step": 3200,
      "unclipped_grad_norm": 0.6299659609794617
    },
    {
      "bits_per_byte": 1.7133916057634035,
      "learning_rate": 0.0002187173767725837,
      "loss_nats": 1.1876325607299805,
      "step": 3210,
      "unclipped_grad_norm": 0.6491534113883972
    },
    {
      "bits_per_byte": 1.710321027514899,
      "learning_rate": 0.0002182049533352588,
      "loss_nats": 1.1855041980743408,
      "step": 3220,
      "unclipped_grad_norm": 0.6104377508163452
    },
    {
      "bits_per_byte": 1.6818119794293238,
      "learning_rate": 0.00021769162077207884,
      "loss_nats": 1.1657432317733765,
      "step": 3230,
      "unclipped_grad_norm": 0.6708391308784485
    },
    {
      "bits_per_byte": 1.7427318460005987,
      "learning_rate": 0.00021717738785448102,
      "loss_nats": 1.2079696655273438,
      "step": 3240,
      "unclipped_grad_norm": 0.6418254375457764
    },
    {
      "bits_per_byte": 1.733884714474914,
      "learning_rate": 0.00021666226336928708,
      "loss_nats": 1.2018373012542725,
      "step": 3250,
      "unclipped_grad_norm": 0.6041762828826904
    },
    {
      "bits_per_byte": 1.83323737205185,
      "learning_rate": 0.0002161462561185532,
      "loss_nats": 1.2707033157348633,
      "step": 3260,
      "unclipped_grad_norm": 0.7252490520477295
    },
    {
      "bits_per_byte": 1.6948699341787334,
      "learning_rate": 0.00021562937491941957,
      "loss_nats": 1.174794316291809,
      "step": 3270,
      "unclipped_grad_norm": 0.6451477408409119
    },
    {
      "bits_per_byte": 1.6338448142251303,
      "learning_rate": 0.00021511162860395975,
      "loss_nats": 1.1324949264526367,
      "step": 3280,
      "unclipped_grad_norm": 0.6251513361930847
    },
    {
      "bits_per_byte": 1.6883934115125416,
      "learning_rate": 0.0002145930260190298,
      "loss_nats": 1.1703051328659058,
      "step": 3290,
      "unclipped_grad_norm": 0.6296114921569824
    },
    {
      "bits_per_byte": 1.744128001160303,
      "learning_rate": 0.00021407357602611698,
      "loss_nats": 1.208937406539917,
      "step": 3300,
      "unclipped_grad_norm": 0.6343309283256531
    },
    {
      "bits_per_byte": 1.735473146238283,
      "learning_rate": 0.0002135532875011885,
      "loss_nats": 1.2029383182525635,
      "step": 3310,
      "unclipped_grad_norm": 0.6353676915168762
    },
    {
      "bits_per_byte": 1.7632710460625405,
      "learning_rate": 0.0002130321693345397,
      "loss_nats": 1.2222063541412354,
      "step": 3320,
      "unclipped_grad_norm": 0.6778877973556519
    },
    {
      "bits_per_byte": 1.6989030993241172,
      "learning_rate": 0.00021251023043064224,
      "loss_nats": 1.1775898933410645,
      "step": 3330,
      "unclipped_grad_norm": 0.629153847694397
    },
    {
      "bits_per_byte": 1.7071973066272779,
      "learning_rate": 0.00021198747970799188,
      "loss_nats": 1.18333899974823,
      "step": 3340,
      "unclipped_grad_norm": 0.6583864688873291
    },
    {
      "bits_per_byte": 1.70429750715108,
      "learning_rate": 0.00021146392609895618,
      "loss_nats": 1.1813290119171143,
      "step": 3350,
      "unclipped_grad_norm": 0.6497669816017151
    },
    {
      "bits_per_byte": 1.7138317093669615,
      "learning_rate": 0.00021093957854962178,
      "loss_nats": 1.187937617301941,
      "step": 3360,
      "unclipped_grad_norm": 0.6299119591712952
    },
    {
      "bits_per_byte": 1.6462306607749682,
      "learning_rate": 0.00021041444601964157,
      "loss_nats": 1.1410801410675049,
      "step": 3370,
      "unclipped_grad_norm": 0.6380488872528076
    },
    {
      "bits_per_byte": 1.7443316286189245,
      "learning_rate": 0.00020988853748208164,
      "loss_nats": 1.2090785503387451,
      "step": 3380,
      "unclipped_grad_norm": 0.702843189239502
    },
    {
      "bits_per_byte": 1.7228774808543925,
      "learning_rate": 0.00020936186192326782,
      "loss_nats": 1.1942076683044434,
      "step": 3390,
      "unclipped_grad_norm": 0.6957465410232544
    },
    {
      "bits_per_byte": 1.7111405248462603,
      "learning_rate": 0.00020883442834263225,
      "loss_nats": 1.1860722303390503,
      "step": 3400,
      "unclipped_grad_norm": 0.6491814255714417
    },
    {
      "bits_per_byte": 1.7448699343161251,
      "learning_rate": 0.0002083062457525596,
      "loss_nats": 1.209451675415039,
      "step": 3410,
      "unclipped_grad_norm": 0.6583186388015747
    },
    {
      "bits_per_byte": 1.7129637129280568,
      "learning_rate": 0.00020777732317823306,
      "loss_nats": 1.1873359680175781,
      "step": 3420,
      "unclipped_grad_norm": 0.6641048192977905
    },
    {
      "bits_per_byte": 1.670370317632723,
      "learning_rate": 0.00020724766965748005,
      "loss_nats": 1.157812476158142,
      "step": 3430,
      "unclipped_grad_norm": 0.629267692565918
    },
    {
      "bits_per_byte": 1.6804600238108913,
      "learning_rate": 0.00020671729424061788,
      "loss_nats": 1.1648061275482178,
      "step": 3440,
      "unclipped_grad_norm": 0.6077277064323425
    },
    {
      "bits_per_byte": 1.6713492428814338,
      "learning_rate": 0.00020618620599029906,
      "loss_nats": 1.1584910154342651,
      "step": 3450,
      "unclipped_grad_norm": 0.6522519588470459
    },
    {
      "bits_per_byte": 1.669551508231965,
      "learning_rate": 0.00020565441398135649,
      "loss_nats": 1.1572449207305908,
      "step": 3460,
      "unclipped_grad_norm": 0.627102255821228
    },
    {
      "bits_per_byte": 1.7251744811393164,
      "learning_rate": 0.00020512192730064827,
      "loss_nats": 1.1957998275756836,
      "step": 3470,
      "unclipped_grad_norm": 0.6381968855857849
    },
    {
      "bits_per_byte": 1.7823232841251224,
      "learning_rate": 0.00020458875504690273,
      "loss_nats": 1.235412359237671,
      "step": 3480,
      "unclipped_grad_norm": 0.6999886631965637
    },
    {
      "bits_per_byte": 1.718624865846508,
      "learning_rate": 0.00020405490633056245,
      "loss_nats": 1.1912599802017212,
      "step": 3490,
      "unclipped_grad_norm": 0.6294557452201843
    },
    {
      "bits_per_byte": 1.6614911973339777,
      "learning_rate": 0.00020352039027362917,
      "loss_nats": 1.1516579389572144,
      "step": 3500,
      "unclipped_grad_norm": 0.6351874470710754
    },
    {
      "bits_per_byte": 1.6797325371977416,
      "learning_rate": 0.0002029852160095075,
      "loss_nats": 1.164301872253418,
      "step": 3510,
      "unclipped_grad_norm": 0.6315440535545349
    },
    {
      "bits_per_byte": 1.717181759423118,
      "learning_rate": 0.000202449392682849,
      "loss_nats": 1.1902596950531006,
      "step": 3520,
      "unclipped_grad_norm": 0.6176374554634094
    },
    {
      "bits_per_byte": 1.6772098956748858,
      "learning_rate": 0.0002019129294493959,
      "loss_nats": 1.162553310394287,
      "step": 3530,
      "unclipped_grad_norm": 0.6610027551651001
    },
    {
      "bits_per_byte": 1.6855345439072493,
      "learning_rate": 0.00020137583547582478,
      "loss_nats": 1.1683235168457031,
      "step": 3540,
      "unclipped_grad_norm": 0.658635139465332
    },
    {
      "bits_per_byte": 1.747215777673894,
      "learning_rate": 0.00020083811993958973,
      "loss_nats": 1.2110776901245117,
      "step": 3550,
      "unclipped_grad_norm": 0.6718185544013977
    },
    {
      "bits_per_byte": 1.666718266041651,
      "learning_rate": 0.00020029979202876563,
      "loss_nats": 1.1552810668945312,
      "step": 3560,
      "unclipped_grad_norm": 0.6717336773872375
    },
    {
      "bits_per_byte": 1.7121595220526225,
      "learning_rate": 0.00019976086094189118,
      "loss_nats": 1.1867785453796387,
      "step": 3570,
      "unclipped_grad_norm": 0.6859359741210938
    },
    {
      "bits_per_byte": 1.739672790589702,
      "learning_rate": 0.00019922133588781175,
      "loss_nats": 1.205849289894104,
      "step": 3580,
      "unclipped_grad_norm": 0.6660984754562378
    },
    {
      "bits_per_byte": 1.6956691375572928,
      "learning_rate": 0.0001986812260855219,
      "loss_nats": 1.1753482818603516,
      "step": 3590,
      "unclipped_grad_norm": 0.6079815030097961
    },
    {
      "bits_per_byte": 1.7521809168043032,
      "learning_rate": 0.00019814054076400787,
      "loss_nats": 1.2145192623138428,
      "step": 3600,
      "unclipped_grad_norm": 0.7036373615264893
    },
    {
      "bits_per_byte": 1.7346027420422663,
      "learning_rate": 0.00019759928916209004,
      "loss_nats": 1.202335000038147,
      "step": 3610,
      "unclipped_grad_norm": 0.6451415419578552
    },
    {
      "bits_per_byte": 1.6701776970637565,
      "learning_rate": 0.00019705748052826493,
      "loss_nats": 1.1576789617538452,
      "step": 3620,
      "unclipped_grad_norm": 0.620152473449707
    },
    {
      "bits_per_byte": 1.6714409096343439,
      "learning_rate": 0.00019651512412054723,
      "loss_nats": 1.1585545539855957,
      "step": 3630,
      "unclipped_grad_norm": 0.6908113956451416
    },
    {
      "bits_per_byte": 1.6125346159393705,
      "learning_rate": 0.00019597222920631144,
      "loss_nats": 1.117723822593689,
      "step": 3640,
      "unclipped_grad_norm": 0.604874312877655
    },
    {
      "bits_per_byte": 1.6950919637809974,
      "learning_rate": 0.0001954288050621338,
      "loss_nats": 1.1749482154846191,
      "step": 3650,
      "unclipped_grad_norm": 0.6512246131896973
    },
    {
      "bits_per_byte": 1.6897813115050047,
      "learning_rate": 0.00019488486097363363,
      "loss_nats": 1.1712671518325806,
      "step": 3660,
      "unclipped_grad_norm": 0.647068440914154
    },
    {
      "bits_per_byte": 1.7204222565306753,
      "learning_rate": 0.00019434040623531452,
      "loss_nats": 1.1925058364868164,
      "step": 3670,
      "unclipped_grad_norm": 0.7147674560546875
    },
    {
      "bits_per_byte": 1.6652036148355023,
      "learning_rate": 0.0001937954501504058,
      "loss_nats": 1.1542311906814575,
      "step": 3680,
      "unclipped_grad_norm": 0.6392054557800293
    },
    {
      "bits_per_byte": 1.7297106955384738,
      "learning_rate": 0.00019325000203070333,
      "loss_nats": 1.198944091796875,
      "step": 3690,
      "unclipped_grad_norm": 0.7120746374130249
    },
    {
      "bits_per_byte": 1.6313792709423611,
      "learning_rate": 0.0001927040711964106,
      "loss_nats": 1.1307859420776367,
      "step": 3700,
      "unclipped_grad_norm": 0.6207358241081238
    },
    {
      "bits_per_byte": 1.6932649920808815,
      "learning_rate": 0.00019215766697597923,
      "loss_nats": 1.1736818552017212,
      "step": 3710,
      "unclipped_grad_norm": 0.6575222611427307
    },
    {
      "bits_per_byte": 1.7191832935138596,
      "learning_rate": 0.00019161079870594978,
      "loss_nats": 1.1916470527648926,
      "step": 3720,
      "unclipped_grad_norm": 0.6928348541259766
    },
    {
      "bits_per_byte": 1.6966805675270171,
      "learning_rate": 0.0001910634757307922,
      "loss_nats": 1.1760493516921997,
      "step": 3730,
      "unclipped_grad_norm": 0.6457412242889404
    },
    {
      "bits_per_byte": 1.6651184834233252,
      "learning_rate": 0.0001905157074027459,
      "loss_nats": 1.1541721820831299,
      "step": 3740,
      "unclipped_grad_norm": 0.6149929761886597
    },
    {
      "bits_per_byte": 1.690470445987012,
      "learning_rate": 0.00018996750308166037,
      "loss_nats": 1.1717448234558105,
      "step": 3750,
      "unclipped_grad_norm": 0.6539504528045654
    },
    {
      "bits_per_byte": 1.7257394441474005,
      "learning_rate": 0.00018941887213483482,
      "loss_nats": 1.196191430091858,
      "step": 3760,
      "unclipped_grad_norm": 0.6810935735702515
    },
    {
      "bits_per_byte": 1.709021182639678,
      "learning_rate": 0.00018886982393685845,
      "loss_nats": 1.184603214263916,
      "step": 3770,
      "unclipped_grad_norm": 0.6034973859786987
    },
    {
      "bits_per_byte": 1.7700676284419878,
      "learning_rate": 0.00018832036786945008,
      "loss_nats": 1.2269173860549927,
      "step": 3780,
      "unclipped_grad_norm": 0.6868588328361511
    },
    {
      "bits_per_byte": 1.6607432447853754,
      "learning_rate": 0.0001877705133212979,
      "loss_nats": 1.151139497756958,
      "step": 3790,
      "unclipped_grad_norm": 0.6561638712882996
    },
    {
      "bits_per_byte": 1.6881032767805362,
      "learning_rate": 0.00018722026968789906,
      "loss_nats": 1.1701040267944336,
      "step": 3800,
      "unclipped_grad_norm": 0.6706013679504395
    },
    {
      "bits_per_byte": 1.6338372469884925,
      "learning_rate": 0.00018666964637139907,
      "loss_nats": 1.1324896812438965,
      "step": 3810,
      "unclipped_grad_norm": 0.6373327374458313
    },
    {
      "bits_per_byte": 1.6310115720348164,
      "learning_rate": 0.00018611865278043115,
      "loss_nats": 1.1305310726165771,
      "step": 3820,
      "unclipped_grad_norm": 0.6431099772453308
    },
    {
      "bits_per_byte": 1.699418359346102,
      "learning_rate": 0.00018556729832995555,
      "loss_nats": 1.1779470443725586,
      "step": 3830,
      "unclipped_grad_norm": 0.6568827629089355
    },
    {
      "bits_per_byte": 1.6534394855685337,
      "learning_rate": 0.00018501559244109864,
      "loss_nats": 1.1460769176483154,
      "step": 3840,
      "unclipped_grad_norm": 0.6673737168312073
    },
    {
      "bits_per_byte": 1.6227488375567638,
      "learning_rate": 0.00018446354454099178,
      "loss_nats": 1.1248037815093994,
      "step": 3850,
      "unclipped_grad_norm": 0.6300650835037231
    },
    {
      "bits_per_byte": 1.6095687751252414,
      "learning_rate": 0.00018391116406261048,
      "loss_nats": 1.1156680583953857,
      "step": 3860,
      "unclipped_grad_norm": 0.6291298270225525
    },
    {
      "bits_per_byte": 1.5641092889532788,
      "learning_rate": 0.00018335846044461305,
      "loss_nats": 1.084157943725586,
      "step": 3870,
      "unclipped_grad_norm": 0.619691789150238
    },
    {
      "bits_per_byte": 1.670964173726152,
      "learning_rate": 0.0001828054431311793,
      "loss_nats": 1.158224105834961,
      "step": 3880,
      "unclipped_grad_norm": 0.6558656692504883
    },
    {
      "bits_per_byte": 1.6390438497607132,
      "learning_rate": 0.00018225212157184934,
      "loss_nats": 1.1360986232757568,
      "step": 3890,
      "unclipped_grad_norm": 0.6659069061279297
    },
    {
      "bits_per_byte": 1.6436039697483404,
      "learning_rate": 0.00018169850522136192,
      "loss_nats": 1.1392594575881958,
      "step": 3900,
      "unclipped_grad_norm": 0.6627041101455688
    },
    {
      "bits_per_byte": 1.6594528589559516,
      "learning_rate": 0.000181144603539493,
      "loss_nats": 1.1502450704574585,
      "step": 3910,
      "unclipped_grad_norm": 0.6734365224838257
    },
    {
      "bits_per_byte": 1.6941402117911224,
      "learning_rate": 0.00018059042599089412,
      "loss_nats": 1.1742885112762451,
      "step": 3920,
      "unclipped_grad_norm": 0.6930717825889587
    },
    {
      "bits_per_byte": 1.6772669939149722,
      "learning_rate": 0.00018003598204493048,
      "loss_nats": 1.162592887878418,
      "step": 3930,
      "unclipped_grad_norm": 0.7069801688194275
    },
    {
      "bits_per_byte": 1.6121323485190024,
      "learning_rate": 0.00017948128117551936,
      "loss_nats": 1.1174449920654297,
      "step": 3940,
      "unclipped_grad_norm": 0.6367635726928711
    },
    {
      "bits_per_byte": 1.7182599186613772,
      "learning_rate": 0.00017892633286096816,
      "loss_nats": 1.1910070180892944,
      "step": 3950,
      "unclipped_grad_norm": 0.690847635269165
    },
    {
      "bits_per_byte": 1.6993077745015972,
      "learning_rate": 0.00017837114658381247,
      "loss_nats": 1.1778703927993774,
      "step": 3960,
      "unclipped_grad_norm": 0.6660118699073792
    },
    {
      "bits_per_byte": 1.6431734971732308,
      "learning_rate": 0.00017781573183065386,
      "loss_nats": 1.1389610767364502,
      "step": 3970,
      "unclipped_grad_norm": 0.669096827507019
    },
    {
      "bits_per_byte": 1.6275112091418058,
      "learning_rate": 0.0001772600980919981,
      "loss_nats": 1.12810480594635,
      "step": 3980,
      "unclipped_grad_norm": 0.6583772301673889
    },
    {
      "bits_per_byte": 1.669720739160414,
      "learning_rate": 0.0001767042548620927,
      "loss_nats": 1.1573622226715088,
      "step": 3990,
      "unclipped_grad_norm": 0.6523779630661011
    },
    {
      "bits_per_byte": 1.5952216384249642,
      "learning_rate": 0.00017614821163876486,
      "loss_nats": 1.1057233810424805,
      "step": 4000,
      "unclipped_grad_norm": 0.6264373064041138
    },
    {
      "bits_per_byte": 1.6774286576067834,
      "learning_rate": 0.00017559197792325906,
      "loss_nats": 1.1627049446105957,
      "step": 4010,
      "unclipped_grad_norm": 0.6995967030525208
    },
    {
      "bits_per_byte": 1.6794896976947233,
      "learning_rate": 0.0001750355632200748,
      "loss_nats": 1.1641335487365723,
      "step": 4020,
      "unclipped_grad_norm": 0.6873654127120972
    },
    {
      "bits_per_byte": 1.6416081110850775,
      "learning_rate": 0.00017447897703680418,
      "loss_nats": 1.137876033782959,
      "step": 4030,
      "unclipped_grad_norm": 0.7148170471191406
    },
    {
      "bits_per_byte": 1.62684030482079,
      "learning_rate": 0.0001739222288839694,
      "loss_nats": 1.1276397705078125,
      "step": 4040,
      "unclipped_grad_norm": 0.6498348712921143
    },
    {
      "bits_per_byte": 1.644293964143602,
      "learning_rate": 0.00017336532827486026,
      "loss_nats": 1.1397377252578735,
      "step": 4050,
      "unclipped_grad_norm": 0.6837151646614075
    },
    {
      "bits_per_byte": 1.6349320885438852,
      "learning_rate": 0.00017280828472537163,
      "loss_nats": 1.1332485675811768,
      "step": 4060,
      "unclipped_grad_norm": 0.6708939671516418
    },
    {
      "bits_per_byte": 1.769896849669681,
      "learning_rate": 0.0001722511077538408,
      "loss_nats": 1.2267990112304688,
      "step": 4070,
      "unclipped_grad_norm": 0.7203540802001953
    },
    {
      "bits_per_byte": 1.5942710903136454,
      "learning_rate": 0.00017169380688088497,
      "loss_nats": 1.1050645112991333,
      "step": 4080,
      "unclipped_grad_norm": 0.6567085385322571
    },
    {
      "bits_per_byte": 1.6276448396615262,
      "learning_rate": 0.0001711363916292384,
      "loss_nats": 1.128197431564331,
      "step": 4090,
      "unclipped_grad_norm": 0.6684355139732361
    },
    {
      "bits_per_byte": 1.7285615074654084,
      "learning_rate": 0.00017057887152358974,
      "loss_nats": 1.1981475353240967,
      "step": 4100,
      "unclipped_grad_norm": 0.7356466054916382
    },
    {
      "bits_per_byte": 1.6342331510507786,
      "learning_rate": 0.0001700212560904194,
      "loss_nats": 1.1327641010284424,
      "step": 4110,
      "unclipped_grad_norm": 0.655012309551239
    },
    {
      "bits_per_byte": 1.6424126459258137,
      "learning_rate": 0.00016946355485783656,
      "loss_nats": 1.1384336948394775,
      "step": 4120,
      "unclipped_grad_norm": 0.7107924222946167
    },
    {
      "bits_per_byte": 1.6944599275390764,
      "learning_rate": 0.0001689057773554166,
      "loss_nats": 1.17451012134552,
      "step": 4130,
      "unclipped_grad_norm": 0.6998885869979858
    },
    {
      "bits_per_byte": 1.7017107160994525,
      "learning_rate": 0.00016834793311403806,
      "loss_nats": 1.179535984992981,
      "step": 4140,
      "unclipped_grad_norm": 0.6857012510299683
    },
    {
      "bits_per_byte": 1.6170882005862652,
      "learning_rate": 0.00016779003166571981,
      "loss_nats": 1.120880126953125,
      "step": 4150,
      "unclipped_grad_norm": 0.6785045266151428
    },
    {
      "bits_per_byte": 1.6430269679546958,
      "learning_rate": 0.00016723208254345838,
      "loss_nats": 1.138859510421753,
      "step": 4160,
      "unclipped_grad_norm": 0.6962723135948181
    },
    {
      "bits_per_byte": 1.6682418603456448,
      "learning_rate": 0.00016667409528106481,
      "loss_nats": 1.1563371419906616,
      "step": 4170,
      "unclipped_grad_norm": 0.7071301937103271
    },
    {
      "bits_per_byte": 1.6402630347726797,
      "learning_rate": 0.00016611607941300184,
      "loss_nats": 1.1369436979293823,
      "step": 4180,
      "unclipped_grad_norm": 0.6712477207183838
    },
    {
      "bits_per_byte": 1.67100166594404,
      "learning_rate": 0.0001655580444742211,
      "loss_nats": 1.158250093460083,
      "step": 4190,
      "unclipped_grad_norm": 0.6872346997261047
    },
    {
      "bits_per_byte": 1.6579426792987255,
      "learning_rate": 0.00016499999999999997,
      "loss_nats": 1.149198293685913,
      "step": 4200,
      "unclipped_grad_norm": 0.7030718326568604
    },
    {
      "bits_per_byte": 1.6808303024581988,
      "learning_rate": 0.0001644419555257789,
      "loss_nats": 1.1650627851486206,
      "step": 4210,
      "unclipped_grad_norm": 0.658928394317627
    },
    {
      "bits_per_byte": 1.659455094730413,
      "learning_rate": 0.0001638839205869981,
      "loss_nats": 1.1502466201782227,
      "step": 4220,
      "unclipped_grad_norm": 0.6767720580101013
    },
    {
      "bits_per_byte": 1.6554740403282406,
      "learning_rate": 0.0001633259047189352,
      "loss_nats": 1.1474871635437012,
      "step": 4230,
      "unclipped_grad_norm": 0.7309125065803528
    },
    {
      "bits_per_byte": 1.6771517655388943,
      "learning_rate": 0.0001627679174565416,
      "loss_nats": 1.162513017654419,
      "step": 4240,
      "unclipped_grad_norm": 0.7190505862236023
    },
    {
      "bits_per_byte": 1.6138194983239667,
      "learning_rate": 0.00016220996833428016,
      "loss_nats": 1.1186144351959229,
      "step": 4250,
      "unclipped_grad_norm": 0.6701101660728455
    },
    {
      "bits_per_byte": 1.6526860295751036,
      "learning_rate": 0.00016165206688596191,
      "loss_nats": 1.1455546617507935,
      "step": 4260,
      "unclipped_grad_norm": 0.6757012009620667
    },
    {
      "bits_per_byte": 1.5671267245626668,
      "learning_rate": 0.00016109422264458335,
      "loss_nats": 1.0862494707107544,
      "step": 4270,
      "unclipped_grad_norm": 0.647323727607727
    },
    {
      "bits_per_byte": 1.640895586962553,
      "learning_rate": 0.00016053644514216336,
      "loss_nats": 1.13738214969635,
      "step": 4280,
      "unclipped_grad_norm": 0.666140079498291
    },
    {
      "bits_per_byte": 1.6259669769197085,
      "learning_rate": 0.00015997874390958058,
      "loss_nats": 1.1270344257354736,
      "step": 4290,
      "unclipped_grad_norm": 0.6681356430053711
    },
    {
      "bits_per_byte": 1.5856241465935652,
      "learning_rate": 0.00015942112847641018,
      "loss_nats": 1.0990709066390991,
      "step": 4300,
      "unclipped_grad_norm": 0.6779436469078064
    },
    {
      "bits_per_byte": 1.5644216094472456,
      "learning_rate": 0.00015886360837076156,
      "loss_nats": 1.0843744277954102,
      "step": 4310,
      "unclipped_grad_norm": 0.6528535485267639
    },
    {
      "bits_per_byte": 1.6990217673532124,
      "learning_rate": 0.00015830619311911495,
      "loss_nats": 1.1776721477508545,
      "step": 4320,
      "unclipped_grad_norm": 0.6877329349517822
    },
    {
      "bits_per_byte": 1.6415279671697756,
      "learning_rate": 0.00015774889224615915,
      "loss_nats": 1.1378204822540283,
      "step": 4330,
      "unclipped_grad_norm": 0.7055521011352539
    },
    {
      "bits_per_byte": 1.604049851859054,
      "learning_rate": 0.00015719171527462834,
      "loss_nats": 1.1118426322937012,
      "step": 4340,
      "unclipped_grad_norm": 0.6865817308425903
    },
    {
      "bits_per_byte": 1.5633534252027368,
      "learning_rate": 0.0001566346717251397,
      "loss_nats": 1.0836340188980103,
      "step": 4350,
      "unclipped_grad_norm": 0.6733583211898804
    },
    {
      "bits_per_byte": 1.6505902489916895,
      "learning_rate": 0.00015607777111603056,
      "loss_nats": 1.1441019773483276,
      "step": 4360,
      "unclipped_grad_norm": 0.6889402270317078
    },
    {
      "bits_per_byte": 1.6184719729951074,
      "learning_rate": 0.00015552102296319576,
      "loss_nats": 1.1218392848968506,
      "step": 4370,
      "unclipped_grad_norm": 0.6762850284576416
    },
    {
      "bits_per_byte": 1.556092145700658,
      "learning_rate": 0.00015496443677992516,
      "loss_nats": 1.0786008834838867,
      "step": 4380,
      "unclipped_grad_norm": 0.6198620200157166
    },
    {
      "bits_per_byte": 1.7130244228038114,
      "learning_rate": 0.0001544080220767409,
      "loss_nats": 1.1873780488967896,
      "step": 4390,
      "unclipped_grad_norm": 0.6756254434585571
    },
    {
      "bits_per_byte": 1.6522954569749941,
      "learning_rate": 0.00015385178836123511,
      "loss_nats": 1.1452839374542236,
      "step": 4400,
      "unclipped_grad_norm": 0.6876093745231628
    },
    {
      "bits_per_byte": 1.661213789318136,
      "learning_rate": 0.00015329574513790726,
      "loss_nats": 1.151465654373169,
      "step": 4410,
      "unclipped_grad_norm": 0.6821660399436951
    },
    {
      "bits_per_byte": 1.6753433679650718,
      "learning_rate": 0.00015273990190800186,
      "loss_nats": 1.1612595319747925,
      "step": 4420,
      "unclipped_grad_norm": 0.7177299857139587
    },
    {
      "bits_per_byte": 1.6206171125993192,
      "learning_rate": 0.0001521842681693461,
      "loss_nats": 1.1233261823654175,
      "step": 4430,
      "unclipped_grad_norm": 0.6967712640762329
    },
    {
      "bits_per_byte": 1.6227820302083802,
      "learning_rate": 0.00015162885341618752,
      "loss_nats": 1.1248267889022827,
      "step": 4440,
      "unclipped_grad_norm": 0.6610331535339355
    },
    {
      "bits_per_byte": 1.6554300127696198,
      "learning_rate": 0.00015107366713903178,
      "loss_nats": 1.1474566459655762,
      "step": 4450,
      "unclipped_grad_norm": 0.6704992055892944
    },
    {
      "bits_per_byte": 1.6458450756717338,
      "learning_rate": 0.0001505187188244806,
      "loss_nats": 1.140812873840332,
      "step": 4460,
      "unclipped_grad_norm": 0.7582810521125793
    },
    {
      "bits_per_byte": 1.6144303806998315,
      "learning_rate": 0.00014996401795506952,
      "loss_nats": 1.1190378665924072,
      "step": 4470,
      "unclipped_grad_norm": 0.7301307916641235
    },
    {
      "bits_per_byte": 1.6560006512051826,
      "learning_rate": 0.00014940957400910585,
      "loss_nats": 1.1478521823883057,
      "step": 4480,
      "unclipped_grad_norm": 0.7387185096740723
    },
    {
      "bits_per_byte": 1.6943782357799166,
      "learning_rate": 0.00014885539646050699,
      "loss_nats": 1.1744534969329834,
      "step": 4490,
      "unclipped_grad_norm": 0.7154380083084106
    },
    {
      "bits_per_byte": 1.6097710267226562,
      "learning_rate": 0.00014830149477863805,
      "loss_nats": 1.1158082485198975,
      "step": 4500,
      "unclipped_grad_norm": 0.6710496544837952
    },
    {
      "bits_per_byte": 1.6061318738303991,
      "learning_rate": 0.00014774787842815066,
      "loss_nats": 1.113285779953003,
      "step": 4510,
      "unclipped_grad_norm": 0.7097434401512146
    },
    {
      "bits_per_byte": 1.676021495557424,
      "learning_rate": 0.00014719455686882066,
      "loss_nats": 1.1617295742034912,
      "step": 4520,
      "unclipped_grad_norm": 0.7576807141304016
    },
    {
      "bits_per_byte": 1.635137779794317,
      "learning_rate": 0.00014664153955538695,
      "loss_nats": 1.1333911418914795,
      "step": 4530,
      "unclipped_grad_norm": 0.6911670565605164
    },
    {
      "bits_per_byte": 1.64362185594403,
      "learning_rate": 0.00014608883593738946,
      "loss_nats": 1.139271855354309,
      "step": 4540,
      "unclipped_grad_norm": 0.6786331534385681
    },
    {
      "bits_per_byte": 1.650524379636409,
      "learning_rate": 0.00014553645545900817,
      "loss_nats": 1.1440563201904297,
      "step": 4550,
      "unclipped_grad_norm": 0.6853471994400024
    },
    {
      "bits_per_byte": 1.6363368428361325,
      "learning_rate": 0.0001449844075589013,
      "loss_nats": 1.1342222690582275,
      "step": 4560,
      "unclipped_grad_norm": 0.7256327867507935
    },
    {
      "bits_per_byte": 1.6365012582503573,
      "learning_rate": 0.0001444327016700444,
      "loss_nats": 1.134336233139038,
      "step": 4570,
      "unclipped_grad_norm": 0.697358250617981
    },
    {
      "bits_per_byte": 1.6112284077060677,
      "learning_rate": 0.0001438813472195688,
      "loss_nats": 1.1168184280395508,
      "step": 4580,
      "unclipped_grad_norm": 0.6777223348617554
    },
    {
      "bits_per_byte": 1.5962301446896239,
      "learning_rate": 0.00014333035362860093,
      "loss_nats": 1.1064224243164062,
      "step": 4590,
      "unclipped_grad_norm": 0.7330402731895447
    },
    {
      "bits_per_byte": 1.5998985346325274,
      "learning_rate": 0.00014277973031210088,
      "loss_nats": 1.1089651584625244,
      "step": 4600,
      "unclipped_grad_norm": 0.6789212822914124
    },
    {
      "bits_per_byte": 1.596745060746307,
      "learning_rate": 0.00014222948667870207,
      "loss_nats": 1.1067793369293213,
      "step": 4610,
      "unclipped_grad_norm": 0.6819480061531067
    },
    {
      "bits_per_byte": 1.6577524664868712,
      "learning_rate": 0.00014167963213054991,
      "loss_nats": 1.14906644821167,
      "step": 4620,
      "unclipped_grad_norm": 0.7283186912536621
    },
    {
      "bits_per_byte": 1.646706020821953,
      "learning_rate": 0.00014113017606314152,
      "loss_nats": 1.1414096355438232,
      "step": 4630,
      "unclipped_grad_norm": 0.7197591662406921
    },
    {
      "bits_per_byte": 1.6542598428131492,
      "learning_rate": 0.00014058112786516518,
      "loss_nats": 1.1466455459594727,
      "step": 4640,
      "unclipped_grad_norm": 0.7083833813667297
    },
    {
      "bits_per_byte": 1.6790158854915962,
      "learning_rate": 0.0001400324969183396,
      "loss_nats": 1.1638051271438599,
      "step": 4650,
      "unclipped_grad_norm": 0.684671938419342
    },
    {
      "bits_per_byte": 1.6168221434253804,
      "learning_rate": 0.00013948429259725406,
      "loss_nats": 1.12069571018219,
      "step": 4660,
      "unclipped_grad_norm": 0.7103760838508606
    },
    {
      "bits_per_byte": 1.6034386255178876,
      "learning_rate": 0.00013893652426920776,
      "loss_nats": 1.1114189624786377,
      "step": 4670,
      "unclipped_grad_norm": 0.6704027652740479
    },
    {
      "bits_per_byte": 1.591610346722147,
      "learning_rate": 0.00013838920129405016,
      "loss_nats": 1.1032202243804932,
      "step": 4680,
      "unclipped_grad_norm": 0.7572834491729736
    },
    {
      "bits_per_byte": 1.6094411639983013,
      "learning_rate": 0.0001378423330240207,
      "loss_nats": 1.115579605102539,
      "step": 4690,
      "unclipped_grad_norm": 0.6773369312286377
    },
    {
      "bits_per_byte": 1.6221410508686145,
      "learning_rate": 0.00013729592880358936,
      "loss_nats": 1.124382495880127,
      "step": 4700,
      "unclipped_grad_norm": 0.7124287486076355
    },
    {
      "bits_per_byte": 1.6214163159778787,
      "learning_rate": 0.00013674999796929659,
      "loss_nats": 1.12388014793396,
      "step": 4710,
      "unclipped_grad_norm": 0.7075444459915161
    },
    {
      "bits_per_byte": 1.6415025137374477,
      "learning_rate": 0.00013620454984959417,
      "loss_nats": 1.1378028392791748,
      "step": 4720,
      "unclipped_grad_norm": 0.7044992446899414
    },
    {
      "bits_per_byte": 1.654741394235565,
      "learning_rate": 0.00013565959376468542,
      "loss_nats": 1.1469793319702148,
      "step": 4730,
      "unclipped_grad_norm": 0.7124809622764587
    },
    {
      "bits_per_byte": 1.6107767812649019,
      "learning_rate": 0.00013511513902636634,
      "loss_nats": 1.1165053844451904,
      "step": 4740,
      "unclipped_grad_norm": 0.7306708693504333
    },
    {
      "bits_per_byte": 1.5756537963750956,
      "learning_rate": 0.0001345711949378662,
      "loss_nats": 1.0921599864959717,
      "step": 4750,
      "unclipped_grad_norm": 0.6872034668922424
    },
    {
      "bits_per_byte": 1.550611402582962,
      "learning_rate": 0.00013402777079368853,
      "loss_nats": 1.0748019218444824,
      "step": 4760,
      "unclipped_grad_norm": 0.6966266632080078
    },
    {
      "bits_per_byte": 1.5558193812163896,
      "learning_rate": 0.0001334848758794528,
      "loss_nats": 1.0784118175506592,
      "step": 4770,
      "unclipped_grad_norm": 0.6661347150802612
    },
    {
      "bits_per_byte": 1.5972586009417833,
      "learning_rate": 0.00013294251947173504,
      "loss_nats": 1.10713529586792,
      "step": 4780,
      "unclipped_grad_norm": 0.7209707498550415
    },
    {
      "bits_per_byte": 1.6013201432245594,
      "learning_rate": 0.00013240071083790996,
      "loss_nats": 1.1099505424499512,
      "step": 4790,
      "unclipped_grad_norm": 0.7224242091178894
    },
    {
      "bits_per_byte": 1.6214809814546032,
      "learning_rate": 0.0001318594592359921,
      "loss_nats": 1.123924970626831,
      "step": 4800,
      "unclipped_grad_norm": 0.7595614790916443
    },
    {
      "bits_per_byte": 1.598943858937588,
      "learning_rate": 0.0001313187739144781,
      "loss_nats": 1.108303427696228,
      "step": 4810,
      "unclipped_grad_norm": 0.6932185292243958
    },
    {
      "bits_per_byte": 1.5882840302718095,
      "learning_rate": 0.0001307786641121882,
      "loss_nats": 1.1009145975112915,
      "step": 4820,
      "unclipped_grad_norm": 0.7010871171951294
    },
    {
      "bits_per_byte": 1.5957891811728115,
      "learning_rate": 0.00013023913905810876,
      "loss_nats": 1.106116771697998,
      "step": 4830,
      "unclipped_grad_norm": 0.6711499691009521
    },
    {
      "bits_per_byte": 1.5337598545208206,
      "learning_rate": 0.00012970020797123432,
      "loss_nats": 1.0631213188171387,
      "step": 4840,
      "unclipped_grad_norm": 0.7005295157432556
    },
    {
      "bits_per_byte": 1.6063714456630511,
      "learning_rate": 0.00012916188006041027,
      "loss_nats": 1.1134518384933472,
      "step": 4850,
      "unclipped_grad_norm": 0.6924180388450623
    },
    {
      "bits_per_byte": 1.58919708616524,
      "learning_rate": 0.00012862416452417516,
      "loss_nats": 1.1015474796295166,
      "step": 4860,
      "unclipped_grad_norm": 0.6728304028511047
    },
    {
      "bits_per_byte": 1.5949746713383253,
      "learning_rate": 0.00012808707055060405,
      "loss_nats": 1.1055521965026855,
      "step": 4870,
      "unclipped_grad_norm": 0.6908511519432068
    },
    {
      "bits_per_byte": 1.562132004416309,
      "learning_rate": 0.000127550607317151,
      "loss_nats": 1.0827873945236206,
      "step": 4880,
      "unclipped_grad_norm": 0.6754211187362671
    },
    {
      "bits_per_byte": 1.5204766024987808,
      "learning_rate": 0.00012701478399049248,
      "loss_nats": 1.0539140701293945,
      "step": 4890,
      "unclipped_grad_norm": 0.686815619468689
    },
    {
      "bits_per_byte": 1.522231341485534,
      "learning_rate": 0.0001264796097263708,
      "loss_nats": 1.0551303625106812,
      "step": 4900,
      "unclipped_grad_norm": 0.6894630789756775
    },
    {
      "bits_per_byte": 1.5875051208460518,
      "learning_rate": 0.0001259450936694375,
      "loss_nats": 1.100374698638916,
      "step": 4910,
      "unclipped_grad_norm": 0.7035151124000549
    },
    {
      "bits_per_byte": 1.5873929881576891,
      "learning_rate": 0.00012541124495309727,
      "loss_nats": 1.100296974182129,
      "step": 4920,
      "unclipped_grad_norm": 0.707631528377533
    },
    {
      "bits_per_byte": 1.5818637459324498,
      "learning_rate": 0.00012487807269935167,
      "loss_nats": 1.0964643955230713,
      "step": 4930,
      "unclipped_grad_norm": 0.7146247029304504
    },
    {
      "bits_per_byte": 1.6602286726939939,
      "learning_rate": 0.00012434558601864349,
      "loss_nats": 1.150782823562622,
      "step": 4940,
      "unclipped_grad_norm": 0.7488285899162292
    },
    {
      "bits_per_byte": 1.5109480757103777,
      "learning_rate": 0.0001238137940097009,
      "loss_nats": 1.047309398651123,
      "step": 4950,
      "unclipped_grad_norm": 0.6982936263084412
    },
    {
      "bits_per_byte": 1.6013034609074257,
      "learning_rate": 0.0001232827057593821,
      "loss_nats": 1.1099389791488647,
      "step": 4960,
      "unclipped_grad_norm": 0.7697318196296692
    },
    {
      "bits_per_byte": 1.673659485830474,
      "learning_rate": 0.00012275233034251992,
      "loss_nats": 1.1600923538208008,
      "step": 4970,
      "unclipped_grad_norm": 0.7759159207344055
    },
    {
      "bits_per_byte": 1.5500858236019253,
      "learning_rate": 0.0001222226768217669,
      "loss_nats": 1.0744376182556152,
      "step": 4980,
      "unclipped_grad_norm": 0.7765741944313049
    },
    {
      "bits_per_byte": 1.6026017579387892,
      "learning_rate": 0.00012169375424744034,
      "loss_nats": 1.1108388900756836,
      "step": 4990,
      "unclipped_grad_norm": 0.7224560976028442
    },
    {
      "bits_per_byte": 1.6136506113608193,
      "learning_rate": 0.00012116557165736772,
      "loss_nats": 1.118497371673584,
      "step": 5000,
      "unclipped_grad_norm": 0.7415750622749329
    },
    {
      "bits_per_byte": 1.5910434919049032,
      "learning_rate": 0.00012063813807673214,
      "loss_nats": 1.1028273105621338,
      "step": 5010,
      "unclipped_grad_norm": 0.6908667087554932
    },
    {
      "bits_per_byte": 1.598975331762696,
      "learning_rate": 0.00012011146251791832,
      "loss_nats": 1.1083252429962158,
      "step": 5020,
      "unclipped_grad_norm": 0.730786919593811
    },
    {
      "bits_per_byte": 1.5824023955949522,
      "learning_rate": 0.0001195855539803584,
      "loss_nats": 1.0968377590179443,
      "step": 5030,
      "unclipped_grad_norm": 0.7392095923423767
    },
    {
      "bits_per_byte": 1.5712477728426413,
      "learning_rate": 0.00011906042145037817,
      "loss_nats": 1.0891059637069702,
      "step": 5040,
      "unclipped_grad_norm": 0.7138375639915466
    },
    {
      "bits_per_byte": 1.600523519585763,
      "learning_rate": 0.0001185360739010438,
      "loss_nats": 1.109398365020752,
      "step": 5050,
      "unclipped_grad_norm": 0.7417458295822144
    },
    {
      "bits_per_byte": 1.556087158203783,
      "learning_rate": 0.00011801252029200808,
      "loss_nats": 1.0785974264144897,
      "step": 5060,
      "unclipped_grad_norm": 0.766182541847229
    },
    {
      "bits_per_byte": 1.6239489324944845,
      "learning_rate": 0.00011748976956935777,
      "loss_nats": 1.1256356239318848,
      "step": 5070,
      "unclipped_grad_norm": 0.7354997992515564
    },
    {
      "bits_per_byte": 1.6499730032577429,
      "learning_rate": 0.00011696783066546024,
      "loss_nats": 1.1436741352081299,
      "step": 5080,
      "unclipped_grad_norm": 0.8061900734901428
    },
    {
      "bits_per_byte": 1.621231606610852,
      "learning_rate": 0.00011644671249881147,
      "loss_nats": 1.1237521171569824,
      "step": 5090,
      "unclipped_grad_norm": 0.789067804813385
    },
    {
      "bits_per_byte": 1.5443199332490989,
      "learning_rate": 0.00011592642397388293,
      "loss_nats": 1.0704410076141357,
      "step": 5100,
      "unclipped_grad_norm": 0.7130411267280579
    },
    {
      "bits_per_byte": 1.5718306220464153,
      "learning_rate": 0.00011540697398097015,
      "loss_nats": 1.0895099639892578,
      "step": 5110,
      "unclipped_grad_norm": 0.6817747950553894
    },
    {
      "bits_per_byte": 1.5553593276253317,
      "learning_rate": 0.00011488837139604016,
      "loss_nats": 1.0780929327011108,
      "step": 5120,
      "unclipped_grad_norm": 0.7432519793510437
    },
    {
      "bits_per_byte": 1.611734380664906,
      "learning_rate": 0.00011437062508058039,
      "loss_nats": 1.1171691417694092,
      "step": 5130,
      "unclipped_grad_norm": 0.7314459681510925
    },
    {
      "bits_per_byte": 1.5509013653323167,
      "learning_rate": 0.00011385374388144673,
      "loss_nats": 1.075002908706665,
      "step": 5140,
      "unclipped_grad_norm": 0.7262561321258545
    },
    {
      "bits_per_byte": 1.5519597465657264,
      "learning_rate": 0.00011333773663071286,
      "loss_nats": 1.0757365226745605,
      "step": 5150,
      "unclipped_grad_norm": 0.7442956566810608
    },
    {
      "bits_per_byte": 1.582139434121783,
      "learning_rate": 0.00011282261214551897,
      "loss_nats": 1.0966554880142212,
      "step": 5160,
      "unclipped_grad_norm": 0.7529576420783997
    },
    {
      "bits_per_byte": 1.5941560339202183,
      "learning_rate": 0.00011230837922792112,
      "loss_nats": 1.1049847602844238,
      "step": 5170,
      "unclipped_grad_norm": 0.7270960211753845
    },
    {
      "bits_per_byte": 1.6180967068509247,
      "learning_rate": 0.0001117950466647412,
      "loss_nats": 1.1215791702270508,
      "step": 5180,
      "unclipped_grad_norm": 0.759490966796875
    },
    {
      "bits_per_byte": 1.5759898504748815,
      "learning_rate": 0.00011128262322741627,
      "loss_nats": 1.092392921447754,
      "step": 5190,
      "unclipped_grad_norm": 0.7248204946517944
    },
    {
      "bits_per_byte": 1.5225916451390915,
      "learning_rate": 0.00011077111767184913,
      "loss_nats": 1.05538010597229,
      "step": 5200,
      "unclipped_grad_norm": 0.7128092646598816
    },
    {
      "bits_per_byte": 1.601221769148266,
      "learning_rate": 0.00011026053873825837,
      "loss_nats": 1.1098823547363281,
      "step": 5210,
      "unclipped_grad_norm": 0.7404454946517944
    },
    {
      "bits_per_byte": 1.5843179383602621,
      "learning_rate": 0.0001097508951510293,
      "loss_nats": 1.098165512084961,
      "step": 5220,
      "unclipped_grad_norm": 0.7370045781135559
    },
    {
      "bits_per_byte": 1.614152456736037,
      "learning_rate": 0.0001092421956185646,
      "loss_nats": 1.1188452243804932,
      "step": 5230,
      "unclipped_grad_norm": 0.8054904937744141
    },
    {
      "bits_per_byte": 1.5701584347320763,
      "learning_rate": 0.00010873444883313581,
      "loss_nats": 1.0883508920669556,
      "step": 5240,
      "unclipped_grad_norm": 0.7422505617141724
    },
    {
      "bits_per_byte": 1.5364362485335477,
      "learning_rate": 0.00010822766347073447,
      "loss_nats": 1.064976453781128,
      "step": 5250,
      "unclipped_grad_norm": 0.7392460107803345
    },
    {
      "bits_per_byte": 1.510300904995181,
      "learning_rate": 0.00010772184819092424,
      "loss_nats": 1.0468608140945435,
      "step": 5260,
      "unclipped_grad_norm": 0.7152841091156006
    },
    {
      "bits_per_byte": 1.6197508359869233,
      "learning_rate": 0.0001072170116366926,
      "loss_nats": 1.1227257251739502,
      "step": 5270,
      "unclipped_grad_norm": 0.762697160243988
    },
    {
      "bits_per_byte": 1.6105141637570344,
      "learning_rate": 0.00010671316243430345,
      "loss_nats": 1.1163233518600464,
      "step": 5280,
      "unclipped_grad_norm": 0.7536966800689697
    },
    {
      "bits_per_byte": 1.5885342650288148,
      "learning_rate": 0.00010621030919314944,
      "loss_nats": 1.101088047027588,
      "step": 5290,
      "unclipped_grad_norm": 0.7395375967025757
    },
    {
      "bits_per_byte": 1.5242677880544004,
      "learning_rate": 0.000105708460505605,
      "loss_nats": 1.056541919708252,
      "step": 5300,
      "unclipped_grad_norm": 0.6971617937088013
    },
    {
      "bits_per_byte": 1.589467786857698,
      "learning_rate": 0.00010520762494687964,
      "loss_nats": 1.1017351150512695,
      "step": 5310,
      "unclipped_grad_norm": 0.7348577380180359
    },
    {
      "bits_per_byte": 1.5472704676073012,
      "learning_rate": 0.00010470781107487111,
      "loss_nats": 1.072486162185669,
      "step": 5320,
      "unclipped_grad_norm": 0.7314221858978271
    },
    {
      "bits_per_byte": 1.6553672391020549,
      "learning_rate": 0.00010420902743001955,
      "loss_nats": 1.1474131345748901,
      "step": 5330,
      "unclipped_grad_norm": 0.7674097418785095
    },
    {
      "bits_per_byte": 1.5389176142201964,
      "learning_rate": 0.00010371128253516117,
      "loss_nats": 1.0666964054107666,
      "step": 5340,
      "unclipped_grad_norm": 0.7431036829948425
    },
    {
      "bits_per_byte": 1.593487021408362,
      "learning_rate": 0.000103214584895383,
      "loss_nats": 1.1045210361480713,
      "step": 5350,
      "unclipped_grad_norm": 0.7701666951179504
    },
    {
      "bits_per_byte": 1.5047200679746828,
      "learning_rate": 0.00010271894299787715,
      "loss_nats": 1.0429924726486206,
      "step": 5360,
      "unclipped_grad_norm": 0.7093538045883179
    },
    {
      "bits_per_byte": 1.5323434054083145,
      "learning_rate": 0.00010222436531179622,
      "loss_nats": 1.0621395111083984,
      "step": 5370,
      "unclipped_grad_norm": 0.8088909983634949
    },
    {
      "bits_per_byte": 1.5949007187984543,
      "learning_rate": 0.0001017308602881082,
      "loss_nats": 1.1055009365081787,
      "step": 5380,
      "unclipped_grad_norm": 0.7591733336448669
    },
    {
      "bits_per_byte": 1.5063042001717801,
      "learning_rate": 0.00010123843635945242,
      "loss_nats": 1.0440905094146729,
      "step": 5390,
      "unclipped_grad_norm": 0.7258779406547546
    },
    {
      "bits_per_byte": 1.5763400071520381,
      "learning_rate": 0.00010074710193999505,
      "loss_nats": 1.0926356315612793,
      "step": 5400,
      "unclipped_grad_norm": 0.7020314335823059
    },
    {
      "bits_per_byte": 1.540318240928823,
      "learning_rate": 0.0001002568654252858,
      "loss_nats": 1.0676672458648682,
      "step": 5410,
      "unclipped_grad_norm": 0.73995441198349
    },
    {
      "bits_per_byte": 1.5024278832039832,
      "learning_rate": 9.976773519211404e-05,
      "loss_nats": 1.0414036512374878,
      "step": 5420,
      "unclipped_grad_norm": 0.7208534479141235
    },
    {
      "bits_per_byte": 1.556975964543442,
      "learning_rate": 9.92797195983659e-05,
      "loss_nats": 1.0792135000228882,
      "step": 5430,
      "unclipped_grad_norm": 0.7300911545753479
    },
    {
      "bits_per_byte": 1.5377587951186826,
      "learning_rate": 9.879282698288142e-05,
      "loss_nats": 1.0658931732177734,
      "step": 5440,
      "unclipped_grad_norm": 0.7053164839744568
    },
    {
      "bits_per_byte": 1.5519281017579678,
      "learning_rate": 9.830706566531197e-05,
      "loss_nats": 1.0757145881652832,
      "step": 5450,
      "unclipped_grad_norm": 0.7570145726203918
    },
    {
      "bits_per_byte": 1.5295854916190785,
      "learning_rate": 9.782244394597826e-05,
      "loss_nats": 1.060227870941162,
      "step": 5460,
      "unclipped_grad_norm": 0.7293761968612671
    },
    {
      "bits_per_byte": 1.5453655877663446,
      "learning_rate": 9.733897010572829e-05,
      "loss_nats": 1.0711658000946045,
      "step": 5470,
      "unclipped_grad_norm": 0.7067918181419373
    },
    {
      "bits_per_byte": 1.5185650153344405,
      "learning_rate": 9.685665240579612e-05,
      "loss_nats": 1.0525890588760376,
      "step": 5480,
      "unclipped_grad_norm": 0.7616989612579346
    },
    {
      "bits_per_byte": 1.5811864782533522,
      "learning_rate": 9.637549908766041e-05,
      "loss_nats": 1.0959949493408203,
      "step": 5490,
      "unclipped_grad_norm": 0.8041682243347168
    },
    {
      "bits_per_byte": 1.5613092394145813,
      "learning_rate": 9.589551837290389e-05,
      "loss_nats": 1.0822170972824097,
      "step": 5500,
      "unclipped_grad_norm": 0.7645291686058044
    },
    {
      "bits_per_byte": 1.5218208188979245,
      "learning_rate": 9.541671846307257e-05,
      "loss_nats": 1.0548458099365234,
      "step": 5510,
      "unclipped_grad_norm": 0.7549375891685486
    },
    {
      "bits_per_byte": 1.4947696677437132,
      "learning_rate": 9.493910753953597e-05,
      "loss_nats": 1.036095380783081,
      "step": 5520,
      "unclipped_grad_norm": 0.7179425358772278
    },
    {
      "bits_per_byte": 1.5059726176209167,
      "learning_rate": 9.446269376334689e-05,
      "loss_nats": 1.043860673904419,
      "step": 5530,
      "unclipped_grad_norm": 0.7216454744338989
    },
    {
      "bits_per_byte": 1.625827155024557,
      "learning_rate": 9.39874852751024e-05,
      "loss_nats": 1.1269375085830688,
      "step": 5540,
      "unclipped_grad_norm": 0.7688449621200562
    },
    {
      "bits_per_byte": 1.5739950237075238,
      "learning_rate": 9.351349019480433e-05,
      "loss_nats": 1.0910102128982544,
      "step": 5550,
      "unclipped_grad_norm": 0.7435398697853088
    },
    {
      "bits_per_byte": 1.5977501273579493,
      "learning_rate": 9.304071662172086e-05,
      "loss_nats": 1.107475996017456,
      "step": 5560,
      "unclipped_grad_norm": 0.8213235139846802
    },
    {
      "bits_per_byte": 1.537488782356828,
      "learning_rate": 9.256917263424787e-05,
      "loss_nats": 1.0657060146331787,
      "step": 5570,
      "unclipped_grad_norm": 0.7458736896514893
    },
    {
      "bits_per_byte": 1.569628556184768,
      "learning_rate": 9.209886628977114e-05,
      "loss_nats": 1.0879836082458496,
      "step": 5580,
      "unclipped_grad_norm": 0.745820939540863
    },
    {
      "bits_per_byte": 1.5662062734152489,
      "learning_rate": 9.162980562452843e-05,
      "loss_nats": 1.0856114625930786,
      "step": 5590,
      "unclipped_grad_norm": 0.8023808002471924
    },
    {
      "bits_per_byte": 1.5614218880508965,
      "learning_rate": 9.116199865347238e-05,
      "loss_nats": 1.0822951793670654,
      "step": 5600,
      "unclipped_grad_norm": 0.7724120020866394
    },
    {
      "bits_per_byte": 1.5000096351502012,
      "learning_rate": 9.069545337013333e-05,
      "loss_nats": 1.0397274494171143,
      "step": 5610,
      "unclipped_grad_norm": 0.7330023050308228
    },
    {
      "bits_per_byte": 1.564252034553495,
      "learning_rate": 9.0230177746483e-05,
      "loss_nats": 1.084256887435913,
      "step": 5620,
      "unclipped_grad_norm": 0.7773064374923706
    },
    {
      "bits_per_byte": 1.4778873347870674,
      "learning_rate": 8.976617973279797e-05,
      "loss_nats": 1.0243934392929077,
      "step": 5630,
      "unclipped_grad_norm": 0.7191233038902283
    },
    {
      "bits_per_byte": 1.4838417181252395,
      "learning_rate": 8.930346725752419e-05,
      "loss_nats": 1.0285207033157349,
      "step": 5640,
      "unclipped_grad_norm": 0.7019065022468567
    },
    {
      "bits_per_byte": 1.5419510442161144,
      "learning_rate": 8.88420482271411e-05,
      "loss_nats": 1.0687990188598633,
      "step": 5650,
      "unclipped_grad_norm": 0.7909054160118103
    },
    {
      "bits_per_byte": 1.4908660055344292,
      "learning_rate": 8.838193052602691e-05,
      "loss_nats": 1.0333895683288574,
      "step": 5660,
      "unclipped_grad_norm": 0.7980607151985168
    },
    {
      "bits_per_byte": 1.5729701790910324,
      "learning_rate": 8.792312201632352e-05,
      "loss_nats": 1.0902998447418213,
      "step": 5670,
      "unclipped_grad_norm": 0.796278178691864
    },
    {
      "bits_per_byte": 1.5668854329035062,
      "learning_rate": 8.746563053780261e-05,
      "loss_nats": 1.0860822200775146,
      "step": 5680,
      "unclipped_grad_norm": 0.8280957341194153
    },
    {
      "bits_per_byte": 1.592162067066115,
      "learning_rate": 8.700946390773114e-05,
      "loss_nats": 1.103602647781372,
      "step": 5690,
      "unclipped_grad_norm": 0.7912419438362122
    },
    {
      "bits_per_byte": 1.5918954939572776,
      "learning_rate": 8.655462992073833e-05,
      "loss_nats": 1.1034178733825684,
      "step": 5700,
      "unclipped_grad_norm": 0.7910536527633667
    },
    {
      "bits_per_byte": 1.5647875885282816,
      "learning_rate": 8.610113634868199e-05,
      "loss_nats": 1.0846281051635742,
      "step": 5710,
      "unclipped_grad_norm": 0.8025376200675964
    },
    {
      "bits_per_byte": 1.576262442976499,
      "learning_rate": 8.564899094051613e-05,
      "loss_nats": 1.092581868171692,
      "step": 5720,
      "unclipped_grad_norm": 0.7657404541969299
    },
    {
      "bits_per_byte": 1.5180726290050204,
      "learning_rate": 8.519820142215816e-05,
      "loss_nats": 1.0522477626800537,
      "step": 5730,
      "unclipped_grad_norm": 0.7721706032752991
    },
    {
      "bits_per_byte": 1.6017487239905095,
      "learning_rate": 8.474877549635728e-05,
      "loss_nats": 1.1102476119995117,
      "step": 5740,
      "unclipped_grad_norm": 0.7985641956329346
    },
    {
      "bits_per_byte": 1.4880221004197618,
      "learning_rate": 8.430072084256247e-05,
      "loss_nats": 1.0314183235168457,
      "step": 5750,
      "unclipped_grad_norm": 0.7275866866111755
    },
    {
      "bits_per_byte": 1.5012069783655082,
      "learning_rate": 8.385404511679161e-05,
      "loss_nats": 1.0405573844909668,
      "step": 5760,
      "unclipped_grad_norm": 0.7711101174354553
    },
    {
      "bits_per_byte": 1.5255566260399664,
      "learning_rate": 8.340875595150034e-05,
      "loss_nats": 1.0574352741241455,
      "step": 5770,
      "unclipped_grad_norm": 0.8030062913894653
    },
    {
      "bits_per_byte": 1.6161017081009161,
      "learning_rate": 8.296486095545203e-05,
      "loss_nats": 1.1201963424682617,
      "step": 5780,
      "unclipped_grad_norm": 0.7891528606414795
    },
    {
      "bits_per_byte": 1.436315000489906,
      "learning_rate": 8.252236771358729e-05,
      "loss_nats": 0.9955776929855347,
      "step": 5790,
      "unclipped_grad_norm": 0.7522260546684265
    },
    {
      "bits_per_byte": 1.5639025658069416,
      "learning_rate": 8.208128378689484e-05,
      "loss_nats": 1.084014654159546,
      "step": 5800,
      "unclipped_grad_norm": 0.77913498878479
    },
    {
      "bits_per_byte": 1.4787818165542048,
      "learning_rate": 8.16416167122819e-05,
      "loss_nats": 1.0250134468078613,
      "step": 5810,
      "unclipped_grad_norm": 0.744920551776886
    },
    {
      "bits_per_byte": 1.485352929678371,
      "learning_rate": 8.120337400244578e-05,
      "loss_nats": 1.0295681953430176,
      "step": 5820,
      "unclipped_grad_norm": 0.8005070090293884
    },
    {
      "bits_per_byte": 1.551830071646976,
      "learning_rate": 8.076656314574521e-05,
      "loss_nats": 1.0756466388702393,
      "step": 5830,
      "unclipped_grad_norm": 0.7964350581169128
    },
    {
      "bits_per_byte": 1.5252578921754176,
      "learning_rate": 8.03311916060725e-05,
      "loss_nats": 1.0572282075881958,
      "step": 5840,
      "unclipped_grad_norm": 0.8099321722984314
    },
    {
      "bits_per_byte": 1.4836750669365535,
      "learning_rate": 7.989726682272606e-05,
      "loss_nats": 1.0284051895141602,
      "step": 5850,
      "unclipped_grad_norm": 0.7372169494628906
    },
    {
      "bits_per_byte": 1.511822779472666,
      "learning_rate": 7.946479621028318e-05,
      "loss_nats": 1.0479156970977783,
      "step": 5860,
      "unclipped_grad_norm": 0.7115837931632996
    },
    {
      "bits_per_byte": 1.5625698722454058,
      "learning_rate": 7.903378715847343e-05,
      "loss_nats": 1.083090901374817,
      "step": 5870,
      "unclipped_grad_norm": 0.7878124713897705
    },
    {
      "bits_per_byte": 1.503004884997628,
      "learning_rate": 7.860424703205224e-05,
      "loss_nats": 1.0418035984039307,
      "step": 5880,
      "unclipped_grad_norm": 0.7451943159103394
    },
    {
      "bits_per_byte": 1.5713764158654868,
      "learning_rate": 7.817618317067534e-05,
      "loss_nats": 1.0891951322555542,
      "step": 5890,
      "unclipped_grad_norm": 0.8092991709709167
    },
    {
      "bits_per_byte": 1.5881084359852786,
      "learning_rate": 7.774960288877295e-05,
      "loss_nats": 1.1007928848266602,
      "step": 5900,
      "unclipped_grad_norm": 0.7848427891731262
    },
    {
      "bits_per_byte": 1.5133455138634053,
      "learning_rate": 7.732451347542522e-05,
      "loss_nats": 1.048971176147461,
      "step": 5910,
      "unclipped_grad_norm": 0.7443398237228394
    },
    {
      "bits_per_byte": 1.4825475486774968,
      "learning_rate": 7.690092219423722e-05,
      "loss_nats": 1.0276236534118652,
      "step": 5920,
      "unclipped_grad_norm": 0.7709684371948242
    },
    {
      "bits_per_byte": 1.5970936695796059,
      "learning_rate": 7.647883628321536e-05,
      "loss_nats": 1.1070209741592407,
      "step": 5930,
      "unclipped_grad_norm": 0.8097158074378967
    },
    {
      "bits_per_byte": 1.5296348506398763,
      "learning_rate": 7.605826295464316e-05,
      "loss_nats": 1.0602620840072632,
      "step": 5940,
      "unclipped_grad_norm": 0.8011659383773804
    },
    {
      "bits_per_byte": 1.5315722352018457,
      "learning_rate": 7.563920939495851e-05,
      "loss_nats": 1.0616049766540527,
      "step": 5950,
      "unclipped_grad_norm": 0.7703563570976257
    },
    {
      "bits_per_byte": 1.4753576419755263,
      "learning_rate": 7.522168276463047e-05,
      "loss_nats": 1.0226399898529053,
      "step": 5960,
      "unclipped_grad_norm": 0.7793571949005127
    },
    {
      "bits_per_byte": 1.514912963743369,
      "learning_rate": 7.480569019803719e-05,
      "loss_nats": 1.0500576496124268,
      "step": 5970,
      "unclipped_grad_norm": 0.7870919108390808
    },
    {
      "bits_per_byte": 1.4934732625215095,
      "learning_rate": 7.439123880334388e-05,
      "loss_nats": 1.0351967811584473,
      "step": 5980,
      "unclipped_grad_norm": 0.8144509196281433
    },
    {
      "bits_per_byte": 1.5080035607449556,
      "learning_rate": 7.39783356623813e-05,
      "loss_nats": 1.0452684164047241,
      "step": 5990,
      "unclipped_grad_norm": 0.7810203433036804
    },
    {
      "bits_per_byte": 1.5782032671914858,
      "learning_rate": 7.356698783052497e-05,
      "loss_nats": 1.0939271450042725,
      "step": 6000,
      "unclipped_grad_norm": 0.790114164352417
    },
    {
      "bits_per_byte": 1.5490276143511665,
      "learning_rate": 7.315720233657431e-05,
      "loss_nats": 1.0737041234970093,
      "step": 6010,
      "unclipped_grad_norm": 0.8144342303276062
    },
    {
      "bits_per_byte": 1.515940732064925,
      "learning_rate": 7.274898618263286e-05,
      "loss_nats": 1.0507700443267822,
      "step": 6020,
      "unclipped_grad_norm": 0.7434252500534058
    },
    {
      "bits_per_byte": 1.493858503659442,
      "learning_rate": 7.234234634398831e-05,
      "loss_nats": 1.035463809967041,
      "step": 6030,
      "unclipped_grad_norm": 0.8062746524810791
    },
    {
      "bits_per_byte": 1.5237423810560147,
      "learning_rate": 7.193728976899362e-05,
      "loss_nats": 1.0561777353286743,
      "step": 6040,
      "unclipped_grad_norm": 0.7702685594558716
    },
    {
      "bits_per_byte": 1.5275683071071087,
      "learning_rate": 7.153382337894797e-05,
      "loss_nats": 1.058829665184021,
      "step": 6050,
      "unclipped_grad_norm": 0.7716134786605835
    },
    {
      "bits_per_byte": 1.5215145177967377,
      "learning_rate": 7.113195406797887e-05,
      "loss_nats": 1.0546334981918335,
      "step": 6060,
      "unclipped_grad_norm": 0.7933171987533569
    },
    {
      "bits_per_byte": 1.415498478403448,
      "learning_rate": 7.073168870292397e-05,
      "loss_nats": 0.9811487793922424,
      "step": 6070,
      "unclipped_grad_norm": 0.7639470100402832
    },
    {
      "bits_per_byte": 1.4966260484771265,
      "learning_rate": 7.033303412321404e-05,
      "loss_nats": 1.0373821258544922,
      "step": 6080,
      "unclipped_grad_norm": 0.7900221943855286
    },
    {
      "bits_per_byte": 1.4925770609278632,
      "learning_rate": 6.993599714075586e-05,
      "loss_nats": 1.0345755815505981,
      "step": 6090,
      "unclipped_grad_norm": 0.809319019317627
    },
    {
      "bits_per_byte": 1.5936792980120267,
      "learning_rate": 6.954058453981609e-05,
      "loss_nats": 1.104654312133789,
      "step": 6100,
      "unclipped_grad_norm": 0.824783205986023
    },
    {
      "bits_per_byte": 1.5281914002511847,
      "learning_rate": 6.914680307690504e-05,
      "loss_nats": 1.0592615604400635,
      "step": 6110,
      "unclipped_grad_norm": 0.8245336413383484
    },
    {
      "bits_per_byte": 1.48945093228313,
      "learning_rate": 6.875465948066139e-05,
      "loss_nats": 1.0324087142944336,
      "step": 6120,
      "unclipped_grad_norm": 0.8355178833007812
    },
    {
      "bits_per_byte": 1.5475705773330568,
      "learning_rate": 6.836416045173733e-05,
      "loss_nats": 1.072694182395935,
      "step": 6130,
      "unclipped_grad_norm": 0.7615814208984375
    },
    {
      "bits_per_byte": 1.4980500648262705,
      "learning_rate": 6.797531266268377e-05,
      "loss_nats": 1.0383691787719727,
      "step": 6140,
      "unclipped_grad_norm": 0.7743073105812073
    },
    {
      "bits_per_byte": 1.483834838819205,
      "learning_rate": 6.758812275783663e-05,
      "loss_nats": 1.0285159349441528,
      "step": 6150,
      "unclipped_grad_norm": 0.7909993529319763
    },
    {
      "bits_per_byte": 1.5470265962083778,
      "learning_rate": 6.720259735320298e-05,
      "loss_nats": 1.072317123413086,
      "step": 6160,
      "unclipped_grad_norm": 0.7707679867744446
    },
    {
      "bits_per_byte": 1.478142213075646,
      "learning_rate": 6.681874303634842e-05,
      "loss_nats": 1.024570107460022,
      "step": 6170,
      "unclipped_grad_norm": 0.7621827721595764
    },
    {
      "bits_per_byte": 1.4841139666615555,
      "learning_rate": 6.643656636628402e-05,
      "loss_nats": 1.0287094116210938,
      "step": 6180,
      "unclipped_grad_norm": 0.7684755921363831
    },
    {
      "bits_per_byte": 1.493858503659442,
      "learning_rate": 6.605607387335466e-05,
      "loss_nats": 1.035463809967041,
      "step": 6190,
      "unclipped_grad_norm": 0.8198496699333191
    },
    {
      "bits_per_byte": 1.4883674415826944,
      "learning_rate": 6.567727205912724e-05,
      "loss_nats": 1.0316576957702637,
      "step": 6200,
      "unclipped_grad_norm": 0.7631723284721375
    },
    {
      "bits_per_byte": 1.5053304344025948,
      "learning_rate": 6.530016739627959e-05,
      "loss_nats": 1.0434155464172363,
      "step": 6210,
      "unclipped_grad_norm": 0.7891849875450134
    },
    {
      "bits_per_byte": 1.5365181122753584,
      "learning_rate": 6.492476632848993e-05,
      "loss_nats": 1.065033197402954,
      "step": 6220,
      "unclipped_grad_norm": 0.7962750196456909
    },
    {
      "bits_per_byte": 1.4799394317771624,
      "learning_rate": 6.455107527032678e-05,
      "loss_nats": 1.0258158445358276,
      "step": 6230,
      "unclipped_grad_norm": 0.7991603016853333
    },
    {
      "bits_per_byte": 1.5356217386990614,
      "learning_rate": 6.417910060713926e-05,
      "loss_nats": 1.0644118785858154,
      "step": 6240,
      "unclipped_grad_norm": 0.8105412721633911
    },
    {
      "bits_per_byte": 1.4819163723488304,
      "learning_rate": 6.380884869494799e-05,
      "loss_nats": 1.0271861553192139,
      "step": 6250,
      "unclipped_grad_norm": 0.7951217293739319
    },
    {
      "bits_per_byte": 1.4447946050906966,
      "learning_rate": 6.34403258603367e-05,
      "loss_nats": 1.001455307006836,
      "step": 6260,
      "unclipped_grad_norm": 0.7625995874404907
    },
    {
      "bits_per_byte": 1.5022859975170215,
      "learning_rate": 6.30735384003437e-05,
      "loss_nats": 1.0413053035736084,
      "step": 6270,
      "unclipped_grad_norm": 0.8279880881309509
    },
    {
      "bits_per_byte": 1.5067209141348208,
      "learning_rate": 6.270849258235484e-05,
      "loss_nats": 1.0443793535232544,
      "step": 6280,
      "unclipped_grad_norm": 0.7734115719795227
    },
    {
      "bits_per_byte": 1.5122324421470212,
      "learning_rate": 6.234519464399581e-05,
      "loss_nats": 1.0481996536254883,
      "step": 6290,
      "unclipped_grad_norm": 0.7805989980697632
    },
    {
      "bits_per_byte": 1.476189006109797,
      "learning_rate": 6.198365079302614e-05,
      "loss_nats": 1.0232162475585938,
      "step": 6300,
      "unclipped_grad_norm": 0.8065475225448608
    },
    {
      "bits_per_byte": 1.5409463215697738,
      "learning_rate": 6.16238672072326e-05,
      "loss_nats": 1.0681025981903076,
      "step": 6310,
      "unclipped_grad_norm": 0.8388196229934692
    },
    {
      "bits_per_byte": 1.506346851869194,
      "learning_rate": 6.126585003432406e-05,
      "loss_nats": 1.0441200733184814,
      "step": 6320,
      "unclipped_grad_norm": 0.8094848990440369
    },
    {
      "bits_per_byte": 1.5104558613636083,
      "learning_rate": 6.090960539182614e-05,
      "loss_nats": 1.0469682216644287,
      "step": 6330,
      "unclipped_grad_norm": 0.7866588234901428
    },
    {
      "bits_per_byte": 1.427382135612766,
      "learning_rate": 6.0555139366976935e-05,
      "loss_nats": 0.9893859028816223,
      "step": 6340,
      "unclipped_grad_norm": 0.7612236738204956
    },
    {
      "bits_per_byte": 1.541960503261912,
      "learning_rate": 6.020245801662274e-05,
      "loss_nats": 1.0688055753707886,
      "step": 6350,
      "unclipped_grad_norm": 0.783340573310852
    },
    {
      "bits_per_byte": 1.5465876964833758,
      "learning_rate": 5.985156736711483e-05,
      "loss_nats": 1.0720129013061523,
      "step": 6360,
      "unclipped_grad_norm": 0.8162001371383667
    },
    {
      "bits_per_byte": 1.4662069610710688,
      "learning_rate": 5.950247341420618e-05,
      "loss_nats": 1.0162972211837769,
      "step": 6370,
      "unclipped_grad_norm": 0.7845266461372375
    },
    {
      "bits_per_byte": 1.4477447954835971,
      "learning_rate": 5.915518212294935e-05,
      "loss_nats": 1.00350022315979,
      "step": 6380,
      "unclipped_grad_norm": 0.7937688231468201
    },
    {
      "bits_per_byte": 1.5785561755910564,
      "learning_rate": 5.880969942759425e-05,
      "loss_nats": 1.0941717624664307,
      "step": 6390,
      "unclipped_grad_norm": 0.826732337474823
    },
    {
      "bits_per_byte": 1.5478830698096744,
      "learning_rate": 5.846603123148688e-05,
      "loss_nats": 1.0729107856750488,
      "step": 6400,
      "unclipped_grad_norm": 0.8436270952224731
    },
    {
      "bits_per_byte": 1.550731790438566,
      "learning_rate": 5.812418340696858e-05,
      "loss_nats": 1.074885368347168,
      "step": 6410,
      "unclipped_grad_norm": 0.8814867734909058
    },
    {
      "bits_per_byte": 1.4786806907554975,
      "learning_rate": 5.778416179527541e-05,
      "loss_nats": 1.0249433517456055,
      "step": 6420,
      "unclipped_grad_norm": 0.8324126601219177
    },
    {
      "bits_per_byte": 1.5252604719151805,
      "learning_rate": 5.7445972206438635e-05,
      "loss_nats": 1.057229995727539,
      "step": 6430,
      "unclipped_grad_norm": 0.7683992981910706
    },
    {
      "bits_per_byte": 1.4432244034883193,
      "learning_rate": 5.710962041918516e-05,
      "loss_nats": 1.0003669261932373,
      "step": 6440,
      "unclipped_grad_norm": 0.8131992816925049
    },
    {
      "bits_per_byte": 1.465075831176344,
      "learning_rate": 5.677511218083908e-05,
      "loss_nats": 1.0155131816864014,
      "step": 6450,
      "unclipped_grad_norm": 0.758244514465332
    },
    {
      "bits_per_byte": 1.4922360193312023,
      "learning_rate": 5.644245320722316e-05,
      "loss_nats": 1.034339189529419,
      "step": 6460,
      "unclipped_grad_norm": 0.798247754573822
    },
    {
      "bits_per_byte": 1.498638761440174,
      "learning_rate": 5.61116491825615e-05,
      "loss_nats": 1.038777232170105,
      "step": 6470,
      "unclipped_grad_norm": 0.8465205430984497
    },
    {
      "bits_per_byte": 1.456987831071568,
      "learning_rate": 5.5782705759382104e-05,
      "loss_nats": 1.0099070072174072,
      "step": 6480,
      "unclipped_grad_norm": 0.7985926270484924
    },
    {
      "bits_per_byte": 1.542977092711162,
      "learning_rate": 5.5455628558420546e-05,
      "loss_nats": 1.0695102214813232,
      "step": 6490,
      "unclipped_grad_norm": 0.8188008666038513
    },
    {
      "bits_per_byte": 1.582791420351204,
      "learning_rate": 5.513042316852368e-05,
      "loss_nats": 1.0971074104309082,
      "step": 6500,
      "unclipped_grad_norm": 0.8359130620956421
    },
    {
      "bits_per_byte": 1.5352007251697493,
      "learning_rate": 5.480709514655443e-05,
      "loss_nats": 1.0641200542449951,
      "step": 6510,
      "unclipped_grad_norm": 0.854469895362854
    },
    {
      "bits_per_byte": 1.5259941499037615,
      "learning_rate": 5.448565001729653e-05,
      "loss_nats": 1.0577385425567627,
      "step": 6520,
      "unclipped_grad_norm": 0.8313364386558533
    },
    {
      "bits_per_byte": 1.4871191915027322,
      "learning_rate": 5.41660932733604e-05,
      "loss_nats": 1.030792474746704,
      "step": 6530,
      "unclipped_grad_norm": 0.7717711329460144
    },
    {
      "bits_per_byte": 1.4761642406080728,
      "learning_rate": 5.384843037508902e-05,
      "loss_nats": 1.0231990814208984,
      "step": 6540,
      "unclipped_grad_norm": 0.8019536733627319
    },
    {
      "bits_per_byte": 1.5028273989019376,
      "learning_rate": 5.353266675046498e-05,
      "loss_nats": 1.0416805744171143,
      "step": 6550,
      "unclipped_grad_norm": 0.7671847343444824
    },
    {
      "bits_per_byte": 1.5359399066031576,
      "learning_rate": 5.321880779501729e-05,
      "loss_nats": 1.0646324157714844,
      "step": 6560,
      "unclipped_grad_norm": 0.8164054751396179
    },
    {
      "bits_per_byte": 1.504123116193538,
      "learning_rate": 5.290685887172962e-05,
      "loss_nats": 1.0425786972045898,
      "step": 6570,
      "unclipped_grad_norm": 0.7928623557090759
    },
    {
      "bits_per_byte": 1.4852913598893622,
      "learning_rate": 5.2596825310948295e-05,
      "loss_nats": 1.0295255184173584,
      "step": 6580,
      "unclipped_grad_norm": 0.8092948794364929
    },
    {
      "bits_per_byte": 1.5248497773449203,
      "learning_rate": 5.228871241029154e-05,
      "loss_nats": 1.0569453239440918,
      "step": 6590,
      "unclipped_grad_norm": 0.8197707533836365
    },
    {
      "bits_per_byte": 1.426588779644336,
      "learning_rate": 5.198252543455865e-05,
      "loss_nats": 0.9888359904289246,
      "step": 6600,
      "unclipped_grad_norm": 0.7768723964691162
    },
    {
      "bits_per_byte": 1.4662528804388493,
      "learning_rate": 5.167826961564027e-05,
      "loss_nats": 1.016329050064087,
      "step": 6610,
      "unclipped_grad_norm": 0.8190019726753235
    },
    {
      "bits_per_byte": 1.5755777800434143,
      "learning_rate": 5.137595015242884e-05,
      "loss_nats": 1.0921072959899902,
      "step": 6620,
      "unclipped_grad_norm": 0.8630855083465576
    },
    {
      "bits_per_byte": 1.5211641891369303,
      "learning_rate": 5.1075572210729925e-05,
      "loss_nats": 1.0543906688690186,
      "step": 6630,
      "unclipped_grad_norm": 0.8092710971832275
    },
    {
      "bits_per_byte": 1.4689426890983437,
      "learning_rate": 5.07771409231737e-05,
      "loss_nats": 1.0181934833526611,
      "step": 6640,
      "unclipped_grad_norm": 0.7870210409164429
    },
    {
      "bits_per_byte": 1.4737494322073081,
      "learning_rate": 5.048066138912756e-05,
      "loss_nats": 1.021525263786316,
      "step": 6650,
      "unclipped_grad_norm": 0.8340798020362854
    },
    {
      "bits_per_byte": 1.5123179175245,
      "learning_rate": 5.018613867460863e-05,
      "loss_nats": 1.048258900642395,
      "step": 6660,
      "unclipped_grad_norm": 0.8463515043258667
    },
    {
      "bits_per_byte": 1.4946623505695749,
      "learning_rate": 4.989357781219755e-05,
      "loss_nats": 1.0360209941864014,
      "step": 6670,
      "unclipped_grad_norm": 0.8076959252357483
    },
    {
      "bits_per_byte": 1.4999131528830674,
      "learning_rate": 4.960298380095222e-05,
      "loss_nats": 1.0396605730056763,
      "step": 6680,
      "unclipped_grad_norm": 0.8417643904685974
    },
    {
      "bits_per_byte": 1.4927383806543726,
      "learning_rate": 4.931436160632256e-05,
      "loss_nats": 1.0346873998641968,
      "step": 6690,
      "unclipped_grad_norm": 0.8966479301452637
    },
    {
      "bits_per_byte": 1.507631906236441,
      "learning_rate": 4.9027716160065485e-05,
      "loss_nats": 1.0450108051300049,
      "step": 6700,
      "unclipped_grad_norm": 0.8065080642700195
    },
    {
      "bits_per_byte": 1.526726795996437,
      "learning_rate": 4.874305236016084e-05,
      "loss_nats": 1.058246374130249,
      "step": 6710,
      "unclipped_grad_norm": 0.8376452326774597
    },
    {
      "bits_per_byte": 1.5822010039107919,
      "learning_rate": 4.846037507072752e-05,
      "loss_nats": 1.0966981649398804,
      "step": 6720,
      "unclipped_grad_norm": 0.8624657392501831
    },
    {
      "bits_per_byte": 1.447227471669802,
      "learning_rate": 4.817968912194049e-05,
      "loss_nats": 1.0031416416168213,
      "step": 6730,
      "unclipped_grad_norm": 0.8040223717689514
    },
    {
      "bits_per_byte": 1.4848161718250283,
      "learning_rate": 4.790099930994813e-05,
      "loss_nats": 1.0291961431503296,
      "step": 6740,
      "unclipped_grad_norm": 0.8131929636001587
    },
    {
      "bits_per_byte": 1.4599282184533695,
      "learning_rate": 4.762431039679047e-05,
      "loss_nats": 1.011945128440857,
      "step": 6750,
      "unclipped_grad_norm": 0.8242661356925964
    },
    {
      "bits_per_byte": 1.4561461479782454,
      "learning_rate": 4.734962711031755e-05,
      "loss_nats": 1.0093235969543457,
      "step": 6760,
      "unclipped_grad_norm": 0.858147919178009
    },
    {
      "bits_per_byte": 1.4679376224867013,
      "learning_rate": 4.707695414410892e-05,
      "loss_nats": 1.0174968242645264,
      "step": 6770,
      "unclipped_grad_norm": 0.8389740586280823
    },
    {
      "bits_per_byte": 1.446337461451587,
      "learning_rate": 4.680629615739322e-05,
      "loss_nats": 1.002524733543396,
      "step": 6780,
      "unclipped_grad_norm": 0.806380033493042
    },
    {
      "bits_per_byte": 1.4957231395600967,
      "learning_rate": 4.653765777496863e-05,
      "loss_nats": 1.0367562770843506,
      "step": 6790,
      "unclipped_grad_norm": 0.8443998694419861
    },
    {
      "bits_per_byte": 1.483685729860907,
      "learning_rate": 4.6271043587123986e-05,
      "loss_nats": 1.0284125804901123,
      "step": 6800,
      "unclipped_grad_norm": 0.8614793419837952
    },
    {
      "bits_per_byte": 1.4736859706091399,
      "learning_rate": 4.600645814956006e-05,
      "loss_nats": 1.0214812755584717,
      "step": 6810,
      "unclipped_grad_norm": 0.8073354959487915
    },
    {
      "bits_per_byte": 1.4900140034820548,
      "learning_rate": 4.574390598331204e-05,
      "loss_nats": 1.0327990055084229,
      "step": 6820,
      "unclipped_grad_norm": 0.8066654801368713
    },
    {
      "bits_per_byte": 1.4893948659389489,
      "learning_rate": 4.548339157467199e-05,
      "loss_nats": 1.03236985206604,
      "step": 6830,
      "unclipped_grad_norm": 0.8152909278869629
    },
    {
      "bits_per_byte": 1.4517927511369553,
      "learning_rate": 4.52249193751124e-05,
      "loss_nats": 1.0063060522079468,
      "step": 6840,
      "unclipped_grad_norm": 0.8145732283592224
    },
    {
      "bits_per_byte": 1.5401706798143826,
      "learning_rate": 4.4968493801209985e-05,
      "loss_nats": 1.0675649642944336,
      "step": 6850,
      "unclipped_grad_norm": 0.8944696187973022
    },
    {
      "bits_per_byte": 1.5020013662298435,
      "learning_rate": 4.471411923457034e-05,
      "loss_nats": 1.0411080121994019,
      "step": 6860,
      "unclipped_grad_norm": 0.8296581506729126
    },
    {
      "bits_per_byte": 1.4315734388056183,
      "learning_rate": 4.446180002175291e-05,
      "loss_nats": 0.9922910928726196,
      "step": 6870,
      "unclipped_grad_norm": 0.8298041224479675
    },
    {
      "bits_per_byte": 1.4506086505857647,
      "learning_rate": 4.421154047419693e-05,
      "loss_nats": 1.0054852962493896,
      "step": 6880,
      "unclipped_grad_norm": 0.8620857000350952
    },
    {
      "bits_per_byte": 1.4600797351687795,
      "learning_rate": 4.396334486814753e-05,
      "loss_nats": 1.0120501518249512,
      "step": 6890,
      "unclipped_grad_norm": 0.8483932614326477
    },
    {
      "bits_per_byte": 1.5166142161257037,
      "learning_rate": 4.371721744458285e-05,
      "loss_nats": 1.051236867904663,
      "step": 6900,
      "unclipped_grad_norm": 0.8512884378433228
    },
    {
      "bits_per_byte": 1.5060785589338481,
      "learning_rate": 4.347316240914145e-05,
      "loss_nats": 1.0439341068267822,
      "step": 6910,
      "unclipped_grad_norm": 0.8712192177772522
    },
    {
      "bits_per_byte": 1.4495721111490145,
      "learning_rate": 4.3231183932050546e-05,
      "loss_nats": 1.004766821861267,
      "step": 6920,
      "unclipped_grad_norm": 0.8044716715812683
    },
    {
      "bits_per_byte": 1.4331765750856358,
      "learning_rate": 4.299128614805464e-05,
      "loss_nats": 0.9934023022651672,
      "step": 6930,
      "unclipped_grad_norm": 0.8494104743003845
    },
    {
      "bits_per_byte": 1.5111775005666286,
      "learning_rate": 4.2753473156344965e-05,
      "loss_nats": 1.0474684238433838,
      "step": 6940,
      "unclipped_grad_norm": 0.8407061696052551
    },
    {
      "bits_per_byte": 1.4275953081085107,
      "learning_rate": 4.251774902048942e-05,
      "loss_nats": 0.9895336627960205,
      "step": 6950,
      "unclipped_grad_norm": 0.8335278630256653
    },
    {
      "bits_per_byte": 1.4337008641967912,
      "learning_rate": 4.228411776836306e-05,
      "loss_nats": 0.9937657117843628,
      "step": 6960,
      "unclipped_grad_norm": 0.8460164666175842
    },
    {
      "bits_per_byte": 1.404613954404317,
      "learning_rate": 4.2052583392079427e-05,
      "loss_nats": 0.9736042022705078,
      "step": 6970,
      "unclipped_grad_norm": 0.8476742506027222
    },
    {
      "bits_per_byte": 1.5285965913766173,
      "learning_rate": 4.182314984792214e-05,
      "loss_nats": 1.0595424175262451,
      "step": 6980,
      "unclipped_grad_norm": 0.9341830611228943
    },
    {
      "bits_per_byte": 1.4927865357966141,
      "learning_rate": 4.159582105627748e-05,
      "loss_nats": 1.034720778465271,
      "step": 6990,
      "unclipped_grad_norm": 0.8535965085029602
    },
    {
      "bits_per_byte": 1.5015640143486995,
      "learning_rate": 4.137060090156724e-05,
      "loss_nats": 1.0408048629760742,
      "step": 7000,
      "unclipped_grad_norm": 0.8839671015739441
    },
    {
      "bits_per_byte": 1.409006391307353,
      "learning_rate": 4.114749323218249e-05,
      "loss_nats": 0.9766488075256348,
      "step": 7010,
      "unclipped_grad_norm": 0.8374313712120056
    },
    {
      "bits_per_byte": 1.4918236049344333,
      "learning_rate": 4.0926501860417706e-05,
      "loss_nats": 1.0340533256530762,
      "step": 7020,
      "unclipped_grad_norm": 0.8843756318092346
    },
    {
      "bits_per_byte": 1.4765720114732686,
      "learning_rate": 4.070763056240571e-05,
      "loss_nats": 1.0234817266464233,
      "step": 7030,
      "unclipped_grad_norm": 0.8468295931816101
    },
    {
      "bits_per_byte": 1.4469567709773439,
      "learning_rate": 4.049088307805306e-05,
      "loss_nats": 1.0029540061950684,
      "step": 7040,
      "unclipped_grad_norm": 0.8328400254249573
    },
    {
      "bits_per_byte": 1.4604876780166263,
      "learning_rate": 4.027626311097629e-05,
      "loss_nats": 1.0123329162597656,
      "step": 7050,
      "unclipped_grad_norm": 0.8563967943191528
    },
    {
      "bits_per_byte": 1.4275743262251053,
      "learning_rate": 4.006377432843846e-05,
      "loss_nats": 0.9895191192626953,
      "step": 7060,
      "unclipped_grad_norm": 0.830858588218689
    },
    {
      "bits_per_byte": 1.4821155282585297,
      "learning_rate": 3.985342036128656e-05,
      "loss_nats": 1.0273241996765137,
      "step": 7070,
      "unclipped_grad_norm": 0.8345243334770203
    },
    {
      "bits_per_byte": 1.482162651504866,
      "learning_rate": 3.964520480388956e-05,
      "loss_nats": 1.0273568630218506,
      "step": 7080,
      "unclipped_grad_norm": 0.9049980044364929
    },
    {
      "bits_per_byte": 1.4400559671114737,
      "learning_rate": 3.943913121407678e-05,
      "loss_nats": 0.9981707334518433,
      "step": 7090,
      "unclipped_grad_norm": 0.86240553855896
    },
    {
      "bits_per_byte": 1.4348600272636065,
      "learning_rate": 3.923520311307735e-05,
      "loss_nats": 0.9945691823959351,
      "step": 7100,
      "unclipped_grad_norm": 0.8350730538368225
    },
    {
      "bits_per_byte": 1.4152483296377678,
      "learning_rate": 3.9033423985459816e-05,
      "loss_nats": 0.9809753894805908,
      "step": 7110,
      "unclipped_grad_norm": 0.8401304483413696
    },
    {
      "bits_per_byte": 1.4797989219514076,
      "learning_rate": 3.88337972790728e-05,
      "loss_nats": 1.0257184505462646,
      "step": 7120,
      "unclipped_grad_norm": 0.8903806805610657
    },
    {
      "bits_per_byte": 1.5240091261475028,
      "learning_rate": 3.8636326404985845e-05,
      "loss_nats": 1.0563626289367676,
      "step": 7130,
      "unclipped_grad_norm": 0.8470056056976318
    },
    {
      "bits_per_byte": 1.4628170110399121,
      "learning_rate": 3.844101473743143e-05,
      "loss_nats": 1.0139474868774414,
      "step": 7140,
      "unclipped_grad_norm": 0.8429069519042969
    },
    {
      "bits_per_byte": 1.5096671489267515,
      "learning_rate": 3.824786561374701e-05,
      "loss_nats": 1.0464215278625488,
      "step": 7150,
      "unclipped_grad_norm": 0.8920191526412964
    },
    {
      "bits_per_byte": 1.5009496923198173,
      "learning_rate": 3.805688233431824e-05,
      "loss_nats": 1.0403790473937988,
      "step": 7160,
      "unclipped_grad_norm": 0.8647960424423218
    },
    {
      "bits_per_byte": 1.5036095759980617,
      "learning_rate": 3.786806816252236e-05,
      "loss_nats": 1.0422227382659912,
      "step": 7170,
      "unclipped_grad_norm": 0.9017168283462524
    },
    {
      "bits_per_byte": 1.4616292988530535,
      "learning_rate": 3.768142632467269e-05,
      "loss_nats": 1.0131242275238037,
      "step": 7180,
      "unclipped_grad_norm": 0.8551055788993835
    },
    {
      "bits_per_byte": 1.5122007973392624,
      "learning_rate": 3.7496960009963236e-05,
      "loss_nats": 1.048177719116211,
      "step": 7190,
      "unclipped_grad_norm": 0.8699921369552612
    },
    {
      "bits_per_byte": 1.441016404225217,
      "learning_rate": 3.7314672370414325e-05,
      "loss_nats": 0.9988364577293396,
      "step": 7200,
      "unclipped_grad_norm": 0.8101796507835388
    },
    {
      "bits_per_byte": 1.4373777672809125,
      "learning_rate": 3.713456652081882e-05,
      "loss_nats": 0.9963143467903137,
      "step": 7210,
      "unclipped_grad_norm": 0.8248227834701538
    },
    {
      "bits_per_byte": 1.4388484769197658,
      "learning_rate": 3.695664553868873e-05,
      "loss_nats": 0.9973337650299072,
      "step": 7220,
      "unclipped_grad_norm": 0.8497165441513062
    },
    {
      "bits_per_byte": 1.4764554072359837,
      "learning_rate": 3.678091246420277e-05,
      "loss_nats": 1.023400902748108,
      "step": 7230,
      "unclipped_grad_norm": 0.869898796081543
    },
    {
      "bits_per_byte": 1.5094511387172678,
      "learning_rate": 3.660737030015427e-05,
      "loss_nats": 1.046271800994873,
      "step": 7240,
      "unclipped_grad_norm": 0.8670166730880737
    },
    {
      "bits_per_byte": 1.4896672864579155,
      "learning_rate": 3.6436022011900056e-05,
      "loss_nats": 1.0325586795806885,
      "step": 7250,
      "unclipped_grad_norm": 0.8726136684417725
    },
    {
      "bits_per_byte": 1.4660583680607233,
      "learning_rate": 3.626687052730954e-05,
      "loss_nats": 1.016194224357605,
      "step": 7260,
      "unclipped_grad_norm": 0.8553854823112488
    },
    {
      "bits_per_byte": 1.4854459722924878,
      "learning_rate": 3.6099918736714955e-05,
      "loss_nats": 1.0296326875686646,
      "step": 7270,
      "unclipped_grad_norm": 0.8842402100563049
    },
    {
      "bits_per_byte": 1.4706281190767991,
      "learning_rate": 3.5935169492861716e-05,
      "loss_nats": 1.0193617343902588,
      "step": 7280,
      "unclipped_grad_norm": 0.8827332854270935
    },
    {
      "bits_per_byte": 1.465586275684105,
      "learning_rate": 3.5772625610859886e-05,
      "loss_nats": 1.015866994857788,
      "step": 7290,
      "unclipped_grad_norm": 0.9209113121032715
    },
    {
      "bits_per_byte": 1.4604543133823589,
      "learning_rate": 3.5612289868135887e-05,
      "loss_nats": 1.0123097896575928,
      "step": 7300,
      "unclipped_grad_norm": 0.8443734645843506
    },
    {
      "bits_per_byte": 1.44415740936925,
      "learning_rate": 3.545416500438522e-05,
      "loss_nats": 1.0010136365890503,
      "step": 7310,
      "unclipped_grad_norm": 0.8828385472297668
    },
    {
      "bits_per_byte": 1.4200490533539523,
      "learning_rate": 3.52982537215255e-05,
      "loss_nats": 0.9843029975891113,
      "step": 7320,
      "unclipped_grad_norm": 0.878987193107605
    },
    {
      "bits_per_byte": 1.4725175204791778,
      "learning_rate": 3.5144558683650396e-05,
      "loss_nats": 1.0206713676452637,
      "step": 7330,
      "unclipped_grad_norm": 0.9326861500740051
    },
    {
      "bits_per_byte": 1.4404713912046327,
      "learning_rate": 3.499308251698404e-05,
      "loss_nats": 0.9984586834907532,
      "step": 7340,
      "unclipped_grad_norm": 0.8777600526809692
    },
    {
      "bits_per_byte": 1.448391966198794,
      "learning_rate": 3.484382780983616e-05,
      "loss_nats": 1.0039488077163696,
      "step": 7350,
      "unclipped_grad_norm": 0.8631419539451599
    },
    {
      "bits_per_byte": 1.453881480431684,
      "learning_rate": 3.469679711255795e-05,
      "loss_nats": 1.007753849029541,
      "step": 7360,
      "unclipped_grad_norm": 0.855156421661377
    },
    {
      "bits_per_byte": 1.4104983408035878,
      "learning_rate": 3.4551992937498284e-05,
      "loss_nats": 0.9776829481124878,
      "step": 7370,
      "unclipped_grad_norm": 0.8616177439689636
    },
    {
      "bits_per_byte": 1.406926175153842,
      "learning_rate": 3.440941775896104e-05,
      "loss_nats": 0.9752069115638733,
      "step": 7380,
      "unclipped_grad_norm": 0.8499169945716858
    },
    {
      "bits_per_byte": 1.4229885808224993,
      "learning_rate": 3.4269074013162596e-05,
      "loss_nats": 0.9863405227661133,
      "step": 7390,
      "unclipped_grad_norm": 0.8316128849983215
    },
    {
      "bits_per_byte": 1.4424061960268393,
      "learning_rate": 3.41309640981904e-05,
      "loss_nats": 0.9997997879981995,
      "step": 7400,
      "unclipped_grad_norm": 0.8269481062889099
    },
    {
      "bits_per_byte": 1.4779136481326494,
      "learning_rate": 3.3995090373961795e-05,
      "loss_nats": 1.024411678314209,
      "step": 7410,
      "unclipped_grad_norm": 0.882456362247467
    },
    {
      "bits_per_byte": 1.464608382331299,
      "learning_rate": 3.3861455162183876e-05,
      "loss_nats": 1.0151891708374023,
      "step": 7420,
      "unclipped_grad_norm": 0.8758774995803833
    },
    {
      "bits_per_byte": 1.3991048341492285,
      "learning_rate": 3.3730060746313664e-05,
      "loss_nats": 0.9697855710983276,
      "step": 7430,
      "unclipped_grad_norm": 0.8292478919029236
    },
    {
      "bits_per_byte": 1.401245502204518,
      "learning_rate": 3.3600909371519225e-05,
      "loss_nats": 0.9712693691253662,
      "step": 7440,
      "unclipped_grad_norm": 0.875770092010498
    },
    {
      "bits_per_byte": 1.5201309173705464,
      "learning_rate": 3.3474003244641194e-05,
      "loss_nats": 1.0536744594573975,
      "step": 7450,
      "unclipped_grad_norm": 0.9047821760177612
    },
    {
      "bits_per_byte": 1.4276059710328641,
      "learning_rate": 3.334934453415514e-05,
      "loss_nats": 0.9895410537719727,
      "step": 7460,
      "unclipped_grad_norm": 0.8659260272979736
    },
    {
      "bits_per_byte": 1.4078822267099882,
      "learning_rate": 3.3226935370134486e-05,
      "loss_nats": 0.9758695960044861,
      "step": 7470,
      "unclipped_grad_norm": 0.8766598105430603
    },
    {
      "bits_per_byte": 1.486473568631393,
      "learning_rate": 3.31067778442141e-05,
      "loss_nats": 1.0303449630737305,
      "step": 7480,
      "unclipped_grad_norm": 0.8612073659896851
    },
    {
      "bits_per_byte": 1.450941608997835,
      "learning_rate": 3.298887400955457e-05,
      "loss_nats": 1.00571608543396,
      "step": 7490,
      "unclipped_grad_norm": 0.888404905796051
    },
    {
      "bits_per_byte": 1.4814055838757678,
      "learning_rate": 3.287322588080714e-05,
      "loss_nats": 1.026832103729248,
      "step": 7500,
      "unclipped_grad_norm": 0.9002490639686584
    },
    {
      "bits_per_byte": 1.4606080658722302,
      "learning_rate": 3.275983543407924e-05,
      "loss_nats": 1.0124163627624512,
      "step": 7510,
      "unclipped_grad_norm": 0.8679240345954895
    },
    {
      "bits_per_byte": 1.4798904167216664,
      "learning_rate": 3.2648704606900735e-05,
      "loss_nats": 1.0257818698883057,
      "step": 7520,
      "unclipped_grad_norm": 0.9162602424621582
    },
    {
      "bits_per_byte": 1.4597773896685629,
      "learning_rate": 3.253983529819086e-05,
      "loss_nats": 1.011840581893921,
      "step": 7530,
      "unclipped_grad_norm": 0.8853410482406616
    },
    {
      "bits_per_byte": 1.4621115382060728,
      "learning_rate": 3.243322936822571e-05,
      "loss_nats": 1.013458490371704,
      "step": 7540,
      "unclipped_grad_norm": 0.869069516658783
    },
    {
      "bits_per_byte": 1.4571945542179052,
      "learning_rate": 3.232888863860648e-05,
      "loss_nats": 1.0100502967834473,
      "step": 7550,
      "unclipped_grad_norm": 0.8861801028251648
    },
    {
      "bits_per_byte": 1.38000641874489,
      "learning_rate": 3.222681489222838e-05,
      "loss_nats": 0.9565475583076477,
      "step": 7560,
      "unclipped_grad_norm": 0.8688139915466309
    },
    {
      "bits_per_byte": 1.4141796294453064,
      "learning_rate": 3.212700987325009e-05,
      "loss_nats": 0.9802346229553223,
      "step": 7570,
      "unclipped_grad_norm": 0.8542686700820923
    },
    {
      "bits_per_byte": 1.4002238392670674,
      "learning_rate": 3.202947528706403e-05,
      "loss_nats": 0.9705612063407898,
      "step": 7580,
      "unclipped_grad_norm": 0.898608922958374
    },
    {
      "bits_per_byte": 1.4730534184192665,
      "learning_rate": 3.1934212800267116e-05,
      "loss_nats": 1.021042823791504,
      "step": 7590,
      "unclipped_grad_norm": 0.8726481199264526
    },
    {
      "bits_per_byte": 1.5103581752179183,
      "learning_rate": 3.1841224040632484e-05,
      "loss_nats": 1.0469005107879639,
      "step": 7600,
      "unclipped_grad_norm": 0.8932017087936401
    },
    {
      "bits_per_byte": 1.4571630813927972,
      "learning_rate": 3.175051059708143e-05,
      "loss_nats": 1.0100284814834595,
      "step": 7610,
      "unclipped_grad_norm": 0.8839172124862671
    },
    {
      "bits_per_byte": 1.4709910024701196,
      "learning_rate": 3.1662074019656414e-05,
      "loss_nats": 1.019613265991211,
      "step": 7620,
      "unclipped_grad_norm": 0.8883293271064758
    },
    {
      "bits_per_byte": 1.436151788954237,
      "learning_rate": 3.157591581949455e-05,
      "loss_nats": 0.995464563369751,
      "step": 7630,
      "unclipped_grad_norm": 0.8680411577224731
    },
    {
      "bits_per_byte": 1.4213451146108544,
      "learning_rate": 3.149203746880175e-05,
      "loss_nats": 0.985201358795166,
      "step": 7640,
      "unclipped_grad_norm": 0.9127815961837769
    },
    {
      "bits_per_byte": 1.3999687889958379,
      "learning_rate": 3.141044040082763e-05,
      "loss_nats": 0.970384418964386,
      "step": 7650,
      "unclipped_grad_norm": 0.8844171166419983
    },
    {
      "bits_per_byte": 1.4213168234647875,
      "learning_rate": 3.133112600984091e-05,
      "loss_nats": 0.9851817488670349,
      "step": 7660,
      "unclipped_grad_norm": 0.8884557485580444
    },
    {
      "bits_per_byte": 1.467116749294133,
      "learning_rate": 3.125409565110571e-05,
      "loss_nats": 1.0169278383255005,
      "step": 7670,
      "unclipped_grad_norm": 0.9028065204620361
    },
    {
      "bits_per_byte": 1.4497504571579594,
      "learning_rate": 3.117935064085835e-05,
      "loss_nats": 1.0048904418945312,
      "step": 7680,
      "unclipped_grad_norm": 0.8733177781105042
    },
    {
      "bits_per_byte": 1.436977993608982,
      "learning_rate": 3.110689225628479e-05,
      "loss_nats": 0.9960372447967529,
      "step": 7690,
      "unclipped_grad_norm": 0.8574365973472595
    },
    {
      "bits_per_byte": 1.4762694939904009,
      "learning_rate": 3.1036721735498926e-05,
      "loss_nats": 1.0232720375061035,
      "step": 7700,
      "unclipped_grad_norm": 0.8926270008087158
    },
    {
      "bits_per_byte": 1.4168368473924622,
      "learning_rate": 3.096884027752132e-05,
      "loss_nats": 0.9820764660835266,
      "step": 7710,
      "unclipped_grad_norm": 0.8520281314849854
    },
    {
      "bits_per_byte": 1.4226952644114528,
      "learning_rate": 3.090324904225885e-05,
      "loss_nats": 0.9861372113227844,
      "step": 7720,
      "unclipped_grad_norm": 0.8500117063522339
    },
    {
      "bits_per_byte": 1.4242657239878067,
      "learning_rate": 3.083994915048472e-05,
      "loss_nats": 0.9872257709503174,
      "step": 7730,
      "unclipped_grad_norm": 0.8397486209869385
    },
    {
      "bits_per_byte": 1.4715379072998633,
      "learning_rate": 3.077894168381942e-05,
      "loss_nats": 1.0199923515319824,
      "step": 7740,
      "unclipped_grad_norm": 0.9084712266921997
    },
    {
      "bits_per_byte": 1.4017008262726773,
      "learning_rate": 3.0720227684712254e-05,
      "loss_nats": 0.9715849757194519,
      "step": 7750,
      "unclipped_grad_norm": 0.8779424428939819
    },
    {
      "bits_per_byte": 1.4074879564828853,
      "learning_rate": 3.0663808156423456e-05,
      "loss_nats": 0.9755963087081909,
      "step": 7760,
      "unclipped_grad_norm": 0.8754757642745972
    },
    {
      "bits_per_byte": 1.4701193943955468,
      "learning_rate": 3.060968406300711e-05,
      "loss_nats": 1.0190091133117676,
      "step": 7770,
      "unclipped_grad_norm": 0.9014414548873901
    },
    {
      "bits_per_byte": 1.4337202982363386,
      "learning_rate": 3.055785632929463e-05,
      "loss_nats": 0.993779182434082,
      "step": 7780,
      "unclipped_grad_norm": 0.9682468175888062
    },
    {
      "bits_per_byte": 1.461853220264477,
      "learning_rate": 3.0508325840878984e-05,
      "loss_nats": 1.0132794380187988,
      "step": 7790,
      "unclipped_grad_norm": 0.8698559403419495
    },
    {
      "bits_per_byte": 1.446109240473892,
      "learning_rate": 3.046109344409957e-05,
      "loss_nats": 1.002366542816162,
      "step": 7800,
      "unclipped_grad_norm": 0.8869026303291321
    },
    {
      "bits_per_byte": 1.4204974121247516,
      "learning_rate": 3.0416159946027727e-05,
      "loss_nats": 0.9846137762069702,
      "step": 7810,
      "unclipped_grad_norm": 0.893868088722229
    },
    {
      "bits_per_byte": 1.471287500560207,
      "learning_rate": 3.0373526114452946e-05,
      "loss_nats": 1.0198187828063965,
      "step": 7820,
      "unclipped_grad_norm": 0.9250245690345764
    },
    {
      "bits_per_byte": 1.457534563918661,
      "learning_rate": 3.0333192677869803e-05,
      "loss_nats": 1.0102859735488892,
      "step": 7830,
      "unclipped_grad_norm": 0.902686595916748
    },
    {
      "bits_per_byte": 1.4368130622468045,
      "learning_rate": 3.029516032546544e-05,
      "loss_nats": 0.9959229230880737,
      "step": 7840,
      "unclipped_grad_norm": 0.880992591381073
    },
    {
      "bits_per_byte": 1.4160461571551204,
      "learning_rate": 3.02594297071078e-05,
      "loss_nats": 0.9815284013748169,
      "step": 7850,
      "unclipped_grad_norm": 0.8318398594856262
    },
    {
      "bits_per_byte": 1.463385069735712,
      "learning_rate": 3.0226001433334603e-05,
      "loss_nats": 1.0143412351608276,
      "step": 7860,
      "unclipped_grad_norm": 0.8814117908477783
    },
    {
      "bits_per_byte": 1.4569142224969986,
      "learning_rate": 3.019487607534278e-05,
      "loss_nats": 1.0098559856414795,
      "step": 7870,
      "unclipped_grad_norm": 0.8954723477363586
    },
    {
      "bits_per_byte": 1.4148711716844258,
      "learning_rate": 3.016605416497886e-05,
      "loss_nats": 0.980713963508606,
      "step": 7880,
      "unclipped_grad_norm": 0.9006734490394592
    },
    {
      "bits_per_byte": 1.44079876018055,
      "learning_rate": 3.0139536194729734e-05,
      "loss_nats": 0.9986855983734131,
      "step": 7890,
      "unclipped_grad_norm": 0.8986613154411316
    },
    {
      "bits_per_byte": 1.4040539788931077,
      "learning_rate": 3.0115322617714362e-05,
      "loss_nats": 0.9732160568237305,
      "step": 7900,
      "unclipped_grad_norm": 0.8486817479133606
    },
    {
      "bits_per_byte": 1.4565468675547557,
      "learning_rate": 3.0093413847675965e-05,
      "loss_nats": 1.009601354598999,
      "step": 7910,
      "unclipped_grad_norm": 0.960801362991333
    },
    {
      "bits_per_byte": 1.4869515084181408,
      "learning_rate": 3.0073810258974985e-05,
      "loss_nats": 1.030676245689392,
      "step": 7920,
      "unclipped_grad_norm": 0.9004883766174316
    },
    {
      "bits_per_byte": 1.426570807457321,
      "learning_rate": 3.0056512186582658e-05,
      "loss_nats": 0.9888235330581665,
      "step": 7930,
      "unclipped_grad_norm": 0.9118951559066772
    },
    {
      "bits_per_byte": 1.4575464307215704,
      "learning_rate": 3.0041519926075316e-05,
      "loss_nats": 1.0102941989898682,
      "step": 7940,
      "unclipped_grad_norm": 0.9135404825210571
    },
    {
      "bits_per_byte": 1.4061704833859507,
      "learning_rate": 3.0028833733629313e-05,
      "loss_nats": 0.9746831059455872,
      "step": 7950,
      "unclipped_grad_norm": 0.8805200457572937
    },
    {
      "bits_per_byte": 1.4792671515949398,
      "learning_rate": 3.0018453826016686e-05,
      "loss_nats": 1.0253498554229736,
      "step": 7960,
      "unclipped_grad_norm": 0.9637572765350342
    },
    {
      "bits_per_byte": 1.4296003678435947,
      "learning_rate": 3.001038038060143e-05,
      "loss_nats": 0.9909234642982483,
      "step": 7970,
      "unclipped_grad_norm": 0.8976891040802002
    },
    {
      "bits_per_byte": 1.4664819613297986,
      "learning_rate": 3.0004613535336433e-05,
      "loss_nats": 1.0164878368377686,
      "step": 7980,
      "unclipped_grad_norm": 0.9626050591468811
    },
    {
      "bits_per_byte": 1.4209173077668331,
      "learning_rate": 3.000115338876117e-05,
      "loss_nats": 0.9849048256874084,
      "step": 7990,
      "unclipped_grad_norm": 0.9410001635551453
    },
    {
      "bits_per_byte": 1.4482875727297202,
      "learning_rate": 3e-05,
      "loss_nats": 1.0038764476776123,
      "step": 8000,
      "unclipped_grad_norm": 0.9042860269546509
    }
  ]
}