llm-atlas/experiments/k3/attnres_gradient/results/raw/formal-depth-16-baseline-seed-2026073002.json

{
  "architecture": "baseline",
  "batch_size": 32,
  "canonical_sha256_without_self": "68d80bba3d8494c2df7d715607540a0248ed186d1f5b4bbcc53645c92dfa1a84",
  "depth": 16,
  "diagnostics": [
    {
      "activation_grad_rms_by_block": [
        0.00027575084823183715,
        0.00024360731185879558,
        0.00021477397240232676,
        0.00019141066877637058,
        0.00017414781905245036,
        0.00016276225505862385,
        0.00015418927068822086,
        0.00014831383305136114,
        0.0001412670244462788,
        0.00013702295836992562,
        0.0001326367782894522,
        0.00012906579650007188,
        0.00012653179874178022,
        0.00012431222421582788,
        0.000122369994642213,
        0.000120684111607261
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00023138570031733252,
        "first_to_last_ratio": 1.8739548634355478,
        "imbalance_abs_log_ratio": 0.6280510978460953,
        "last_quartile_mean": 0.00012347453230177052,
        "mean": 0.0001624279166207998,
        "normalized": [
          1.6976813713346965,
          1.4997872097782008,
          1.3222725309204868,
          1.1784345496669344,
          1.072154483511671,
          1.0020583803866954,
          0.9492781407040226,
          0.9131055556023103,
          0.8697213347633911,
          0.8435924145343563,
          0.8165885584748509,
          0.79460353358706,
          0.77900278089005,
          0.7653377990807093,
          0.7533803128676146,
          0.74300104389695
        ],
        "population_cv": 0.2778958958835334
      },
      "activation_output_rms_by_block": [
        0.028587408363819122,
        0.028957132250070572,
        0.029405318200588226,
        0.029973436146974564,
        0.030426617711782455,
        0.030960483476519585,
        0.03149084374308586,
        0.03222988545894623,
        0.03307616710662842,
        0.03391994163393974,
        0.034928515553474426,
        0.035913772881031036,
        0.03729887306690216,
        0.038532089442014694,
        0.03956005349755287,
        0.0403735376894474
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.02923082374036312,
        "first_to_last_ratio": 0.7506412221981493,
        "imbalance_abs_log_ratio": 0.2868274747935402,
        "last_quartile_mean": 0.03894113842397928,
        "mean": 0.033477129763923585,
        "normalized": [
          0.8539384518748725,
          0.8649825255113729,
          0.8783703503840009,
          0.8953406805883115,
          0.9088777301503161,
          0.9248249086719481,
          0.9406673739700877,
          0.962743391868635,
          0.9880227886881968,
          1.0132272949664087,
          1.0433545468140735,
          1.0727853054993168,
          1.114159825825243,
          1.1509974037122668,
          1.1817038610097486,
          1.206003560465201
        ],
        "population_cv": 0.11262664842931464
      },
      "bits_per_byte": 8.031398550585404,
      "branch_output_rms_by_sublayer": [
        0.0029695050325244665,
        0.003912033978849649,
        0.0030677192844450474,
        0.0038352999836206436,
        0.00318979867734015,
        0.0038548391312360764,
        0.003840027377009392,
        0.003801720915362239,
        0.00428530341014266,
        0.00395593885332346,
        0.004158117808401585,
        0.003868808038532734,
        0.004507514648139477,
        0.0038615495432168245,
        0.0055657969787716866,
        0.003801694605499506,
        0.005897593684494495,
        0.0038540444802492857,
        0.00619681179523468,
        0.003802293911576271,
        0.006646857596933842,
        0.0038319535087794065,
        0.00662674754858017,
        0.003906778059899807,
        0.007378070615231991,
        0.003958481829613447,
        0.007767242379486561,
        0.0038770362734794617,
        0.00770326005294919,
        0.0037339557893574238,
        0.007914260029792786,
        0.003793976968154311
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.007112265943118693,
        0.0061890396223896905,
        0.005352992918648162,
        0.005122911821075447,
        0.005146431820875539,
        0.004279647792521989,
        0.004250462252416264,
        0.004421502222176574,
        0.004575182382709001,
        0.004269268868925463,
        0.0040238830908244155,
        0.003973260546139901,
        0.0038964174441780434,
        0.004243476176619415,
        0.00401474940360252,
        0.003997243047861786
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.005944302576307998,
        "first_to_last_ratio": 1.472101165080992,
        "imbalance_abs_log_ratio": 0.38669074422298993,
        "last_quartile_mean": 0.004037971518065441,
        "mean": 0.004679295959630182,
        "normalized": [
          1.5199435993103534,
          1.3226433369003716,
          1.1439740005398642,
          1.0948039759127195,
          1.0998303730466061,
          0.9145922440990935,
          0.9083550792867974,
          0.944907580183498,
          0.9777501620287747,
          0.9123741917070097,
          0.8599334441633468,
          0.8491150336329483,
          0.8326930969517022,
          0.9068621034508766,
          0.857981507953136,
          0.854240270832901
        ],
        "population_cv": 0.1873316910212935
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.02822216972708702,
        0.02833663299679756,
        0.028587408363819122,
        0.02871040627360344,
        0.028957132250070572,
        0.029123734682798386,
        0.029405318200588226,
        0.0297223012894392,
        0.029973436146974564,
        0.03016654960811138,
        0.030426617711782455,
        0.030734822154045105,
        0.030960483476519585,
        0.03127134218811989,
        0.03149084374308586,
        0.0320059098303318,
        0.03222988545894623,
        0.03279566392302513,
        0.03307616710662842,
        0.033656880259513855,
        0.03391994163393974,
        0.03468930721282959,
        0.034928515553474426,
        0.03560047596693039,
        0.035913772881031036,
        0.03701025992631912,
        0.03729887306690216,
        0.038355886936187744,
        0.038532089442014694,
        0.0393846333026886,
        0.03956005349755287,
        0.04016231372952461
      ],
      "loss_nats": 5.566941261291504,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 0,
      "stream_state_rms_by_sublayer": [
        0.02833663299679756,
        0.028587408363819122,
        0.02871040627360344,
        0.028957132250070572,
        0.029123734682798386,
        0.029405318200588226,
        0.0297223012894392,
        0.029973436146974564,
        0.03016654960811138,
        0.030426617711782455,
        0.030734822154045105,
        0.030960483476519585,
        0.03127134218811989,
        0.03149084374308586,
        0.0320059098303318,
        0.03222988545894623,
        0.03279566392302513,
        0.03307616710662842,
        0.033656880259513855,
        0.03391994163393974,
        0.03468930721282959,
        0.034928515553474426,
        0.03560047596693039,
        0.035913772881031036,
        0.03701025992631912,
        0.03729887306690216,
        0.038355886936187744,
        0.038532089442014694,
        0.0393846333026886,
        0.03956005349755287,
        0.04016231372952461,
        0.0403735376894474
      ]
    },
    {
      "activation_grad_rms_by_block": [
        9.082402539206669e-05,
        8.28480115160346e-05,
        7.297925185412169e-05,
        6.382335413945839e-05,
        5.4197575082071126e-05,
        4.6132205170579255e-05,
        3.93990776501596e-05,
        3.41209233738482e-05,
        2.9922650355729274e-05,
        2.6855812393478118e-05,
        2.4530303562642075e-05,
        2.2887908926350065e-05,
        2.182140451623127e-05,
        2.1062218365841545e-05,
        2.0474079065024853e-05,
        2.0058148948010057e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 7.761866072542034e-05,
        "first_to_last_ratio": 3.722010140399952,
        "imbalance_abs_log_ratio": 1.3142638826655557,
        "last_quartile_mean": 2.085396272377693e-05,
        "mean": 4.1996059394477925e-05,
        "normalized": [
          2.1626797061823653,
          1.972756794579834,
          1.7377642785150873,
          1.5197462585704062,
          1.2905395378404856,
          1.098488901952672,
          0.938161299375155,
          0.8124791674700501,
          0.7125109066700628,
          0.6394841035254226,
          0.584109650198932,
          0.5450013467063437,
          0.5196060018693224,
          0.5015284450381319,
          0.4875238143823798,
          0.47761978712334874
        ],
        "population_cv": 0.55073751796069
      },
      "activation_output_rms_by_block": [
        0.03037329390645027,
        0.03339420258998871,
        0.03790459781885147,
        0.04353494197130203,
        0.04983566701412201,
        0.057033151388168335,
        0.06467092037200928,
        0.07435981929302216,
        0.08538646996021271,
        0.09887228161096573,
        0.11526436358690262,
        0.13244163990020752,
        0.15128590166568756,
        0.17302390933036804,
        0.19715538620948792,
        0.22326189279556274
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.03630175907164812,
        "first_to_last_ratio": 0.194980199104046,
        "imbalance_abs_log_ratio": 1.63485726863041,
        "last_quartile_mean": 0.18618177250027657,
        "mean": 0.09798740246333182,
        "normalized": [
          0.30997141615031953,
          0.34080097798781084,
          0.38683133613053855,
          0.4442912137363105,
          0.5085925921219432,
          0.5820457523558795,
          0.65999219028395,
          0.7588712163366979,
          0.8714025253620276,
          1.0090305398999126,
          1.1763181867182986,
          1.351619050722045,
          1.543932157220932,
          1.7657770793048206,
          2.012048296547774,
          2.2784754691207403
        ],
        "population_cv": 0.6071319453340859
      },
      "bits_per_byte": 5.218263195949167,
      "branch_output_rms_by_sublayer": [
        0.004278926644474268,
        0.007860537618398666,
        0.004119343124330044,
        0.008471532724797726,
        0.004720318131148815,
        0.009602016769349575,
        0.0057782139629125595,
        0.009444202296435833,
        0.005580307450145483,
        0.010230599902570248,
        0.00646675331518054,
        0.010376903228461742,
        0.00653326790779829,
        0.010733475908637047,
        0.008962317369878292,
        0.01155572384595871,
        0.010035770013928413,
        0.012197046540677547,
        0.011996372602880001,
        0.012902887538075447,
        0.015837425366044044,
        0.013319648802280426,
        0.01564962975680828,
        0.013529929332435131,
        0.01653609797358513,
        0.013358287513256073,
        0.01997774839401245,
        0.013715346343815327,
        0.020573746412992477,
        0.01530109066516161,
        0.024493996053934097,
        0.014329549856483936
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0003576967965622096,
        0.00031529422707157425,
        0.00030800051934586363,
        0.00027582768007438345,
        0.00025927860434436177,
        0.0002499710324331713,
        0.00022865411268768897,
        0.00021433881461782036,
        0.00020517591210118428,
        0.00017936399546700514,
        0.000174300235585937,
        0.00015047843844280973,
        0.00014083116496318749,
        0.00013997136590409212,
        0.0001362051425885804,
        0.00013592122810918616
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00031420480576350773,
        "first_to_last_ratio": 2.2730213947880955,
        "imbalance_abs_log_ratio": 0.8211099574033609,
        "last_quartile_mean": 0.00013823222539126155,
        "mean": 0.000216956829393691,
        "normalized": [
          1.6487003315905355,
          1.4532579036700417,
          1.4196396592197809,
          1.271348225567332,
          1.19507002876537,
          1.1521694575448123,
          1.0539152573656578,
          0.9879330151386018,
          0.9456992558131047,
          0.8267266624805357,
          0.8033867201739516,
          0.6935870092835418,
          0.6491206815510496,
          0.6451576854955755,
          0.6277983641686699,
          0.6264897421714382
        ],
        "population_cv": 0.3194559569879116
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.028607700020074844,
        0.028983348980545998,
        0.03037329390645027,
        0.030827771872282028,
        0.03339420258998871,
        0.03403528779745102,
        0.03790459781885147,
        0.03889948129653931,
        0.04353494197130203,
        0.04432978481054306,
        0.04983566701412201,
        0.050834495574235916,
        0.057033151388168335,
        0.0581708587706089,
        0.06467092037200928,
        0.06689786911010742,
        0.07435981929302216,
        0.07719690352678299,
        0.08538646996021271,
        0.0897849053144455,
        0.09887228161096573,
        0.10603746026754379,
        0.11526436358690262,
        0.12274596095085144,
        0.13244163990020752,
        0.14162033796310425,
        0.15128590166568756,
        0.16265419125556946,
        0.17302390933036804,
        0.18535073101520538,
        0.19715538620948792,
        0.21195416152477264
      ],
      "loss_nats": 3.6170244216918945,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 100,
      "stream_state_rms_by_sublayer": [
        0.028983348980545998,
        0.03037329390645027,
        0.030827771872282028,
        0.03339420258998871,
        0.03403528779745102,
        0.03790459781885147,
        0.03889948129653931,
        0.04353494197130203,
        0.04432978481054306,
        0.04983566701412201,
        0.050834495574235916,
        0.057033151388168335,
        0.0581708587706089,
        0.06467092037200928,
        0.06689786911010742,
        0.07435981929302216,
        0.07719690352678299,
        0.08538646996021271,
        0.0897849053144455,
        0.09887228161096573,
        0.10603746026754379,
        0.11526436358690262,
        0.12274596095085144,
        0.13244163990020752,
        0.14162033796310425,
        0.15128590166568756,
        0.16265419125556946,
        0.17302390933036804,
        0.18535073101520538,
        0.19715538620948792,
        0.21195416152477264,
        0.22326189279556274
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.00015646869724150747,
        0.0001555694470880553,
        0.00014404315152205527,
        0.00012938148574903607,
        0.00011473113409010693,
        0.00010209716856479645,
        9.211692668031901e-05,
        8.257795707322657e-05,
        7.45681973057799e-05,
        6.577855674549937e-05,
        5.7623117754701525e-05,
        5.006180435884744e-05,
        4.3452899262774736e-05,
        3.91966023016721e-05,
        3.658098648884334e-05,
        3.536837175488472e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00014636569540016353,
        "first_to_last_ratio": 3.786979944917394,
        "imbalance_abs_log_ratio": 1.3315688531352086,
        "last_quartile_mean": 3.864971495204372e-05,
        "mean": 8.622603149888164e-05,
        "normalized": [
          1.8146341020407148,
          1.8042051151347847,
          1.670529757871595,
          1.5004921773619393,
          1.330585811450629,
          1.1840643340534658,
          1.068319219602653,
          0.9576917276344513,
          0.8647991332727294,
          0.7628619293044065,
          0.6682798309632156,
          0.5805880601091649,
          0.5039417738173225,
          0.4545796857435157,
          0.42424527550381114,
          0.41018206613560143
        ],
        "population_cv": 0.48370287188714023
      },
      "activation_output_rms_by_block": [
        0.05259466543793678,
        0.067360520362854,
        0.0797014907002449,
        0.08837522566318512,
        0.09281065315008163,
        0.09652531147003174,
        0.09793278574943542,
        0.09766053408384323,
        0.09857666492462158,
        0.10091027617454529,
        0.10140916705131531,
        0.11094733327627182,
        0.12530675530433655,
        0.14505283534526825,
        0.17291401326656342,
        0.21681295335292816
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.0720079755410552,
        "first_to_last_ratio": 0.43635474619550435,
        "imbalance_abs_log_ratio": 0.829299728394033,
        "last_quartile_mean": 0.1650216393172741,
        "mean": 0.10905569908209145,
        "normalized": [
          0.4822734243200464,
          0.6176707951057974,
          0.7308328805471209,
          0.8103677882680927,
          0.8510390005406192,
          0.8851010289464333,
          0.8980070420319504,
          0.8955105960150644,
          0.9039111734125715,
          0.9253095163654427,
          0.9298841592403141,
          1.0173455785447438,
          1.1490161115744364,
          1.33008028526854,
          1.5855568734321968,
          1.98809374638663
        ],
        "population_cv": 0.3579222353559594
      },
      "bits_per_byte": 3.109639980064452,
      "branch_output_rms_by_sublayer": [
        0.015709737315773964,
        0.018998395651578903,
        0.02321256324648857,
        0.011548326350748539,
        0.022954270243644714,
        0.009733532555401325,
        0.020237641409039497,
        0.00895718764513731,
        0.01714940369129181,
        0.010070553049445152,
        0.016518093645572662,
        0.010933448560535908,
        0.01985226757824421,
        0.011251374147832394,
        0.022037845104932785,
        0.012155328877270222,
        0.027303192764520645,
        0.013005394488573074,
        0.03429069370031357,
        0.01562904566526413,
        0.035662416368722916,
        0.018316252157092094,
        0.019603708758950233,
        0.021821532398462296,
        0.018347281962633133,
        0.02647775411605835,
        0.01771368645131588,
        0.02924429066479206,
        0.014302654191851616,
        0.03788068890571594,
        0.024004697799682617,
        0.04663809388875961
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0010353898057018447,
        0.0006549276400495494,
        0.000583207798204784,
        0.0004740072073606198,
        0.00043399488807089015,
        0.0004525702167816539,
        0.0003859790612774441,
        0.0004183888521327983,
        0.00036657723102634496,
        0.00038828410964056754,
        0.0003961803157723344,
        0.00030130391836939404,
        0.00026972366751228246,
        0.0002437005745447571,
        0.00023517814034756533,
        0.0002654411672498285
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.0006868831128291994,
        "first_to_last_ratio": 2.7094817103793067,
        "imbalance_abs_log_ratio": 0.9967573658183029,
        "last_quartile_mean": 0.00025351088741360834,
        "mean": 0.0004315534121276662,
        "normalized": [
          2.3992158944987,
          1.5176050557000407,
          1.3514151013878533,
          1.0983743704484823,
          1.0056574131373146,
          1.0487003556532426,
          0.8943946460172122,
          0.9694949463382451,
          0.8494365256412354,
          0.8997359277643753,
          0.9180330977319039,
          0.6981845349892853,
          0.6250064532741785,
          0.5647054749103996,
          0.5449572028363264,
          0.6150829996712046
        ],
        "population_cv": 0.44646086778633726
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.04346943646669388,
        0.04716380685567856,
        0.05259466543793678,
        0.06381504237651825,
        0.067360520362854,
        0.076678566634655,
        0.0797014907002449,
        0.08585279434919357,
        0.08837522566318512,
        0.09026700258255005,
        0.09281065315008163,
        0.09405135363340378,
        0.09652531147003174,
        0.09571083635091782,
        0.09793278574943542,
        0.09508897364139557,
        0.09766053408384323,
        0.09508920460939407,
        0.09857666492462158,
        0.09573151916265488,
        0.10091027617454529,
        0.09348288178443909,
        0.10140916705131531,
        0.09920287877321243,
        0.11094733327627182,
        0.10943257063627243,
        0.12530675530433655,
        0.12546418607234955,
        0.14505283534526825,
        0.1458611935377121,
        0.17291401326656342,
        0.1839449107646942
      ],
      "loss_nats": 2.155438184738159,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 500,
      "stream_state_rms_by_sublayer": [
        0.04716380685567856,
        0.05259466543793678,
        0.06381504237651825,
        0.067360520362854,
        0.076678566634655,
        0.0797014907002449,
        0.08585279434919357,
        0.08837522566318512,
        0.09026700258255005,
        0.09281065315008163,
        0.09405135363340378,
        0.09652531147003174,
        0.09571083635091782,
        0.09793278574943542,
        0.09508897364139557,
        0.09766053408384323,
        0.09508920460939407,
        0.09857666492462158,
        0.09573151916265488,
        0.10091027617454529,
        0.09348288178443909,
        0.10140916705131531,
        0.09920287877321243,
        0.11094733327627182,
        0.10943257063627243,
        0.12530675530433655,
        0.12546418607234955,
        0.14505283534526825,
        0.1458611935377121,
        0.17291401326656342,
        0.1839449107646942,
        0.21681295335292816
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.0001656424137763679,
        0.00018305814592167735,
        0.00020359577320050448,
        0.00018629668920766562,
        0.0001775996934156865,
        0.00016671977937221527,
        0.00015271134907379746,
        0.0001390485413139686,
        0.00012298233923502266,
        0.00011299804464215413,
        9.434259118279442e-05,
        8.304810035042465e-05,
        7.153192564146593e-05,
        5.851028618053533e-05,
        4.2740048229461536e-05,
        2.919154394476209e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00018464825552655384,
        "first_to_last_ratio": 3.6568753347836163,
        "imbalance_abs_log_ratio": 1.2966090491094597,
        "last_quartile_mean": 5.049345099905622e-05,
        "mean": 0.0001243760790430315,
        "normalized": [
          1.3317867475068026,
          1.4718115197886492,
          1.6369367386960691,
          1.4978498328702814,
          1.4279248452127258,
          1.3404489083028075,
          1.2278192900819984,
          1.1179685224347742,
          0.9887941490137615,
          0.9085191100377042,
          0.7585268156761388,
          0.6677176269698272,
          0.5751260707994935,
          0.4704303804294394,
          0.34363559744212857,
          0.23470384473739866
        ],
        "population_cv": 0.4334270573824917
      },
      "activation_output_rms_by_block": [
        0.08618219196796417,
        0.145728200674057,
        0.16391748189926147,
        0.16589416563510895,
        0.16637063026428223,
        0.1656494140625,
        0.16352342069149017,
        0.1510169357061386,
        0.14240477979183197,
        0.14064612984657288,
        0.14404520392417908,
        0.15505897998809814,
        0.1711372584104538,
        0.19663043320178986,
        0.24908031523227692,
        0.4027986526489258
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.1404305100440979,
        "first_to_last_ratio": 0.5508987205974385,
        "imbalance_abs_log_ratio": 0.5962042968923701,
        "last_quartile_mean": 0.2549116648733616,
        "mean": 0.1756302621215582,
        "normalized": [
          0.49070240473885574,
          0.8297442531469593,
          0.9333100111517798,
          0.9445648126419658,
          0.9472776972178796,
          0.9431712511358084,
          0.9310663134939208,
          0.859857144673712,
          0.8108214271938511,
          0.8008080620481458,
          0.820161640620235,
          0.8828716538655387,
          0.9744178272193509,
          1.1195703452614385,
          1.4182084125108352,
          2.293446743079724
        ],
        "population_cv": 0.38002251655845937
      },
      "bits_per_byte": 2.081392170877702,
      "branch_output_rms_by_sublayer": [
        0.03403226286172867,
        0.04189683124423027,
        0.08070491254329681,
        0.007666230201721191,
        0.03132990375161171,
        0.004042334854602814,
        0.023764250800013542,
        0.004978273529559374,
        0.022670205682516098,
        0.009758620522916317,
        0.027493907138705254,
        0.012959085404872894,
        0.036309871822595596,
        0.017362121492624283,
        0.039587654173374176,
        0.02089655213057995,
        0.04225468263030052,
        0.02445071190595627,
        0.032471224665641785,
        0.02915186993777752,
        0.04851555824279785,
        0.047949694097042084,
        0.01816859096288681,
        0.05110293999314308,
        0.02429918199777603,
        0.05613848567008972,
        0.01771482080221176,
        0.06749960780143738,
        0.015686625614762306,
        0.09889734536409378,
        0.016557788476347923,
        0.2156895101070404
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00041094985323999886,
        0.0003483840396349268,
        0.0002459421498427246,
        0.00023334166050981232,
        0.000254581953755271,
        0.0003388594313501643,
        0.0004326353023247584,
        0.0004560987158898983,
        0.0004819816052452772,
        0.0004606041111206281,
        0.00053396017856909,
        0.0004147477851776299,
        0.00037952749197896316,
        0.00033346538787636917,
        0.0002981284512971557,
        0.0002588526279031963
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00030965442580686565,
        "first_to_last_ratio": 0.9753095285106969,
        "imbalance_abs_log_ratio": 0.0250003932265083,
        "last_quartile_mean": 0.0003174934897639211,
        "mean": 0.0003676287966072415,
        "normalized": [
          1.117839127490983,
          0.9476516607242959,
          0.6689958787572303,
          0.6347208452201429,
          0.6924973127914547,
          0.9217434392447414,
          1.176826479093712,
          1.2406501343178897,
          1.3110550906059877,
          1.2529054181049841,
          1.4524438332820528,
          1.1281700155298995,
          1.0323660591377277,
          0.9070709121642312,
          0.8109496700163646,
          0.7041141235183029
        ],
        "population_cv": 0.2454972418327454
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.05505011975765228,
        0.06996718049049377,
        0.08618219196796417,
        0.1434549242258072,
        0.145728200674057,
        0.16383236646652222,
        0.16391748189926147,
        0.16620540618896484,
        0.16589416563510895,
        0.16680952906608582,
        0.16637063026428223,
        0.16661377251148224,
        0.1656494140625,
        0.1662609726190567,
        0.16352342069149017,
        0.15300041437149048,
        0.1510169357061386,
        0.14022879302501678,
        0.14240477979183197,
        0.13479726016521454,
        0.14064612984657288,
        0.13225789368152618,
        0.14404520392417908,
        0.1422654092311859,
        0.15505897998809814,
        0.1554868519306183,
        0.1711372584104538,
        0.17103315889835358,
        0.19663043320178986,
        0.19691547751426697,
        0.24908031523227692,
        0.25020092725753784
      ],
      "loss_nats": 1.4427111148834229,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 2000,
      "stream_state_rms_by_sublayer": [
        0.06996718049049377,
        0.08618219196796417,
        0.1434549242258072,
        0.145728200674057,
        0.16383236646652222,
        0.16391748189926147,
        0.16620540618896484,
        0.16589416563510895,
        0.16680952906608582,
        0.16637063026428223,
        0.16661377251148224,
        0.1656494140625,
        0.1662609726190567,
        0.16352342069149017,
        0.15300041437149048,
        0.1510169357061386,
        0.14022879302501678,
        0.14240477979183197,
        0.13479726016521454,
        0.14064612984657288,
        0.13225789368152618,
        0.14404520392417908,
        0.1422654092311859,
        0.15505897998809814,
        0.1554868519306183,
        0.1711372584104538,
        0.17103315889835358,
        0.19663043320178986,
        0.19691547751426697,
        0.24908031523227692,
        0.25020092725753784,
        0.4027986526489258
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.0001670715573709458,
        0.0001843688660301268,
        0.00019890567637048662,
        0.00018651749996934086,
        0.00017948838649317622,
        0.0001703005691524595,
        0.00015747325960546732,
        0.0001445972011424601,
        0.00012834250810556114,
        0.00011769140110118315,
        9.774974023457617e-05,
        8.55058096931316e-05,
        7.324934267671779e-05,
        5.983562004985288e-05,
        4.2463150748517364e-05,
        2.6351406631874852e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00018421589993522502,
        "first_to_last_ratio": 3.6496550331101454,
        "imbalance_abs_log_ratio": 1.2946326516512365,
        "last_quartile_mean": 5.047488002674072e-05,
        "mean": 0.00012624449971099239,
        "normalized": [
          1.323396724240799,
          1.4604110789158873,
          1.5755591477318631,
          1.477430703090666,
          1.42175213101619,
          1.348974169506975,
          1.2473672909787434,
          1.145374265599547,
          1.016618612291004,
          0.9322497326268504,
          0.7742891013735379,
          0.6773032479741881,
          0.5802180914368962,
          0.47396615446084944,
          0.336356442028975,
          0.2087331067270282
        ],
        "population_cv": 0.42845315545725604
      },
      "activation_output_rms_by_block": [
        0.10210617631673813,
        0.1711120903491974,
        0.18304166197776794,
        0.1841333508491516,
        0.18528582155704498,
        0.18522554636001587,
        0.18429037928581238,
        0.1677425056695938,
        0.15884697437286377,
        0.1616142988204956,
        0.1705332249403,
        0.18593955039978027,
        0.20567089319229126,
        0.23390550911426544,
        0.2939614951610565,
        0.5084910988807678
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.16009831987321377,
        "first_to_last_ratio": 0.5156025192452343,
        "imbalance_abs_log_ratio": 0.6624191218832031,
        "last_quartile_mean": 0.31050724908709526,
        "mean": 0.20511878607794642,
        "normalized": [
          0.4977904670220559,
          0.8342097455869972,
          0.8923690778289365,
          0.8976913054622885,
          0.9033098581552409,
          0.9030160030765246,
          0.8984568542433792,
          0.8177822659590556,
          0.7744145595347903,
          0.7879058857099571,
          0.8313876471338725,
          0.9064969326072458,
          1.002691645777072,
          1.1403417189948652,
          1.4331280950997307,
          2.479007937807988
        ],
        "population_cv": 0.4251348773789681
      },
      "bits_per_byte": 1.9581638498120857,
      "branch_output_rms_by_sublayer": [
        0.052206385880708694,
        0.0456010177731514,
        0.08834584802389145,
        0.009068594314157963,
        0.02721724845468998,
        0.003163012443110347,
        0.024817200377583504,
        0.004634208977222443,
        0.025762483477592468,
        0.010066481307148933,
        0.02899768576025963,
        0.014108272269368172,
        0.04359622299671173,
        0.02107245847582817,
        0.046488646417856216,
        0.02715279720723629,
        0.04875380918383598,
        0.032857682555913925,
        0.03842322900891304,
        0.03946002945303917,
        0.05467399209737778,
        0.0617416612803936,
        0.025746947154402733,
        0.06822259724140167,
        0.032913338392972946,
        0.0761992558836937,
        0.024072155356407166,
        0.08795499801635742,
        0.022052984684705734,
        0.12569737434387207,
        0.025944042950868607,
        0.30153149366378784
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00042380566817482924,
        0.00033628063183600165,
        0.000202638705536708,
        0.00018804477448005897,
        0.00023208100123381636,
        0.0003223172253228011,
        0.0004741333361060697,
        0.0004905991387745841,
        0.0005898184863404294,
        0.0005622263144383919,
        0.000639932357888532,
        0.0005200270037733745,
        0.0004834555123403282,
        0.0004217229809514017,
        0.0003667204712485774,
        0.0003029597160326848
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.0002876924450068995,
        "first_to_last_ratio": 0.7307130437944471,
        "imbalance_abs_log_ratio": 0.3137344492502099,
        "last_quartile_mean": 0.000393714670143248,
        "mean": 0.0004097977077799118,
        "normalized": [
          1.0341826226183788,
          0.8206015442541381,
          0.4944847218265511,
          0.4588721969646822,
          0.5663306476044494,
          0.7865276432827353,
          1.156993626622993,
          1.1971739457314583,
          1.4392918143339164,
          1.371960613162688,
          1.561581106335074,
          1.2689846573096573,
          1.1797418657109118,
          1.029100390131744,
          0.894881704525127,
          0.739290899585495
        ],
        "population_cv": 0.3269281757976474
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.05859006941318512,
        0.08463093638420105,
        0.10210617631673813,
        0.16779249906539917,
        0.1711120903491974,
        0.1829725056886673,
        0.18304166197776794,
        0.18442119657993317,
        0.1841333508491516,
        0.1860278993844986,
        0.18528582155704498,
        0.1865086406469345,
        0.18522554636001587,
        0.18776042759418488,
        0.18429037928581238,
        0.17156346142292023,
        0.1677425056695938,
        0.15597018599510193,
        0.15884697437286377,
        0.1531117558479309,
        0.1616142988204956,
        0.15448127686977386,
        0.1705332249403,
        0.1696566343307495,
        0.18593955039978027,
        0.1874322146177292,
        0.20567089319229126,
        0.20625530183315277,
        0.23390550911426544,
        0.234551340341568,
        0.2939614951610565,
        0.2957085371017456
      ],
      "loss_nats": 1.3572957515716553,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 4000,
      "stream_state_rms_by_sublayer": [
        0.08463093638420105,
        0.10210617631673813,
        0.16779249906539917,
        0.1711120903491974,
        0.1829725056886673,
        0.18304166197776794,
        0.18442119657993317,
        0.1841333508491516,
        0.1860278993844986,
        0.18528582155704498,
        0.1865086406469345,
        0.18522554636001587,
        0.18776042759418488,
        0.18429037928581238,
        0.17156346142292023,
        0.1677425056695938,
        0.15597018599510193,
        0.15884697437286377,
        0.1531117558479309,
        0.1616142988204956,
        0.15448127686977386,
        0.1705332249403,
        0.1696566343307495,
        0.18593955039978027,
        0.1874322146177292,
        0.20567089319229126,
        0.20625530183315277,
        0.23390550911426544,
        0.234551340341568,
        0.2939614951610565,
        0.2957085371017456,
        0.5084910988807678
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.000266833056230098,
        0.0002936421660706401,
        0.00031041845795698464,
        0.0002957055112347007,
        0.0002881943073589355,
        0.0002774772292468697,
        0.00026020428049378097,
        0.00024136141291819513,
        0.00021430903871078044,
        0.00019666840671561658,
        0.00016092424630187452,
        0.0001381110487272963,
        0.00011530883784871548,
        9.161087655229494e-05,
        6.125834624981508e-05,
        3.126815863652155e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00029164979787310585,
        "first_to_last_ratio": 3.8958554703706607,
        "imbalance_abs_log_ratio": 1.3599132881647322,
        "last_quartile_mean": 7.486155482183676e-05,
        "mean": 0.00020270596132831997,
        "normalized": [
          1.3163552491577308,
          1.4486113982362465,
          1.531373107741041,
          1.4587904040757373,
          1.4217357273087359,
          1.3688656585557626,
          1.2836538145631138,
          1.1906971622174716,
          1.05724092822148,
          0.9702152093942396,
          0.7938801860949112,
          0.6813368872936094,
          0.5688477886545793,
          0.4519397256596421,
          0.3022029833182646,
          0.15425376950743425
        ],
        "population_cv": 0.4374351028998013
      },
      "activation_output_rms_by_block": [
        0.10593125969171524,
        0.16277404129505157,
        0.16965323686599731,
        0.16991853713989258,
        0.16967523097991943,
        0.1679684966802597,
        0.16474221646785736,
        0.14505790174007416,
        0.1350426822900772,
        0.13761113584041595,
        0.1448632776737213,
        0.15946830809116364,
        0.17768383026123047,
        0.2030501663684845,
        0.25671765208244324,
        0.47381165623664856
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.15206926874816418,
        "first_to_last_ratio": 0.5473743911850654,
        "imbalance_abs_log_ratio": 0.6026222661058785,
        "last_quartile_mean": 0.2778158262372017,
        "mean": 0.18399810185655951,
        "normalized": [
          0.5757193070083771,
          0.8846506548309193,
          0.9220379729691717,
          0.9234798371580865,
          0.9221575074301264,
          0.9128816818512828,
          0.8953473693782256,
          0.7883662944151337,
          0.7339351924149371,
          0.7478943231039105,
          0.7873085440123357,
          0.8666845281669334,
          0.9656829525325674,
          1.1035448970380004,
          1.3952190239580522,
          2.57508991373194
        ],
        "population_cv": 0.4416592993774559
      },
      "bits_per_byte": 1.8703918160136477,
      "branch_output_rms_by_sublayer": [
        0.06665868312120438,
        0.03488925099372864,
        0.07055649906396866,
        0.008275732398033142,
        0.021501976996660233,
        0.0027294722385704517,
        0.020079251378774643,
        0.004083180334419012,
        0.019392874091863632,
        0.008691227994859219,
        0.02471928857266903,
        0.012035587802529335,
        0.03809819743037224,
        0.018656741827726364,
        0.04367014393210411,
        0.024902483448386192,
        0.04372077062726021,
        0.03141119331121445,
        0.03309999778866768,
        0.037526171654462814,
        0.0474277026951313,
        0.05794820562005043,
        0.02467101812362671,
        0.0651254877448082,
        0.02942940965294838,
        0.07373781502246857,
        0.023407138884067535,
        0.08589385449886322,
        0.019243014976382256,
        0.1193988099694252,
        0.02799036167562008,
        0.300537645816803
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.000552485054575078,
        0.0004878954173704819,
        0.00026717869748665266,
        0.00024508760509843496,
        0.0003479630309338483,
        0.0004644527429795662,
        0.0007202858637767138,
        0.0008028534351930377,
        0.0009144494502519988,
        0.0008971273720051169,
        0.001000538678877296,
        0.0008212401598758333,
        0.0007501935670996717,
        0.0006492653356624181,
        0.0005392275192896546,
        0.00040293680800761964
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00038816169363266187,
        "first_to_last_ratio": 0.6630643028303428,
        "imbalance_abs_log_ratio": 0.4108833058215335,
        "last_quartile_mean": 0.000585405807514841,
        "mean": 0.0006164487961552139,
        "normalized": [
          0.8962383542979121,
          0.7914613840005554,
          0.4334158800423393,
          0.3975798259758869,
          0.5644638015421408,
          0.7534328007068132,
          1.1684439458218279,
          1.3023846265909318,
          1.4834150962016837,
          1.4553153118320499,
          1.6230685908021032,
          1.3322114748181866,
          1.2169600650996852,
          1.0532348180609332,
          0.8747320501764496,
          0.6536419740305005
        ],
        "population_cv": 0.37363268494405877
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.060181986540555954,
        0.0962739959359169,
        0.10593125969171524,
        0.15956835448741913,
        0.16277404129505157,
        0.16968216001987457,
        0.16965323686599731,
        0.1704123467206955,
        0.16991853713989258,
        0.17062334716320038,
        0.16967523097991943,
        0.16979995369911194,
        0.1679684966802597,
        0.16850441694259644,
        0.16474221646785736,
        0.14905613660812378,
        0.14505790174007416,
        0.13280805945396423,
        0.1350426822900772,
        0.13030488789081573,
        0.13761113584041595,
        0.13074533641338348,
        0.1448632776737213,
        0.1443873643875122,
        0.15946830809116364,
        0.16084782779216766,
        0.17768383026123047,
        0.17834986746311188,
        0.2030501663684845,
        0.20354533195495605,
        0.25671765208244324,
        0.2584022283554077
      ],
      "loss_nats": 1.2964568138122559,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 8000,
      "stream_state_rms_by_sublayer": [
        0.0962739959359169,
        0.10593125969171524,
        0.15956835448741913,
        0.16277404129505157,
        0.16968216001987457,
        0.16965323686599731,
        0.1704123467206955,
        0.16991853713989258,
        0.17062334716320038,
        0.16967523097991943,
        0.16979995369911194,
        0.1679684966802597,
        0.16850441694259644,
        0.16474221646785736,
        0.14905613660812378,
        0.14505790174007416,
        0.13280805945396423,
        0.1350426822900772,
        0.13030488789081573,
        0.13761113584041595,
        0.13074533641338348,
        0.1448632776737213,
        0.1443873643875122,
        0.15946830809116364,
        0.16084782779216766,
        0.17768383026123047,
        0.17834986746311188,
        0.2030501663684845,
        0.20354533195495605,
        0.25671765208244324,
        0.2584022283554077,
        0.47381165623664856
      ]
    }
  ],
  "environment": {
    "autocast": "cuda-bfloat16-forward-fp32-cross-entropy",
    "compile": false,
    "compute_capability": [
      12,
      0
    ],
    "cublas_workspace_config": ":4096:8",
    "cuda": "12.8",
    "deterministic_algorithms": true,
    "gpu": "NVIDIA GeForce RTX 5090",
    "python": "3.10.14",
    "torch": "2.11.0+cu128"
  },
  "evaluations": [
    {
      "bits_per_byte": 8.032388224749795,
      "cross_entropy_nats": 5.567627251148224,
      "step": 0
    },
    {
      "bits_per_byte": 5.232053968773903,
      "cross_entropy_nats": 3.626583456993103,
      "step": 100
    },
    {
      "bits_per_byte": 3.1492899782815944,
      "cross_entropy_nats": 2.1829214692115784,
      "step": 500
    },
    {
      "bits_per_byte": 1.9917407321684666,
      "cross_entropy_nats": 1.3805694729089737,
      "step": 2000
    },
    {
      "bits_per_byte": 1.8310010171491868,
      "cross_entropy_nats": 1.269153192639351,
      "step": 4000
    },
    {
      "bits_per_byte": 1.7368251448523782,
      "cross_entropy_nats": 1.2038754522800446,
      "step": 8000
    }
  ],
  "gradient_gate": null,
  "hashes": {
    "final_mixer_parameters": null,
    "final_model_state": "5f353d7386c2ddcfa7f2378c6aecd3f06c446e23fb55dc072e8237f189736ab6",
    "final_optimizer_state": "34afebd4020a97001cb564ade8591d46cc164cbd65eb2e1cad4d8d953e6b7cce",
    "final_public_parameters": "5f353d7386c2ddcfa7f2378c6aecd3f06c446e23fb55dc072e8237f189736ab6",
    "initial_mixer_parameters": null,
    "initial_public_parameter_elements": 9541824,
    "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe",
    "initial_public_parameter_tensors": 115,
    "initial_public_parameters": "9fd4f04212ab5cea822f469902d8e80ecc368da329f3c20abacfe6b7a50ed523"
  },
  "manifest": {
    "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716",
    "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371",
    "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
    "input_gate_tensor_hashes": {
      "0": "add6249175ac3bb3fa911069fbe8f25d17d9ab05df34b66ee89996983522ce83",
      "1": "619b5ee4f70e975312ead9adf5df4f9d8584ce2839369c749442e429923405cf",
      "7999": "fe22994f7750c2f8323fdb5ac51c9be1e208733abaf6a337f5b1ae1df1cd2730"
    },
    "path": "experiments/k3/attnres_gradient/manifest.json",
    "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338"
  },
  "model": {
    "attnres_aggregation_groups": 8,
    "context": 256,
    "d_ff": 768,
    "d_head": 32,
    "d_model": 192,
    "heads": 6,
    "layers": 16,
    "parameters": {
      "core": 9541824,
      "embedding": 98304,
      "mixer": 0,
      "total": 9541824
    },
    "sublayers": 32,
    "sublayers_per_attnres_group": 4,
    "transformer_blocks_per_attnres_group": 2,
    "vocabulary": 256
  },
  "optimizer": {
    "betas": [
      0.9,
      0.95
    ],
    "epsilon": 1e-08,
    "grad_clip": 1.0,
    "min_lr": 3e-05,
    "name": "AdamW",
    "peak_lr": 0.0003,
    "warmup_steps": 400,
    "weight_decay_ndim_ge_2": 0.1
  },
  "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
  "run_kind": "formal",
  "schema_version": 1,
  "seed": 2026073002,
  "steps": 8000,
  "target_bytes_seen": 65536000,
  "timing": {
    "mean_ms": 21.682085667085165,
    "measured_steps": 7980,
    "median_ms": 21.630175004247576,
    "p95_ms": 22.00388514320366,
    "peak_allocated_bytes": 3036005376,
    "peak_reserved_bytes": 3282042880,
    "warmup_steps_excluded": 20
  },
  "training_history": [
    {
      "bits_per_byte": 8.03605033732594,
      "learning_rate": 7.499999999999999e-07,
      "loss_nats": 5.570165634155273,
      "step": 1,
      "unclipped_grad_norm": 14.13492488861084
    },
    {
      "bits_per_byte": 7.579231395965356,
      "learning_rate": 7.499999999999999e-06,
      "loss_nats": 5.253522872924805,
      "step": 10,
      "unclipped_grad_norm": 10.806709289550781
    },
    {
      "bits_per_byte": 6.980917512225713,
      "learning_rate": 1.4999999999999999e-05,
      "loss_nats": 4.838803291320801,
      "step": 20,
      "unclipped_grad_norm": 4.377180576324463
    },
    {
      "bits_per_byte": 6.73711765808501,
      "learning_rate": 2.2499999999999998e-05,
      "loss_nats": 4.669814109802246,
      "step": 30,
      "unclipped_grad_norm": 3.1878201961517334
    },
    {
      "bits_per_byte": 6.622164454248309,
      "learning_rate": 2.9999999999999997e-05,
      "loss_nats": 4.590134620666504,
      "step": 40,
      "unclipped_grad_norm": 3.092231035232544
    },
    {
      "bits_per_byte": 6.373620630666205,
      "learning_rate": 3.75e-05,
      "loss_nats": 4.4178571701049805,
      "step": 50,
      "unclipped_grad_norm": 3.1614222526550293
    },
    {
      "bits_per_byte": 6.096217430338719,
      "learning_rate": 4.4999999999999996e-05,
      "loss_nats": 4.225575923919678,
      "step": 60,
      "unclipped_grad_norm": 2.885824203491211
    },
    {
      "bits_per_byte": 5.853263042909657,
      "learning_rate": 5.2499999999999995e-05,
      "loss_nats": 4.057172775268555,
      "step": 70,
      "unclipped_grad_norm": 2.6939339637756348
    },
    {
      "bits_per_byte": 5.640556620149043,
      "learning_rate": 5.9999999999999995e-05,
      "loss_nats": 3.909735918045044,
      "step": 80,
      "unclipped_grad_norm": 2.5527353286743164
    },
    {
      "bits_per_byte": 5.452656287018279,
      "learning_rate": 6.749999999999999e-05,
      "loss_nats": 3.7794933319091797,
      "step": 90,
      "unclipped_grad_norm": 2.4638988971710205
    },
    {
      "bits_per_byte": 5.271040199950041,
      "learning_rate": 7.5e-05,
      "loss_nats": 3.653606653213501,
      "step": 100,
      "unclipped_grad_norm": 2.4236807823181152
    },
    {
      "bits_per_byte": 5.106934010531447,
      "learning_rate": 8.249999999999999e-05,
      "loss_nats": 3.5398569107055664,
      "step": 110,
      "unclipped_grad_norm": 2.3251681327819824
    },
    {
      "bits_per_byte": 4.82528283872765,
      "learning_rate": 8.999999999999999e-05,
      "loss_nats": 3.3446311950683594,
      "step": 120,
      "unclipped_grad_norm": 2.082082509994507
    },
    {
      "bits_per_byte": 4.666372933122225,
      "learning_rate": 9.75e-05,
      "loss_nats": 3.234483242034912,
      "step": 130,
      "unclipped_grad_norm": 1.8279871940612793
    },
    {
      "bits_per_byte": 4.4669253408472285,
      "learning_rate": 0.00010499999999999999,
      "loss_nats": 3.0962367057800293,
      "step": 140,
      "unclipped_grad_norm": 1.706978440284729
    },
    {
      "bits_per_byte": 4.329626087218621,
      "learning_rate": 0.0001125,
      "loss_nats": 3.001068115234375,
      "step": 150,
      "unclipped_grad_norm": 1.465320348739624
    },
    {
      "bits_per_byte": 4.211352242359061,
      "learning_rate": 0.00011999999999999999,
      "loss_nats": 2.9190869331359863,
      "step": 160,
      "unclipped_grad_norm": 1.149477243423462
    },
    {
      "bits_per_byte": 4.052300967014626,
      "learning_rate": 0.00012749999999999998,
      "loss_nats": 2.8088409900665283,
      "step": 170,
      "unclipped_grad_norm": 1.175323247909546
    },
    {
      "bits_per_byte": 3.9086503940896553,
      "learning_rate": 0.00013499999999999997,
      "loss_nats": 2.7092700004577637,
      "step": 180,
      "unclipped_grad_norm": 1.1839734315872192
    },
    {
      "bits_per_byte": 3.9167359864373195,
      "learning_rate": 0.0001425,
      "loss_nats": 2.714874505996704,
      "step": 190,
      "unclipped_grad_norm": 0.9742223024368286
    },
    {
      "bits_per_byte": 3.7936297730538158,
      "learning_rate": 0.00015,
      "loss_nats": 2.6295437812805176,
      "step": 200,
      "unclipped_grad_norm": 0.6956769227981567
    },
    {
      "bits_per_byte": 3.735681250741498,
      "learning_rate": 0.0001575,
      "loss_nats": 2.589376926422119,
      "step": 210,
      "unclipped_grad_norm": 0.6548250317573547
    },
    {
      "bits_per_byte": 3.7032831589719617,
      "learning_rate": 0.00016499999999999997,
      "loss_nats": 2.566920280456543,
      "step": 220,
      "unclipped_grad_norm": 0.6789838075637817
    },
    {
      "bits_per_byte": 3.722889181170322,
      "learning_rate": 0.0001725,
      "loss_nats": 2.580510139465332,
      "step": 230,
      "unclipped_grad_norm": 0.730959415435791
    },
    {
      "bits_per_byte": 3.721254658056522,
      "learning_rate": 0.00017999999999999998,
      "loss_nats": 2.5793771743774414,
      "step": 240,
      "unclipped_grad_norm": 0.7681548595428467
    },
    {
      "bits_per_byte": 3.588607879099055,
      "learning_rate": 0.0001875,
      "loss_nats": 2.487433433532715,
      "step": 250,
      "unclipped_grad_norm": 1.0810788869857788
    },
    {
      "bits_per_byte": 3.588186865569743,
      "learning_rate": 0.000195,
      "loss_nats": 2.4871416091918945,
      "step": 260,
      "unclipped_grad_norm": 0.9089803695678711
    },
    {
      "bits_per_byte": 3.5492888614934976,
      "learning_rate": 0.00020249999999999996,
      "loss_nats": 2.460179567337036,
      "step": 270,
      "unclipped_grad_norm": 0.6836683750152588
    },
    {
      "bits_per_byte": 3.551394961035964,
      "learning_rate": 0.00020999999999999998,
      "loss_nats": 2.461639404296875,
      "step": 280,
      "unclipped_grad_norm": 0.6695814728736877
    },
    {
      "bits_per_byte": 3.464650695559082,
      "learning_rate": 0.00021749999999999997,
      "loss_nats": 2.401512861251831,
      "step": 290,
      "unclipped_grad_norm": 0.8494355082511902
    },
    {
      "bits_per_byte": 3.543094390374721,
      "learning_rate": 0.000225,
      "loss_nats": 2.455885887145996,
      "step": 300,
      "unclipped_grad_norm": 0.7996388077735901
    },
    {
      "bits_per_byte": 3.442018122670838,
      "learning_rate": 0.00023249999999999996,
      "loss_nats": 2.3858251571655273,
      "step": 310,
      "unclipped_grad_norm": 0.7709511518478394
    },
    {
      "bits_per_byte": 3.43619410218202,
      "learning_rate": 0.00023999999999999998,
      "loss_nats": 2.3817882537841797,
      "step": 320,
      "unclipped_grad_norm": 0.8889051079750061
    },
    {
      "bits_per_byte": 3.4344873463548575,
      "learning_rate": 0.0002475,
      "loss_nats": 2.3806052207946777,
      "step": 330,
      "unclipped_grad_norm": 0.977308988571167
    },
    {
      "bits_per_byte": 3.4251934039022314,
      "learning_rate": 0.00025499999999999996,
      "loss_nats": 2.3741631507873535,
      "step": 340,
      "unclipped_grad_norm": 0.7606520652770996
    },
    {
      "bits_per_byte": 3.3985436602551338,
      "learning_rate": 0.0002625,
      "loss_nats": 2.3556909561157227,
      "step": 350,
      "unclipped_grad_norm": 0.6722884178161621
    },
    {
      "bits_per_byte": 3.3928290207322647,
      "learning_rate": 0.00026999999999999995,
      "loss_nats": 2.3517298698425293,
      "step": 360,
      "unclipped_grad_norm": 0.8897987008094788
    },
    {
      "bits_per_byte": 3.4104885432881593,
      "learning_rate": 0.00027749999999999997,
      "loss_nats": 2.3639705181121826,
      "step": 370,
      "unclipped_grad_norm": 1.0889073610305786
    },
    {
      "bits_per_byte": 3.4536713231280003,
      "learning_rate": 0.000285,
      "loss_nats": 2.393902540206909,
      "step": 380,
      "unclipped_grad_norm": 1.0755773782730103
    },
    {
      "bits_per_byte": 3.3990293392611703,
      "learning_rate": 0.0002925,
      "loss_nats": 2.356027603149414,
      "step": 390,
      "unclipped_grad_norm": 1.091948390007019
    },
    {
      "bits_per_byte": 3.337470901107242,
      "learning_rate": 0.0003,
      "loss_nats": 2.3133585453033447,
      "step": 400,
      "unclipped_grad_norm": 0.9980842471122742
    },
    {
      "bits_per_byte": 3.3053706832890004,
      "learning_rate": 0.0002999988466112388,
      "loss_nats": 2.2911083698272705,
      "step": 410,
      "unclipped_grad_norm": 0.7333840727806091
    },
    {
      "bits_per_byte": 3.321116382906094,
      "learning_rate": 0.00029999538646466355,
      "loss_nats": 2.3020224571228027,
      "step": 420,
      "unclipped_grad_norm": 0.8894042372703552
    },
    {
      "bits_per_byte": 3.3009490093353175,
      "learning_rate": 0.0002999896196193985,
      "loss_nats": 2.28804349899292,
      "step": 430,
      "unclipped_grad_norm": 0.9840139746665955
    },
    {
      "bits_per_byte": 3.3110151538903176,
      "learning_rate": 0.00029998154617398326,
      "loss_nats": 2.295020818710327,
      "step": 440,
      "unclipped_grad_norm": 1.0280733108520508
    },
    {
      "bits_per_byte": 3.2949894665176194,
      "learning_rate": 0.0002999711662663707,
      "loss_nats": 2.2839126586914062,
      "step": 450,
      "unclipped_grad_norm": 0.928855836391449
    },
    {
      "bits_per_byte": 3.2291744577548367,
      "learning_rate": 0.0002999584800739247,
      "loss_nats": 2.238293170928955,
      "step": 460,
      "unclipped_grad_norm": 0.8088337779045105
    },
    {
      "bits_per_byte": 3.242055958304461,
      "learning_rate": 0.0002999434878134173,
      "loss_nats": 2.2472219467163086,
      "step": 470,
      "unclipped_grad_norm": 1.018490195274353
    },
    {
      "bits_per_byte": 3.1972775234299147,
      "learning_rate": 0.000299926189741025,
      "loss_nats": 2.21618390083313,
      "step": 480,
      "unclipped_grad_norm": 1.066487193107605
    },
    {
      "bits_per_byte": 3.156052938052798,
      "learning_rate": 0.000299906586152324,
      "loss_nats": 2.1876091957092285,
      "step": 490,
      "unclipped_grad_norm": 1.2544180154800415
    },
    {
      "bits_per_byte": 3.153048401142225,
      "learning_rate": 0.0002998846773822856,
      "loss_nats": 2.1855266094207764,
      "step": 500,
      "unclipped_grad_norm": 0.7976559400558472
    },
    {
      "bits_per_byte": 3.126950721804397,
      "learning_rate": 0.0002998604638052702,
      "loss_nats": 2.1674370765686035,
      "step": 510,
      "unclipped_grad_norm": 0.9979055523872375
    },
    {
      "bits_per_byte": 3.0098652770622984,
      "learning_rate": 0.0002998339458350211,
      "loss_nats": 2.0862796306610107,
      "step": 520,
      "unclipped_grad_norm": 1.0761884450912476
    },
    {
      "bits_per_byte": 3.0772147150360794,
      "learning_rate": 0.0002998051239246572,
      "loss_nats": 2.132962703704834,
      "step": 530,
      "unclipped_grad_norm": 1.010690450668335
    },
    {
      "bits_per_byte": 3.0006050432092413,
      "learning_rate": 0.00029977399856666537,
      "loss_nats": 2.0798609256744385,
      "step": 540,
      "unclipped_grad_norm": 1.0196499824523926
    },
    {
      "bits_per_byte": 2.9739387892276614,
      "learning_rate": 0.0002997405702928922,
      "loss_nats": 2.0613772869110107,
      "step": 550,
      "unclipped_grad_norm": 0.949600875377655
    },
    {
      "bits_per_byte": 2.9792871057041928,
      "learning_rate": 0.0002997048396745345,
      "loss_nats": 2.065084457397461,
      "step": 560,
      "unclipped_grad_norm": 1.027385950088501
    },
    {
      "bits_per_byte": 2.952135172716576,
      "learning_rate": 0.0002996668073221302,
      "loss_nats": 2.046264171600342,
      "step": 570,
      "unclipped_grad_norm": 1.2502241134643555
    },
    {
      "bits_per_byte": 2.8483484584350953,
      "learning_rate": 0.000299626473885547,
      "loss_nats": 1.9743247032165527,
      "step": 580,
      "unclipped_grad_norm": 0.9710299968719482
    },
    {
      "bits_per_byte": 2.9027066709280014,
      "learning_rate": 0.00029958384005397226,
      "loss_nats": 2.012002944946289,
      "step": 590,
      "unclipped_grad_norm": 1.0665416717529297
    },
    {
      "bits_per_byte": 2.824687945242644,
      "learning_rate": 0.0002995389065559004,
      "loss_nats": 1.957924485206604,
      "step": 600,
      "unclipped_grad_norm": 1.3016425371170044
    },
    {
      "bits_per_byte": 2.889905658258981,
      "learning_rate": 0.000299491674159121,
      "loss_nats": 2.0031299591064453,
      "step": 610,
      "unclipped_grad_norm": 1.458404779434204
    },
    {
      "bits_per_byte": 2.7619725797963617,
      "learning_rate": 0.00029944214367070535,
      "loss_nats": 1.9144535064697266,
      "step": 620,
      "unclipped_grad_norm": 1.0435335636138916
    },
    {
      "bits_per_byte": 2.780679820661313,
      "learning_rate": 0.00029939031593699284,
      "loss_nats": 1.9274203777313232,
      "step": 630,
      "unclipped_grad_norm": 1.2443503141403198
    },
    {
      "bits_per_byte": 2.796278303129268,
      "learning_rate": 0.00029933619184357654,
      "loss_nats": 1.938232421875,
      "step": 640,
      "unclipped_grad_norm": 1.0311143398284912
    },
    {
      "bits_per_byte": 2.704606046774563,
      "learning_rate": 0.00029927977231528774,
      "loss_nats": 1.874690055847168,
      "step": 650,
      "unclipped_grad_norm": 0.9809540510177612
    },
    {
      "bits_per_byte": 2.691190368111359,
      "learning_rate": 0.00029922105831618057,
      "loss_nats": 1.8653910160064697,
      "step": 660,
      "unclipped_grad_norm": 1.5349526405334473
    },
    {
      "bits_per_byte": 2.7107573502479734,
      "learning_rate": 0.0002991600508495153,
      "loss_nats": 1.8789538145065308,
      "step": 670,
      "unclipped_grad_norm": 1.2438547611236572
    },
    {
      "bits_per_byte": 2.7307020062858824,
      "learning_rate": 0.00029909675095774114,
      "loss_nats": 1.8927783966064453,
      "step": 680,
      "unclipped_grad_norm": 1.507369041442871
    },
    {
      "bits_per_byte": 2.6187595307351854,
      "learning_rate": 0.0002990311597224787,
      "loss_nats": 1.815185785293579,
      "step": 690,
      "unclipped_grad_norm": 1.4796253442764282
    },
    {
      "bits_per_byte": 2.562429709132788,
      "learning_rate": 0.00029896327826450103,
      "loss_nats": 1.7761409282684326,
      "step": 700,
      "unclipped_grad_norm": 1.6683489084243774
    },
    {
      "bits_per_byte": 2.6317965036011897,
      "learning_rate": 0.0002988931077437152,
      "loss_nats": 1.8242223262786865,
      "step": 710,
      "unclipped_grad_norm": 1.1483558416366577
    },
    {
      "bits_per_byte": 2.6874191325432393,
      "learning_rate": 0.00029882064935914164,
      "loss_nats": 1.8627769947052002,
      "step": 720,
      "unclipped_grad_norm": 1.464730143547058
    },
    {
      "bits_per_byte": 2.5059506065894417,
      "learning_rate": 0.0002987459043488943,
      "loss_nats": 1.736992597579956,
      "step": 730,
      "unclipped_grad_norm": 1.4666827917099
    },
    {
      "bits_per_byte": 2.538973683312213,
      "learning_rate": 0.0002986688739901591,
      "loss_nats": 1.7598824501037598,
      "step": 740,
      "unclipped_grad_norm": 1.090103030204773
    },
    {
      "bits_per_byte": 2.545309180204697,
      "learning_rate": 0.0002985895595991723,
      "loss_nats": 1.7642738819122314,
      "step": 750,
      "unclipped_grad_norm": 1.1345367431640625
    },
    {
      "bits_per_byte": 2.5190732268155758,
      "learning_rate": 0.00029850796253119824,
      "loss_nats": 1.7460885047912598,
      "step": 760,
      "unclipped_grad_norm": 1.0966333150863647
    },
    {
      "bits_per_byte": 2.5090782830780327,
      "learning_rate": 0.00029842408418050544,
      "loss_nats": 1.7391605377197266,
      "step": 770,
      "unclipped_grad_norm": 1.297254204750061
    },
    {
      "bits_per_byte": 2.4228073138563255,
      "learning_rate": 0.0002983379259803436,
      "loss_nats": 1.6793620586395264,
      "step": 780,
      "unclipped_grad_norm": 1.1724724769592285
    },
    {
      "bits_per_byte": 2.510659835535367,
      "learning_rate": 0.0002982494894029186,
      "loss_nats": 1.7402567863464355,
      "step": 790,
      "unclipped_grad_norm": 1.0964044332504272
    },
    {
      "bits_per_byte": 2.4699436308742007,
      "learning_rate": 0.0002981587759593675,
      "loss_nats": 1.7120344638824463,
      "step": 800,
      "unclipped_grad_norm": 1.4117348194122314
    },
    {
      "bits_per_byte": 2.5086447148152073,
      "learning_rate": 0.00029806578719973283,
      "loss_nats": 1.738860011100769,
      "step": 810,
      "unclipped_grad_norm": 1.889366626739502
    },
    {
      "bits_per_byte": 2.4598160885128424,
      "learning_rate": 0.00029797052471293596,
      "loss_nats": 1.7050145864486694,
      "step": 820,
      "unclipped_grad_norm": 1.144262671470642
    },
    {
      "bits_per_byte": 2.470564488243815,
      "learning_rate": 0.0002978729901267499,
      "loss_nats": 1.7124648094177246,
      "step": 830,
      "unclipped_grad_norm": 1.5465298891067505
    },
    {
      "bits_per_byte": 2.4487553682879026,
      "learning_rate": 0.0002977731851077716,
      "loss_nats": 1.69734787940979,
      "step": 840,
      "unclipped_grad_norm": 1.3992143869400024
    },
    {
      "bits_per_byte": 2.383892111514994,
      "learning_rate": 0.0002976711113613935,
      "loss_nats": 1.652388095855713,
      "step": 850,
      "unclipped_grad_norm": 1.1111818552017212
    },
    {
      "bits_per_byte": 2.4110481720862316,
      "learning_rate": 0.0002975667706317743,
      "loss_nats": 1.6712112426757812,
      "step": 860,
      "unclipped_grad_norm": 1.363444209098816
    },
    {
      "bits_per_byte": 2.3893498089574745,
      "learning_rate": 0.00029746016470180915,
      "loss_nats": 1.6561710834503174,
      "step": 870,
      "unclipped_grad_norm": 1.2841624021530151
    },
    {
      "bits_per_byte": 2.4391853936806416,
      "learning_rate": 0.00029735129539309926,
      "loss_nats": 1.6907144784927368,
      "step": 880,
      "unclipped_grad_norm": 1.406774640083313
    },
    {
      "bits_per_byte": 2.4004903291324147,
      "learning_rate": 0.0002972401645659207,
      "loss_nats": 1.6638931035995483,
      "step": 890,
      "unclipped_grad_norm": 1.1680277585983276
    },
    {
      "bits_per_byte": 2.3361910034716855,
      "learning_rate": 0.00029712677411919283,
      "loss_nats": 1.6193242073059082,
      "step": 900,
      "unclipped_grad_norm": 1.2712005376815796
    },
    {
      "bits_per_byte": 2.3780436694897533,
      "learning_rate": 0.0002970111259904454,
      "loss_nats": 1.648334264755249,
      "step": 910,
      "unclipped_grad_norm": 1.335773229598999
    },
    {
      "bits_per_byte": 2.3308695162886885,
      "learning_rate": 0.0002968932221557859,
      "loss_nats": 1.615635633468628,
      "step": 920,
      "unclipped_grad_norm": 1.1817855834960938
    },
    {
      "bits_per_byte": 2.330381085560238,
      "learning_rate": 0.0002967730646298655,
      "loss_nats": 1.6152970790863037,
      "step": 930,
      "unclipped_grad_norm": 1.0815848112106323
    },
    {
      "bits_per_byte": 2.2884202424547775,
      "learning_rate": 0.0002966506554658449,
      "loss_nats": 1.5862120389938354,
      "step": 940,
      "unclipped_grad_norm": 1.0937949419021606
    },
    {
      "bits_per_byte": 2.296288964679704,
      "learning_rate": 0.00029652599675535876,
      "loss_nats": 1.5916662216186523,
      "step": 950,
      "unclipped_grad_norm": 1.6217018365859985
    },
    {
      "bits_per_byte": 2.2717374093731326,
      "learning_rate": 0.0002963990906284808,
      "loss_nats": 1.574648380279541,
      "step": 960,
      "unclipped_grad_norm": 1.6217092275619507
    },
    {
      "bits_per_byte": 2.290087270289591,
      "learning_rate": 0.0002962699392536863,
      "loss_nats": 1.5873675346374512,
      "step": 970,
      "unclipped_grad_norm": 1.0035263299942017
    },
    {
      "bits_per_byte": 2.280217357939225,
      "learning_rate": 0.0002961385448378161,
      "loss_nats": 1.5805262327194214,
      "step": 980,
      "unclipped_grad_norm": 1.3000710010528564
    },
    {
      "bits_per_byte": 2.201290563857312,
      "learning_rate": 0.00029600490962603817,
      "loss_nats": 1.5258183479309082,
      "step": 990,
      "unclipped_grad_norm": 1.0893296003341675
    },
    {
      "bits_per_byte": 2.251006276672828,
      "learning_rate": 0.00029586903590180956,
      "loss_nats": 1.5602786540985107,
      "step": 1000,
      "unclipped_grad_norm": 1.206483006477356
    },
    {
      "bits_per_byte": 2.2491729416146304,
      "learning_rate": 0.00029573092598683735,
      "loss_nats": 1.5590078830718994,
      "step": 1010,
      "unclipped_grad_norm": 1.1218043565750122
    },
    {
      "bits_per_byte": 2.2577407733153136,
      "learning_rate": 0.00029559058224103895,
      "loss_nats": 1.5649466514587402,
      "step": 1020,
      "unclipped_grad_norm": 1.0566829442977905
    },
    {
      "bits_per_byte": 2.1604180269665116,
      "learning_rate": 0.0002954480070625017,
      "loss_nats": 1.4974876642227173,
      "step": 1030,
      "unclipped_grad_norm": 1.102095127105713
    },
    {
      "bits_per_byte": 2.2614810520062782,
      "learning_rate": 0.00029530320288744205,
      "loss_nats": 1.5675392150878906,
      "step": 1040,
      "unclipped_grad_norm": 1.2485690116882324
    },
    {
      "bits_per_byte": 2.2470793968056775,
      "learning_rate": 0.0002951561721901638,
      "loss_nats": 1.5575567483901978,
      "step": 1050,
      "unclipped_grad_norm": 1.242668628692627
    },
    {
      "bits_per_byte": 2.1863758844091876,
      "learning_rate": 0.00029500691748301594,
      "loss_nats": 1.5154802799224854,
      "step": 1060,
      "unclipped_grad_norm": 1.1564915180206299
    },
    {
      "bits_per_byte": 2.2939011575551245,
      "learning_rate": 0.00029485544131634963,
      "loss_nats": 1.5900111198425293,
      "step": 1070,
      "unclipped_grad_norm": 1.2426202297210693
    },
    {
      "bits_per_byte": 2.2229925386217375,
      "learning_rate": 0.0002947017462784745,
      "loss_nats": 1.5408610105514526,
      "step": 1080,
      "unclipped_grad_norm": 1.0772974491119385
    },
    {
      "bits_per_byte": 2.1433408376664382,
      "learning_rate": 0.00029454583499561476,
      "loss_nats": 1.485650658607483,
      "step": 1090,
      "unclipped_grad_norm": 1.0020748376846313
    },
    {
      "bits_per_byte": 2.2307484402276976,
      "learning_rate": 0.00029438771013186413,
      "loss_nats": 1.5462369918823242,
      "step": 1100,
      "unclipped_grad_norm": 1.1032538414001465
    },
    {
      "bits_per_byte": 2.263051941539259,
      "learning_rate": 0.0002942273743891401,
      "loss_nats": 1.5686280727386475,
      "step": 1110,
      "unclipped_grad_norm": 1.062880039215088
    },
    {
      "bits_per_byte": 2.1486932817265902,
      "learning_rate": 0.00029406483050713824,
      "loss_nats": 1.4893606901168823,
      "step": 1120,
      "unclipped_grad_norm": 1.1697474718093872
    },
    {
      "bits_per_byte": 2.172017052888652,
      "learning_rate": 0.00029390008126328504,
      "loss_nats": 1.5055274963378906,
      "step": 1130,
      "unclipped_grad_norm": 1.4915344715118408
    },
    {
      "bits_per_byte": 2.1695033544636413,
      "learning_rate": 0.00029373312947269045,
      "loss_nats": 1.5037851333618164,
      "step": 1140,
      "unclipped_grad_norm": 1.5511094331741333
    },
    {
      "bits_per_byte": 2.1545989939745684,
      "learning_rate": 0.00029356397798809995,
      "loss_nats": 1.4934542179107666,
      "step": 1150,
      "unclipped_grad_norm": 1.240431547164917
    },
    {
      "bits_per_byte": 2.097475816403741,
      "learning_rate": 0.0002933926296998457,
      "loss_nats": 1.4538594484329224,
      "step": 1160,
      "unclipped_grad_norm": 1.178969144821167
    },
    {
      "bits_per_byte": 2.2105972330261023,
      "learning_rate": 0.0002932190875357972,
      "loss_nats": 1.5322692394256592,
      "step": 1170,
      "unclipped_grad_norm": 1.1780297756195068
    },
    {
      "bits_per_byte": 2.102323491383611,
      "learning_rate": 0.00029304335446131126,
      "loss_nats": 1.4572196006774902,
      "step": 1180,
      "unclipped_grad_norm": 0.919143557548523
    },
    {
      "bits_per_byte": 2.136596881978155,
      "learning_rate": 0.00029286543347918116,
      "loss_nats": 1.4809761047363281,
      "step": 1190,
      "unclipped_grad_norm": 1.1034574508666992
    },
    {
      "bits_per_byte": 2.1069721824364325,
      "learning_rate": 0.0002926853276295856,
      "loss_nats": 1.4604418277740479,
      "step": 1200,
      "unclipped_grad_norm": 1.1095129251480103
    },
    {
      "bits_per_byte": 2.254480670185558,
      "learning_rate": 0.0002925030399900368,
      "loss_nats": 1.5626869201660156,
      "step": 1210,
      "unclipped_grad_norm": 1.2801767587661743
    },
    {
      "bits_per_byte": 2.1061446019204806,
      "learning_rate": 0.00029231857367532726,
      "loss_nats": 1.4598681926727295,
      "step": 1220,
      "unclipped_grad_norm": 1.0897294282913208
    },
    {
      "bits_per_byte": 2.047843343191243,
      "learning_rate": 0.0002921319318374776,
      "loss_nats": 1.4194568395614624,
      "step": 1230,
      "unclipped_grad_norm": 1.1728222370147705
    },
    {
      "bits_per_byte": 2.0460125878728084,
      "learning_rate": 0.00029194311766568174,
      "loss_nats": 1.4181878566741943,
      "step": 1240,
      "unclipped_grad_norm": 1.1321473121643066
    },
    {
      "bits_per_byte": 2.217450741662985,
      "learning_rate": 0.000291752134386253,
      "loss_nats": 1.5370197296142578,
      "step": 1250,
      "unclipped_grad_norm": 1.0654293298721313
    },
    {
      "bits_per_byte": 2.1496304151911416,
      "learning_rate": 0.00029155898526256856,
      "loss_nats": 1.4900102615356445,
      "step": 1260,
      "unclipped_grad_norm": 1.0989246368408203
    },
    {
      "bits_per_byte": 2.0288079594284456,
      "learning_rate": 0.0002913636735950141,
      "loss_nats": 1.4062625169754028,
      "step": 1270,
      "unclipped_grad_norm": 1.0421466827392578
    },
    {
      "bits_per_byte": 2.1597347398946334,
      "learning_rate": 0.0002911662027209272,
      "loss_nats": 1.497014045715332,
      "step": 1280,
      "unclipped_grad_norm": 1.076604962348938
    },
    {
      "bits_per_byte": 2.060473061140004,
      "learning_rate": 0.00029096657601454013,
      "loss_nats": 1.4282110929489136,
      "step": 1290,
      "unclipped_grad_norm": 1.1023766994476318
    },
    {
      "bits_per_byte": 2.1029859685547345,
      "learning_rate": 0.00029076479688692266,
      "loss_nats": 1.4576787948608398,
      "step": 1300,
      "unclipped_grad_norm": 1.0737801790237427
    },
    {
      "bits_per_byte": 2.17370695441603,
      "learning_rate": 0.00029056086878592323,
      "loss_nats": 1.5066988468170166,
      "step": 1310,
      "unclipped_grad_norm": 1.1388311386108398
    },
    {
      "bits_per_byte": 2.1091708086450627,
      "learning_rate": 0.00029035479519611045,
      "loss_nats": 1.461965799331665,
      "step": 1320,
      "unclipped_grad_norm": 1.289936900138855
    },
    {
      "bits_per_byte": 1.9980870424701236,
      "learning_rate": 0.0002901465796387134,
      "loss_nats": 1.3849684000015259,
      "step": 1330,
      "unclipped_grad_norm": 1.0864324569702148
    },
    {
      "bits_per_byte": 2.0165677821838885,
      "learning_rate": 0.00028993622567156154,
      "loss_nats": 1.3977782726287842,
      "step": 1340,
      "unclipped_grad_norm": 0.974994957447052
    },
    {
      "bits_per_byte": 2.204823603453987,
      "learning_rate": 0.0002897237368890237,
      "loss_nats": 1.52826726436615,
      "step": 1350,
      "unclipped_grad_norm": 1.1115007400512695
    },
    {
      "bits_per_byte": 2.0280969831497786,
      "learning_rate": 0.0002895091169219469,
      "loss_nats": 1.4057697057724,
      "step": 1360,
      "unclipped_grad_norm": 0.916259229183197
    },
    {
      "bits_per_byte": 2.124154625118834,
      "learning_rate": 0.00028929236943759425,
      "loss_nats": 1.4723517894744873,
      "step": 1370,
      "unclipped_grad_norm": 1.1937594413757324
    },
    {
      "bits_per_byte": 2.0873195529396886,
      "learning_rate": 0.00028907349813958225,
      "loss_nats": 1.4468196630477905,
      "step": 1380,
      "unclipped_grad_norm": 0.9523230791091919
    },
    {
      "bits_per_byte": 2.1050263707245707,
      "learning_rate": 0.0002888525067678175,
      "loss_nats": 1.4590930938720703,
      "step": 1390,
      "unclipped_grad_norm": 0.8934341669082642
    },
    {
      "bits_per_byte": 2.0970224701360665,
      "learning_rate": 0.00028862939909843273,
      "loss_nats": 1.4535452127456665,
      "step": 1400,
      "unclipped_grad_norm": 0.9132420420646667
    },
    {
      "bits_per_byte": 2.040656188211686,
      "learning_rate": 0.0002884041789437225,
      "loss_nats": 1.4144750833511353,
      "step": 1410,
      "unclipped_grad_norm": 1.0989789962768555
    },
    {
      "bits_per_byte": 2.069019738974631,
      "learning_rate": 0.0002881768501520778,
      "loss_nats": 1.4341351985931396,
      "step": 1420,
      "unclipped_grad_norm": 1.0159250497817993
    },
    {
      "bits_per_byte": 2.0709008852097686,
      "learning_rate": 0.0002879474166079205,
      "loss_nats": 1.435439109802246,
      "step": 1430,
      "unclipped_grad_norm": 0.9772002696990967
    },
    {
      "bits_per_byte": 2.030792811202053,
      "learning_rate": 0.00028771588223163694,
      "loss_nats": 1.4076383113861084,
      "step": 1440,
      "unclipped_grad_norm": 1.0684847831726074
    },
    {
      "bits_per_byte": 2.0937577234747375,
      "learning_rate": 0.00028748225097951054,
      "loss_nats": 1.451282262802124,
      "step": 1450,
      "unclipped_grad_norm": 1.1867483854293823
    },
    {
      "bits_per_byte": 2.0324108239813703,
      "learning_rate": 0.000287246526843655,
      "loss_nats": 1.4087598323822021,
      "step": 1460,
      "unclipped_grad_norm": 1.0155152082443237
    },
    {
      "bits_per_byte": 2.0236820165194795,
      "learning_rate": 0.00028700871385194533,
      "loss_nats": 1.4027094841003418,
      "step": 1470,
      "unclipped_grad_norm": 0.9900296330451965
    },
    {
      "bits_per_byte": 1.9307554906920323,
      "learning_rate": 0.00028676881606794945,
      "loss_nats": 1.338297724723816,
      "step": 1480,
      "unclipped_grad_norm": 0.8600749969482422
    },
    {
      "bits_per_byte": 2.042708801149734,
      "learning_rate": 0.00028652683759085854,
      "loss_nats": 1.4158978462219238,
      "step": 1490,
      "unclipped_grad_norm": 1.3268300294876099
    },
    {
      "bits_per_byte": 1.9310841495378313,
      "learning_rate": 0.0002862827825554171,
      "loss_nats": 1.3385255336761475,
      "step": 1500,
      "unclipped_grad_norm": 0.9175688028335571
    },
    {
      "bits_per_byte": 2.024790788669592,
      "learning_rate": 0.00028603665513185245,
      "loss_nats": 1.4034780263900757,
      "step": 1510,
      "unclipped_grad_norm": 1.1049447059631348
    },
    {
      "bits_per_byte": 2.021844897842963,
      "learning_rate": 0.000285788459525803,
      "loss_nats": 1.4014360904693604,
      "step": 1520,
      "unclipped_grad_norm": 0.9541988968849182
    },
    {
      "bits_per_byte": 1.9848854822072437,
      "learning_rate": 0.00028553819997824707,
      "loss_nats": 1.3758177757263184,
      "step": 1530,
      "unclipped_grad_norm": 1.0897772312164307
    },
    {
      "bits_per_byte": 2.0563004180647706,
      "learning_rate": 0.00028528588076542964,
      "loss_nats": 1.4253188371658325,
      "step": 1540,
      "unclipped_grad_norm": 0.960680365562439
    },
    {
      "bits_per_byte": 2.0173688773716076,
      "learning_rate": 0.00028503150619878996,
      "loss_nats": 1.3983335494995117,
      "step": 1550,
      "unclipped_grad_norm": 1.0041202306747437
    },
    {
      "bits_per_byte": 2.0953523466135375,
      "learning_rate": 0.0002847750806248876,
      "loss_nats": 1.4523875713348389,
      "step": 1560,
      "unclipped_grad_norm": 0.9772875905036926
    },
    {
      "bits_per_byte": 2.057308064416176,
      "learning_rate": 0.000284516608425328,
      "loss_nats": 1.4260172843933105,
      "step": 1570,
      "unclipped_grad_norm": 1.0609372854232788
    },
    {
      "bits_per_byte": 2.019397928686487,
      "learning_rate": 0.000284256094016688,
      "loss_nats": 1.3997399806976318,
      "step": 1580,
      "unclipped_grad_norm": 1.008042573928833
    },
    {
      "bits_per_byte": 2.0214770269527675,
      "learning_rate": 0.0002839935418504399,
      "loss_nats": 1.4011811017990112,
      "step": 1590,
      "unclipped_grad_norm": 0.9353208541870117
    },
    {
      "bits_per_byte": 2.0397778728137297,
      "learning_rate": 0.000283728956412876,
      "loss_nats": 1.4138662815093994,
      "step": 1600,
      "unclipped_grad_norm": 0.9491218328475952
    },
    {
      "bits_per_byte": 2.06155259623947,
      "learning_rate": 0.00028346234222503136,
      "loss_nats": 1.4289593696594238,
      "step": 1610,
      "unclipped_grad_norm": 0.9174792170524597
    },
    {
      "bits_per_byte": 2.0684931280976895,
      "learning_rate": 0.00028319370384260675,
      "loss_nats": 1.4337701797485352,
      "step": 1620,
      "unclipped_grad_norm": 0.9907702803611755
    },
    {
      "bits_per_byte": 1.9372454280049913,
      "learning_rate": 0.00028292304585589105,
      "loss_nats": 1.3427962064743042,
      "step": 1630,
      "unclipped_grad_norm": 0.9023784399032593
    },
    {
      "bits_per_byte": 2.056802951370592,
      "learning_rate": 0.0002826503728896824,
      "loss_nats": 1.4256671667099,
      "step": 1640,
      "unclipped_grad_norm": 0.9690282344818115
    },
    {
      "bits_per_byte": 2.0155706267741857,
      "learning_rate": 0.0002823756896032095,
      "loss_nats": 1.3970870971679688,
      "step": 1650,
      "unclipped_grad_norm": 1.0204124450683594
    },
    {
      "bits_per_byte": 1.9586099728084236,
      "learning_rate": 0.0002820990006900518,
      "loss_nats": 1.35760498046875,
      "step": 1660,
      "unclipped_grad_norm": 0.942615270614624
    },
    {
      "bits_per_byte": 2.120022913914506,
      "learning_rate": 0.00028182031087805947,
      "loss_nats": 1.4694879055023193,
      "step": 1670,
      "unclipped_grad_norm": 0.9669644236564636
    },
    {
      "bits_per_byte": 2.02882687752004,
      "learning_rate": 0.0002815396249292725,
      "loss_nats": 1.4062756299972534,
      "step": 1680,
      "unclipped_grad_norm": 0.8547722101211548
    },
    {
      "bits_per_byte": 1.9310628236891243,
      "learning_rate": 0.0002812569476398391,
      "loss_nats": 1.3385107517242432,
      "step": 1690,
      "unclipped_grad_norm": 0.9439292550086975
    },
    {
      "bits_per_byte": 1.9561478691786718,
      "learning_rate": 0.00028097228383993446,
      "loss_nats": 1.355898380279541,
      "step": 1700,
      "unclipped_grad_norm": 0.8740168213844299
    },
    {
      "bits_per_byte": 2.0381777462301023,
      "learning_rate": 0.0002806856383936774,
      "loss_nats": 1.412757158279419,
      "step": 1710,
      "unclipped_grad_norm": 1.1610065698623657
    },
    {
      "bits_per_byte": 2.022566537045983,
      "learning_rate": 0.00028039701619904774,
      "loss_nats": 1.4019362926483154,
      "step": 1720,
      "unclipped_grad_norm": 0.8489123582839966
    },
    {
      "bits_per_byte": 1.9268317065125973,
      "learning_rate": 0.0002801064221878024,
      "loss_nats": 1.3355779647827148,
      "step": 1730,
      "unclipped_grad_norm": 0.9296807646751404
    },
    {
      "bits_per_byte": 1.9993203300594609,
      "learning_rate": 0.00027981386132539134,
      "loss_nats": 1.3858232498168945,
      "step": 1740,
      "unclipped_grad_norm": 0.9287921786308289
    },
    {
      "bits_per_byte": 2.0917408829280695,
      "learning_rate": 0.0002795193386108724,
      "loss_nats": 1.449884295463562,
      "step": 1750,
      "unclipped_grad_norm": 0.9826183915138245
    },
    {
      "bits_per_byte": 1.9484261921202333,
      "learning_rate": 0.00027922285907682625,
      "loss_nats": 1.35054612159729,
      "step": 1760,
      "unclipped_grad_norm": 0.8883485198020935
    },
    {
      "bits_per_byte": 2.0047687404387946,
      "learning_rate": 0.00027892442778927005,
      "loss_nats": 1.3895998001098633,
      "step": 1770,
      "unclipped_grad_norm": 1.015872836112976
    },
    {
      "bits_per_byte": 1.9445588182502813,
      "learning_rate": 0.00027862404984757116,
      "loss_nats": 1.3478654623031616,
      "step": 1780,
      "unclipped_grad_norm": 0.9133850932121277
    },
    {
      "bits_per_byte": 1.9244168981118326,
      "learning_rate": 0.0002783217303843597,
      "loss_nats": 1.3339041471481323,
      "step": 1790,
      "unclipped_grad_norm": 1.018885850906372
    },
    {
      "bits_per_byte": 2.0032738672374952,
      "learning_rate": 0.00027801747456544134,
      "loss_nats": 1.388563632965088,
      "step": 1800,
      "unclipped_grad_norm": 1.0286787748336792
    },
    {
      "bits_per_byte": 1.9831790703453829,
      "learning_rate": 0.00027771128758970843,
      "loss_nats": 1.3746349811553955,
      "step": 1810,
      "unclipped_grad_norm": 0.8631220459938049
    },
    {
      "bits_per_byte": 2.0513977086366246,
      "learning_rate": 0.00027740317468905165,
      "loss_nats": 1.4219205379486084,
      "step": 1820,
      "unclipped_grad_norm": 0.8914260268211365
    },
    {
      "bits_per_byte": 1.9040037613329734,
      "learning_rate": 0.0002770931411282704,
      "loss_nats": 1.3197548389434814,
      "step": 1830,
      "unclipped_grad_norm": 1.014155626296997
    },
    {
      "bits_per_byte": 1.9870995868544514,
      "learning_rate": 0.0002767811922049827,
      "loss_nats": 1.3773524761199951,
      "step": 1840,
      "unclipped_grad_norm": 0.853826105594635
    },
    {
      "bits_per_byte": 1.8864264464841909,
      "learning_rate": 0.00027646733324953503,
      "loss_nats": 1.3075711727142334,
      "step": 1850,
      "unclipped_grad_norm": 0.9869223833084106
    },
    {
      "bits_per_byte": 2.0198366564288377,
      "learning_rate": 0.00027615156962491097,
      "loss_nats": 1.4000440835952759,
      "step": 1860,
      "unclipped_grad_norm": 0.9129742980003357
    },
    {
      "bits_per_byte": 2.033820393787842,
      "learning_rate": 0.0002758339067266396,
      "loss_nats": 1.4097368717193604,
      "step": 1870,
      "unclipped_grad_norm": 0.9571877717971802
    },
    {
      "bits_per_byte": 1.9046001971661655,
      "learning_rate": 0.0002755143499827035,
      "loss_nats": 1.3201682567596436,
      "step": 1880,
      "unclipped_grad_norm": 0.9010704159736633
    },
    {
      "bits_per_byte": 1.9853064957365558,
      "learning_rate": 0.0002751929048534456,
      "loss_nats": 1.3761096000671387,
      "step": 1890,
      "unclipped_grad_norm": 0.9147026538848877
    },
    {
      "bits_per_byte": 1.8987844318445888,
      "learning_rate": 0.0002748695768314763,
      "loss_nats": 1.3161370754241943,
      "step": 1900,
      "unclipped_grad_norm": 1.0985209941864014
    },
    {
      "bits_per_byte": 1.886329620251755,
      "learning_rate": 0.00027454437144157947,
      "loss_nats": 1.3075040578842163,
      "step": 1910,
      "unclipped_grad_norm": 0.9578983783721924
    },
    {
      "bits_per_byte": 1.954425118964979,
      "learning_rate": 0.00027421729424061787,
      "loss_nats": 1.3547042608261108,
      "step": 1920,
      "unclipped_grad_norm": 0.9071367979049683
    },
    {
      "bits_per_byte": 1.9264953084475098,
      "learning_rate": 0.0002738883508174385,
      "loss_nats": 1.3353447914123535,
      "step": 1930,
      "unclipped_grad_norm": 0.8819456100463867
    },
    {
      "bits_per_byte": 1.874642539212373,
      "learning_rate": 0.0002735575467927768,
      "loss_nats": 1.299403190612793,
      "step": 1940,
      "unclipped_grad_norm": 0.8353087902069092
    },
    {
      "bits_per_byte": 1.9478835868567612,
      "learning_rate": 0.0002732248878191609,
      "loss_nats": 1.3501700162887573,
      "step": 1950,
      "unclipped_grad_norm": 1.0307022333145142
    },
    {
      "bits_per_byte": 1.9152383560179354,
      "learning_rate": 0.0002728903795808148,
      "loss_nats": 1.3275420665740967,
      "step": 1960,
      "unclipped_grad_norm": 0.9104531407356262
    },
    {
      "bits_per_byte": 1.8790318804276935,
      "learning_rate": 0.0002725540277935614,
      "loss_nats": 1.302445650100708,
      "step": 1970,
      "unclipped_grad_norm": 0.900970458984375
    },
    {
      "bits_per_byte": 1.862672890677623,
      "learning_rate": 0.00027221583820472456,
      "loss_nats": 1.2911064624786377,
      "step": 1980,
      "unclipped_grad_norm": 1.0306406021118164
    },
    {
      "bits_per_byte": 1.9492345105792885,
      "learning_rate": 0.0002718758165930314,
      "loss_nats": 1.3511064052581787,
      "step": 1990,
      "unclipped_grad_norm": 0.9717852473258972
    },
    {
      "bits_per_byte": 1.8977989712551449,
      "learning_rate": 0.0002715339687685131,
      "loss_nats": 1.3154540061950684,
      "step": 2000,
      "unclipped_grad_norm": 0.805345892906189
    },
    {
      "bits_per_byte": 1.9588875528069163,
      "learning_rate": 0.00027119030057240576,
      "loss_nats": 1.357797384262085,
      "step": 2010,
      "unclipped_grad_norm": 0.8472830653190613
    },
    {
      "bits_per_byte": 1.946676612613006,
      "learning_rate": 0.0002708448178770506,
      "loss_nats": 1.34933340549469,
      "step": 2020,
      "unclipped_grad_norm": 1.1016662120819092
    },
    {
      "bits_per_byte": 1.9391907237689006,
      "learning_rate": 0.00027049752658579376,
      "loss_nats": 1.344144582748413,
      "step": 2030,
      "unclipped_grad_norm": 0.868341863155365
    },
    {
      "bits_per_byte": 1.9511490214486933,
      "learning_rate": 0.00027014843263288514,
      "loss_nats": 1.352433443069458,
      "step": 2040,
      "unclipped_grad_norm": 0.8853706121444702
    },
    {
      "bits_per_byte": 1.8935334575484455,
      "learning_rate": 0.00026979754198337724,
      "loss_nats": 1.3124973773956299,
      "step": 2050,
      "unclipped_grad_norm": 0.8925303220748901
    },
    {
      "bits_per_byte": 1.8037261810596972,
      "learning_rate": 0.00026944486063302305,
      "loss_nats": 1.2502477169036865,
      "step": 2060,
      "unclipped_grad_norm": 0.8529354333877563
    },
    {
      "bits_per_byte": 1.9080168045082062,
      "learning_rate": 0.0002690903946081738,
      "loss_nats": 1.3225364685058594,
      "step": 2070,
      "unclipped_grad_norm": 0.9431800842285156
    },
    {
      "bits_per_byte": 2.015688606872678,
      "learning_rate": 0.00026873414996567596,
      "loss_nats": 1.3971688747406006,
      "step": 2080,
      "unclipped_grad_norm": 0.9268007874488831
    },
    {
      "bits_per_byte": 1.8786344285215495,
      "learning_rate": 0.00026837613279276736,
      "loss_nats": 1.3021701574325562,
      "step": 2090,
      "unclipped_grad_norm": 0.8720243573188782
    },
    {
      "bits_per_byte": 1.9356492570223336,
      "learning_rate": 0.0002680163492069739,
      "loss_nats": 1.3416898250579834,
      "step": 2100,
      "unclipped_grad_norm": 0.8494227528572083
    },
    {
      "bits_per_byte": 1.8547725236449384,
      "learning_rate": 0.00026765480535600414,
      "loss_nats": 1.2856303453445435,
      "step": 2110,
      "unclipped_grad_norm": 0.9150698781013489
    },
    {
      "bits_per_byte": 1.8966766124756143,
      "learning_rate": 0.0002672915074176452,
      "loss_nats": 1.31467604637146,
      "step": 2120,
      "unclipped_grad_norm": 0.8588051199913025
    },
    {
      "bits_per_byte": 1.924086863404827,
      "learning_rate": 0.00026692646159965624,
      "loss_nats": 1.3336753845214844,
      "step": 2130,
      "unclipped_grad_norm": 0.870884358882904
    },
    {
      "bits_per_byte": 1.8065243387892351,
      "learning_rate": 0.0002665596741396633,
      "loss_nats": 1.2521872520446777,
      "step": 2140,
      "unclipped_grad_norm": 0.7779732346534729
    },
    {
      "bits_per_byte": 1.8523434406841521,
      "learning_rate": 0.000266191151305052,
      "loss_nats": 1.2839466333389282,
      "step": 2150,
      "unclipped_grad_norm": 0.9460588693618774
    },
    {
      "bits_per_byte": 1.9103203401338626,
      "learning_rate": 0.00026582089939286075,
      "loss_nats": 1.3241331577301025,
      "step": 2160,
      "unclipped_grad_norm": 0.8497743606567383
    },
    {
      "bits_per_byte": 1.8798684040414901,
      "learning_rate": 0.0002654489247296732,
      "loss_nats": 1.303025484085083,
      "step": 2170,
      "unclipped_grad_norm": 0.9395503401756287
    },
    {
      "bits_per_byte": 1.8289068489031246,
      "learning_rate": 0.00026507523367151007,
      "loss_nats": 1.2677016258239746,
      "step": 2180,
      "unclipped_grad_norm": 1.0030585527420044
    },
    {
      "bits_per_byte": 1.947407538879173,
      "learning_rate": 0.00026469983260372037,
      "loss_nats": 1.3498400449752808,
      "step": 2190,
      "unclipped_grad_norm": 0.8407843112945557
    },
    {
      "bits_per_byte": 1.9944045499498493,
      "learning_rate": 0.0002643227279408727,
      "loss_nats": 1.3824158906936646,
      "step": 2200,
      "unclipped_grad_norm": 0.9960005879402161
    },
    {
      "bits_per_byte": 2.0001205653339253,
      "learning_rate": 0.0002639439261266453,
      "loss_nats": 1.3863779306411743,
      "step": 2210,
      "unclipped_grad_norm": 0.9367930889129639
    },
    {
      "bits_per_byte": 1.8803774726880442,
      "learning_rate": 0.000263563433633716,
      "loss_nats": 1.3033783435821533,
      "step": 2220,
      "unclipped_grad_norm": 0.9032418131828308
    },
    {
      "bits_per_byte": 1.8995089947526738,
      "learning_rate": 0.00026318125696365156,
      "loss_nats": 1.3166393041610718,
      "step": 2230,
      "unclipped_grad_norm": 0.9414434432983398
    },
    {
      "bits_per_byte": 1.859138991167694,
      "learning_rate": 0.000262797402646797,
      "loss_nats": 1.2886569499969482,
      "step": 2240,
      "unclipped_grad_norm": 0.9088358283042908
    },
    {
      "bits_per_byte": 1.8658401231759127,
      "learning_rate": 0.00026241187724216334,
      "loss_nats": 1.2933018207550049,
      "step": 2250,
      "unclipped_grad_norm": 0.9021689295768738
    },
    {
      "bits_per_byte": 1.8814774737229627,
      "learning_rate": 0.0002620246873373162,
      "loss_nats": 1.3041408061981201,
      "step": 2260,
      "unclipped_grad_norm": 0.8373098969459534
    },
    {
      "bits_per_byte": 1.9445300971475874,
      "learning_rate": 0.00026163583954826264,
      "loss_nats": 1.3478455543518066,
      "step": 2270,
      "unclipped_grad_norm": 0.9310195446014404
    },
    {
      "bits_per_byte": 1.8355594818037997,
      "learning_rate": 0.0002612453405193386,
      "loss_nats": 1.272312879562378,
      "step": 2280,
      "unclipped_grad_norm": 0.8906505107879639
    },
    {
      "bits_per_byte": 1.8521159076370606,
      "learning_rate": 0.000260853196923095,
      "loss_nats": 1.2837889194488525,
      "step": 2290,
      "unclipped_grad_norm": 0.8472009301185608
    },
    {
      "bits_per_byte": 1.9052471958987114,
      "learning_rate": 0.0002604594154601839,
      "loss_nats": 1.3206167221069336,
      "step": 2300,
      "unclipped_grad_norm": 0.8389962911605835
    },
    {
      "bits_per_byte": 1.8722884406873628,
      "learning_rate": 0.0002600640028592441,
      "loss_nats": 1.2977714538574219,
      "step": 2310,
      "unclipped_grad_norm": 0.9144077897071838
    },
    {
      "bits_per_byte": 1.8873241959216946,
      "learning_rate": 0.0002596669658767859,
      "loss_nats": 1.3081934452056885,
      "step": 2320,
      "unclipped_grad_norm": 0.8859429955482483
    },
    {
      "bits_per_byte": 1.8308435455344907,
      "learning_rate": 0.000259268311297076,
      "loss_nats": 1.269044041633606,
      "step": 2330,
      "unclipped_grad_norm": 0.8459444046020508
    },
    {
      "bits_per_byte": 1.8670001461559822,
      "learning_rate": 0.00025886804593202114,
      "loss_nats": 1.294105887413025,
      "step": 2340,
      "unclipped_grad_norm": 0.9549022912979126
    },
    {
      "bits_per_byte": 1.7832834632648895,
      "learning_rate": 0.00025846617662105193,
      "loss_nats": 1.236077904701233,
      "step": 2350,
      "unclipped_grad_norm": 0.8211755752563477
    },
    {
      "bits_per_byte": 1.8681859665336813,
      "learning_rate": 0.00025806271023100637,
      "loss_nats": 1.2949278354644775,
      "step": 2360,
      "unclipped_grad_norm": 0.8336057662963867
    },
    {
      "bits_per_byte": 1.9109924483334344,
      "learning_rate": 0.00025765765365601165,
      "loss_nats": 1.324599027633667,
      "step": 2370,
      "unclipped_grad_norm": 1.0507880449295044
    },
    {
      "bits_per_byte": 1.8140403245972416,
      "learning_rate": 0.00025725101381736707,
      "loss_nats": 1.257396936416626,
      "step": 2380,
      "unclipped_grad_norm": 0.8251904249191284
    },
    {
      "bits_per_byte": 1.8016899064734815,
      "learning_rate": 0.00025684279766342567,
      "loss_nats": 1.2488362789154053,
      "step": 2390,
      "unclipped_grad_norm": 0.7763776183128357
    },
    {
      "bits_per_byte": 1.9132727663012243,
      "learning_rate": 0.00025643301216947504,
      "loss_nats": 1.3261796236038208,
      "step": 2400,
      "unclipped_grad_norm": 0.8198488354682922
    },
    {
      "bits_per_byte": 1.8847781647583217,
      "learning_rate": 0.00025602166433761866,
      "loss_nats": 1.3064286708831787,
      "step": 2410,
      "unclipped_grad_norm": 0.8494725823402405
    },
    {
      "bits_per_byte": 1.9646795845226739,
      "learning_rate": 0.0002556087611966561,
      "loss_nats": 1.3618121147155762,
      "step": 2420,
      "unclipped_grad_norm": 0.9295504689216614
    },
    {
      "bits_per_byte": 1.8742646933284273,
      "learning_rate": 0.00025519430980196277,
      "loss_nats": 1.29914128780365,
      "step": 2430,
      "unclipped_grad_norm": 0.8960878849029541
    },
    {
      "bits_per_byte": 1.802365110360769,
      "learning_rate": 0.0002547783172353695,
      "loss_nats": 1.2493042945861816,
      "step": 2440,
      "unclipped_grad_norm": 0.8283300399780273
    },
    {
      "bits_per_byte": 1.9114973893963676,
      "learning_rate": 0.0002543607906050415,
      "loss_nats": 1.324949026107788,
      "step": 2450,
      "unclipped_grad_norm": 0.8929033875465393
    },
    {
      "bits_per_byte": 1.9097386948086446,
      "learning_rate": 0.0002539417370453568,
      "loss_nats": 1.3237299919128418,
      "step": 2460,
      "unclipped_grad_norm": 0.8874521255493164
    },
    {
      "bits_per_byte": 1.848397298760105,
      "learning_rate": 0.00025352116371678464,
      "loss_nats": 1.2812113761901855,
      "step": 2470,
      "unclipped_grad_norm": 0.8395382761955261
    },
    {
      "bits_per_byte": 1.8498309461376972,
      "learning_rate": 0.0002530990778057627,
      "loss_nats": 1.2822051048278809,
      "step": 2480,
      "unclipped_grad_norm": 0.8023180961608887
    },
    {
      "bits_per_byte": 1.7784197938984894,
      "learning_rate": 0.00025267548652457475,
      "loss_nats": 1.2327066659927368,
      "step": 2490,
      "unclipped_grad_norm": 0.8236642479896545
    },
    {
      "bits_per_byte": 1.8541779796209057,
      "learning_rate": 0.00025225039711122696,
      "loss_nats": 1.2852182388305664,
      "step": 2500,
      "unclipped_grad_norm": 0.8692671656608582
    },
    {
      "bits_per_byte": 1.8219728523856378,
      "learning_rate": 0.00025182381682932464,
      "loss_nats": 1.2628953456878662,
      "step": 2510,
      "unclipped_grad_norm": 0.809110701084137
    },
    {
      "bits_per_byte": 1.8139226884640514,
      "learning_rate": 0.0002513957529679477,
      "loss_nats": 1.2573153972625732,
      "step": 2520,
      "unclipped_grad_norm": 0.8132631778717041
    },
    {
      "bits_per_byte": 1.8159424527157844,
      "learning_rate": 0.00025096621284152656,
      "loss_nats": 1.2587153911590576,
      "step": 2530,
      "unclipped_grad_norm": 0.8300133347511292
    },
    {
      "bits_per_byte": 1.8460886036549224,
      "learning_rate": 0.0002505352037897168,
      "loss_nats": 1.2796111106872559,
      "step": 2540,
      "unclipped_grad_norm": 0.8400688171386719
    },
    {
      "bits_per_byte": 1.8887621428655585,
      "learning_rate": 0.0002501027331772739,
      "loss_nats": 1.3091901540756226,
      "step": 2550,
      "unclipped_grad_norm": 0.872266411781311
    },
    {
      "bits_per_byte": 1.8326880594649941,
      "learning_rate": 0.00024966880839392746,
      "loss_nats": 1.270322561264038,
      "step": 2560,
      "unclipped_grad_norm": 0.8239099383354187
    },
    {
      "bits_per_byte": 1.8127545822993913,
      "learning_rate": 0.0002492334368542548,
      "loss_nats": 1.2565057277679443,
      "step": 2570,
      "unclipped_grad_norm": 0.8143022060394287
    },
    {
      "bits_per_byte": 1.852552915552903,
      "learning_rate": 0.00024879662599755416,
      "loss_nats": 1.284091830253601,
      "step": 2580,
      "unclipped_grad_norm": 0.9730299115180969
    },
    {
      "bits_per_byte": 1.8163588227135232,
      "learning_rate": 0.00024835838328771803,
      "loss_nats": 1.25900399684906,
      "step": 2590,
      "unclipped_grad_norm": 0.8007379770278931
    },
    {
      "bits_per_byte": 1.8542342179477378,
      "learning_rate": 0.0002479187162131051,
      "loss_nats": 1.2852572202682495,
      "step": 2600,
      "unclipped_grad_norm": 0.8254386782646179
    },
    {
      "bits_per_byte": 1.7603803616668383,
      "learning_rate": 0.00024747763228641264,
      "loss_nats": 1.2202026844024658,
      "step": 2610,
      "unclipped_grad_norm": 0.7949259877204895
    },
    {
      "bits_per_byte": 1.8049837182028061,
      "learning_rate": 0.00024703513904454795,
      "loss_nats": 1.2511193752288818,
      "step": 2620,
      "unclipped_grad_norm": 0.865028977394104
    },
    {
      "bits_per_byte": 1.8277927452908351,
      "learning_rate": 0.00024659124404849956,
      "loss_nats": 1.2669293880462646,
      "step": 2630,
      "unclipped_grad_norm": 0.826142430305481
    },
    {
      "bits_per_byte": 1.8414027643495146,
      "learning_rate": 0.00024614595488320833,
      "loss_nats": 1.2763631343841553,
      "step": 2640,
      "unclipped_grad_norm": 0.841355562210083
    },
    {
      "bits_per_byte": 1.7851867952619882,
      "learning_rate": 0.00024569927915743745,
      "loss_nats": 1.2373971939086914,
      "step": 2650,
      "unclipped_grad_norm": 0.788905143737793
    },
    {
      "bits_per_byte": 1.7802099613113203,
      "learning_rate": 0.0002452512245036427,
      "loss_nats": 1.233947515487671,
      "step": 2660,
      "unclipped_grad_norm": 0.7999458909034729
    },
    {
      "bits_per_byte": 1.815510432296817,
      "learning_rate": 0.00024480179857784177,
      "loss_nats": 1.258415937423706,
      "step": 2670,
      "unclipped_grad_norm": 0.8260414004325867
    },
    {
      "bits_per_byte": 1.8079201499836377,
      "learning_rate": 0.00024435100905948387,
      "loss_nats": 1.2531547546386719,
      "step": 2680,
      "unclipped_grad_norm": 0.8360624313354492
    },
    {
      "bits_per_byte": 1.8778747811526886,
      "learning_rate": 0.00024389886365131793,
      "loss_nats": 1.3016436100006104,
      "step": 2690,
      "unclipped_grad_norm": 0.8545008301734924
    },
    {
      "bits_per_byte": 1.8673174541468243,
      "learning_rate": 0.00024344537007926163,
      "loss_nats": 1.294325828552246,
      "step": 2700,
      "unclipped_grad_norm": 0.8579524755477905
    },
    {
      "bits_per_byte": 1.7489412796100003,
      "learning_rate": 0.00024299053609226878,
      "loss_nats": 1.2122737169265747,
      "step": 2710,
      "unclipped_grad_norm": 0.8124077916145325
    },
    {
      "bits_per_byte": 1.8645750187961658,
      "learning_rate": 0.00024253436946219733,
      "loss_nats": 1.2924249172210693,
      "step": 2720,
      "unclipped_grad_norm": 0.9173712730407715
    },
    {
      "bits_per_byte": 1.7690415799469403,
      "learning_rate": 0.0002420768779836764,
      "loss_nats": 1.2262061834335327,
      "step": 2730,
      "unclipped_grad_norm": 0.7701802849769592
    },
    {
      "bits_per_byte": 1.7960051919318623,
      "learning_rate": 0.00024161806947397305,
      "loss_nats": 1.2448959350585938,
      "step": 2740,
      "unclipped_grad_norm": 0.841517448425293
    },
    {
      "bits_per_byte": 1.8115809726899035,
      "learning_rate": 0.00024115795177285886,
      "loss_nats": 1.2556922435760498,
      "step": 2750,
      "unclipped_grad_norm": 0.8873427510261536
    },
    {
      "bits_per_byte": 1.9016966140716487,
      "learning_rate": 0.0002406965327424758,
      "loss_nats": 1.3181556463241577,
      "step": 2760,
      "unclipped_grad_norm": 0.8982214331626892
    },
    {
      "bits_per_byte": 1.8168608400713917,
      "learning_rate": 0.00024023382026720197,
      "loss_nats": 1.2593519687652588,
      "step": 2770,
      "unclipped_grad_norm": 0.8194716572761536
    },
    {
      "bits_per_byte": 1.8166995203448824,
      "learning_rate": 0.00023976982225351698,
      "loss_nats": 1.2592401504516602,
      "step": 2780,
      "unclipped_grad_norm": 0.8212807774543762
    },
    {
      "bits_per_byte": 1.8228974311166763,
      "learning_rate": 0.0002393045466298666,
      "loss_nats": 1.2635362148284912,
      "step": 2790,
      "unclipped_grad_norm": 0.9329227209091187
    },
    {
      "bits_per_byte": 1.757803201643659,
      "learning_rate": 0.00023883800134652763,
      "loss_nats": 1.2184163331985474,
      "step": 2800,
      "unclipped_grad_norm": 0.8613669276237488
    },
    {
      "bits_per_byte": 1.8264712306016053,
      "learning_rate": 0.0002383701943754715,
      "loss_nats": 1.2660133838653564,
      "step": 2810,
      "unclipped_grad_norm": 0.8505002856254578
    },
    {
      "bits_per_byte": 1.8298420905585164,
      "learning_rate": 0.00023790113371022884,
      "loss_nats": 1.2683498859405518,
      "step": 2820,
      "unclipped_grad_norm": 0.8405188322067261
    },
    {
      "bits_per_byte": 1.7699738978972677,
      "learning_rate": 0.0002374308273657521,
      "loss_nats": 1.2268524169921875,
      "step": 2830,
      "unclipped_grad_norm": 0.8156670928001404
    },
    {
      "bits_per_byte": 1.8402193517289274,
      "learning_rate": 0.00023695928337827911,
      "loss_nats": 1.2755428552627563,
      "step": 2840,
      "unclipped_grad_norm": 0.9026433825492859
    },
    {
      "bits_per_byte": 1.7471456087523418,
      "learning_rate": 0.00023648650980519564,
      "loss_nats": 1.211029052734375,
      "step": 2850,
      "unclipped_grad_norm": 0.7569094300270081
    },
    {
      "bits_per_byte": 1.7464477031551406,
      "learning_rate": 0.00023601251472489757,
      "loss_nats": 1.210545301437378,
      "step": 2860,
      "unclipped_grad_norm": 0.8251471519470215
    },
    {
      "bits_per_byte": 1.8558431156465593,
      "learning_rate": 0.00023553730623665305,
      "loss_nats": 1.286372423171997,
      "step": 2870,
      "unclipped_grad_norm": 0.8680534958839417
    },
    {
      "bits_per_byte": 1.7726970711910293,
      "learning_rate": 0.00023506089246046403,
      "loss_nats": 1.2287399768829346,
      "step": 2880,
      "unclipped_grad_norm": 0.8200762271881104
    },
    {
      "bits_per_byte": 1.8185927053655802,
      "learning_rate": 0.00023458328153692736,
      "loss_nats": 1.2605524063110352,
      "step": 2890,
      "unclipped_grad_norm": 0.856894314289093
    },
    {
      "bits_per_byte": 1.732919547838272,
      "learning_rate": 0.0002341044816270961,
      "loss_nats": 1.2011682987213135,
      "step": 2900,
      "unclipped_grad_norm": 0.8436375856399536
    },
    {
      "bits_per_byte": 1.7766230911449254,
      "learning_rate": 0.00023362450091233953,
      "loss_nats": 1.2314612865447998,
      "step": 2910,
      "unclipped_grad_norm": 0.8930034041404724
    },
    {
      "bits_per_byte": 1.8530752268635733,
      "learning_rate": 0.00023314334759420388,
      "loss_nats": 1.2844538688659668,
      "step": 2920,
      "unclipped_grad_norm": 0.8661487698554993
    },
    {
      "bits_per_byte": 1.7993667648256269,
      "learning_rate": 0.0002326610298942717,
      "loss_nats": 1.2472259998321533,
      "step": 2930,
      "unclipped_grad_norm": 0.9391220808029175
    },
    {
      "bits_per_byte": 1.8410732455904613,
      "learning_rate": 0.00023217755605402175,
      "loss_nats": 1.276134729385376,
      "step": 2940,
      "unclipped_grad_norm": 0.8410946726799011
    },
    {
      "bits_per_byte": 1.7968802396594523,
      "learning_rate": 0.00023169293433468802,
      "loss_nats": 1.2455024719238281,
      "step": 2950,
      "unclipped_grad_norm": 0.8708962798118591
    },
    {
      "bits_per_byte": 1.7886298879322615,
      "learning_rate": 0.00023120717301711857,
      "loss_nats": 1.239783763885498,
      "step": 2960,
      "unclipped_grad_norm": 0.8532378673553467
    },
    {
      "bits_per_byte": 1.841481188438308,
      "learning_rate": 0.00023072028040163408,
      "loss_nats": 1.2764174938201904,
      "step": 2970,
      "unclipped_grad_norm": 0.9022923111915588
    },
    {
      "bits_per_byte": 1.775313615241256,
      "learning_rate": 0.00023023226480788593,
      "loss_nats": 1.2305536270141602,
      "step": 2980,
      "unclipped_grad_norm": 0.7898789644241333
    },
    {
      "bits_per_byte": 1.9084053133165053,
      "learning_rate": 0.00022974313457471416,
      "loss_nats": 1.3228057622909546,
      "step": 2990,
      "unclipped_grad_norm": 0.835300624370575
    },
    {
      "bits_per_byte": 1.8358795415170555,
      "learning_rate": 0.0002292528980600049,
      "loss_nats": 1.272534728050232,
      "step": 3000,
      "unclipped_grad_norm": 0.8411970138549805
    },
    {
      "bits_per_byte": 1.7909437425169699,
      "learning_rate": 0.00022876156364054758,
      "loss_nats": 1.2413876056671143,
      "step": 3010,
      "unclipped_grad_norm": 0.8151383399963379
    },
    {
      "bits_per_byte": 1.7661882157864754,
      "learning_rate": 0.0002282691397118918,
      "loss_nats": 1.2242283821105957,
      "step": 3020,
      "unclipped_grad_norm": 0.8463819622993469
    },
    {
      "bits_per_byte": 1.805475244618972,
      "learning_rate": 0.00022777563468820376,
      "loss_nats": 1.251460075378418,
      "step": 3030,
      "unclipped_grad_norm": 0.9359927177429199
    },
    {
      "bits_per_byte": 1.7607282825695338,
      "learning_rate": 0.00022728105700212282,
      "loss_nats": 1.220443844795227,
      "step": 3040,
      "unclipped_grad_norm": 0.7992194890975952
    },
    {
      "bits_per_byte": 1.7895066554863601,
      "learning_rate": 0.00022678541510461698,
      "loss_nats": 1.240391492843628,
      "step": 3050,
      "unclipped_grad_norm": 0.8191037178039551
    },
    {
      "bits_per_byte": 1.7907009030139516,
      "learning_rate": 0.00022628871746483876,
      "loss_nats": 1.2412192821502686,
      "step": 3060,
      "unclipped_grad_norm": 0.8547786474227905
    },
    {
      "bits_per_byte": 1.763532287709201,
      "learning_rate": 0.0002257909725699804,
      "loss_nats": 1.222387433052063,
      "step": 3070,
      "unclipped_grad_norm": 0.8228699564933777
    },
    {
      "bits_per_byte": 1.772629653991891,
      "learning_rate": 0.00022529218892512886,
      "loss_nats": 1.2286932468414307,
      "step": 3080,
      "unclipped_grad_norm": 0.857837438583374
    },
    {
      "bits_per_byte": 1.7447516102523315,
      "learning_rate": 0.00022479237505312034,
      "loss_nats": 1.2093696594238281,
      "step": 3090,
      "unclipped_grad_norm": 0.8343800902366638
    },
    {
      "bits_per_byte": 1.8342075261853672,
      "learning_rate": 0.00022429153949439496,
      "loss_nats": 1.2713757753372192,
      "step": 3100,
      "unclipped_grad_norm": 0.847975492477417
    },
    {
      "bits_per_byte": 1.8219030274293875,
      "learning_rate": 0.00022378969080685053,
      "loss_nats": 1.2628469467163086,
      "step": 3110,
      "unclipped_grad_norm": 0.872373104095459
    },
    {
      "bits_per_byte": 1.7073493392906405,
      "learning_rate": 0.0002232868375656965,
      "loss_nats": 1.1834443807601929,
      "step": 3120,
      "unclipped_grad_norm": 0.8234026432037354
    },
    {
      "bits_per_byte": 1.8002469720327425,
      "learning_rate": 0.00022278298836330733,
      "loss_nats": 1.2478361129760742,
      "step": 3130,
      "unclipped_grad_norm": 0.8703857660293579
    },
    {
      "bits_per_byte": 1.7476684360109647,
      "learning_rate": 0.00022227815180907572,
      "loss_nats": 1.2113914489746094,
      "step": 3140,
      "unclipped_grad_norm": 0.7984506487846375
    },
    {
      "bits_per_byte": 1.731277285505183,
      "learning_rate": 0.00022177233652926546,
      "loss_nats": 1.200029969215393,
      "step": 3150,
      "unclipped_grad_norm": 0.7827578783035278
    },
    {
      "bits_per_byte": 1.7956350852672056,
      "learning_rate": 0.00022126555116686416,
      "loss_nats": 1.2446393966674805,
      "step": 3160,
      "unclipped_grad_norm": 0.8662806153297424
    },
    {
      "bits_per_byte": 1.8529331691939606,
      "learning_rate": 0.00022075780438143532,
      "loss_nats": 1.2843554019927979,
      "step": 3170,
      "unclipped_grad_norm": 0.8483284711837769
    },
    {
      "bits_per_byte": 1.7207951149177458,
      "learning_rate": 0.00022024910484897065,
      "loss_nats": 1.1927642822265625,
      "step": 3180,
      "unclipped_grad_norm": 0.8697596788406372
    },
    {
      "bits_per_byte": 1.8014959100433083,
      "learning_rate": 0.00021973946126174158,
      "loss_nats": 1.248701810836792,
      "step": 3190,
      "unclipped_grad_norm": 0.9436899423599243
    },
    {
      "bits_per_byte": 1.7115792525886113,
      "learning_rate": 0.00021922888232815086,
      "loss_nats": 1.1863763332366943,
      "step": 3200,
      "unclipped_grad_norm": 0.7923902273178101
    },
    {
      "bits_per_byte": 1.7455042063325072,
      "learning_rate": 0.0002187173767725837,
      "loss_nats": 1.2098913192749023,
      "step": 3210,
      "unclipped_grad_norm": 0.8492945432662964
    },
    {
      "bits_per_byte": 1.7423042971305538,
      "learning_rate": 0.0002182049533352588,
      "loss_nats": 1.2076733112335205,
      "step": 3220,
      "unclipped_grad_norm": 0.8532925844192505
    },
    {
      "bits_per_byte": 1.7197866086530862,
      "learning_rate": 0.00021769162077207884,
      "loss_nats": 1.1920652389526367,
      "step": 3230,
      "unclipped_grad_norm": 0.8590078949928284
    },
    {
      "bits_per_byte": 1.772480717016244,
      "learning_rate": 0.00021717738785448102,
      "loss_nats": 1.2285900115966797,
      "step": 3240,
      "unclipped_grad_norm": 0.8679581880569458
    },
    {
      "bits_per_byte": 1.7710855937524448,
      "learning_rate": 0.00021666226336928708,
      "loss_nats": 1.2276229858398438,
      "step": 3250,
      "unclipped_grad_norm": 0.8697354793548584
    },
    {
      "bits_per_byte": 1.8824907955018464,
      "learning_rate": 0.0002161462561185532,
      "loss_nats": 1.3048431873321533,
      "step": 3260,
      "unclipped_grad_norm": 0.882398247718811
    },
    {
      "bits_per_byte": 1.726842196904733,
      "learning_rate": 0.00021562937491941957,
      "loss_nats": 1.1969558000564575,
      "step": 3270,
      "unclipped_grad_norm": 0.8395750522613525
    },
    {
      "bits_per_byte": 1.6700450984399413,
      "learning_rate": 0.00021511162860395975,
      "loss_nats": 1.1575870513916016,
      "step": 3280,
      "unclipped_grad_norm": 0.7638692259788513
    },
    {
      "bits_per_byte": 1.7148034113443364,
      "learning_rate": 0.0002145930260190298,
      "loss_nats": 1.1886111497879028,
      "step": 3290,
      "unclipped_grad_norm": 0.8540530800819397
    },
    {
      "bits_per_byte": 1.79605059535169,
      "learning_rate": 0.00021407357602611698,
      "loss_nats": 1.2449274063110352,
      "step": 3300,
      "unclipped_grad_norm": 0.8206289410591125
    },
    {
      "bits_per_byte": 1.7794766272880411,
      "learning_rate": 0.0002135532875011885,
      "loss_nats": 1.2334392070770264,
      "step": 3310,
      "unclipped_grad_norm": 0.8657307028770447
    },
    {
      "bits_per_byte": 1.796555880379925,
      "learning_rate": 0.0002130321693345397,
      "loss_nats": 1.2452776432037354,
      "step": 3320,
      "unclipped_grad_norm": 0.8429055213928223
    },
    {
      "bits_per_byte": 1.7430792509553417,
      "learning_rate": 0.00021251023043064224,
      "loss_nats": 1.2082104682922363,
      "step": 3330,
      "unclipped_grad_norm": 0.8332963585853577
    },
    {
      "bits_per_byte": 1.747674627386396,
      "learning_rate": 0.00021198747970799188,
      "loss_nats": 1.2113957405090332,
      "step": 3340,
      "unclipped_grad_norm": 0.8822024464607239
    },
    {
      "bits_per_byte": 1.7362042444871009,
      "learning_rate": 0.00021146392609895618,
      "loss_nats": 1.2034450769424438,
      "step": 3350,
      "unclipped_grad_norm": 0.7802554368972778
    },
    {
      "bits_per_byte": 1.7416999500954218,
      "learning_rate": 0.00021093957854962178,
      "loss_nats": 1.207254409790039,
      "step": 3360,
      "unclipped_grad_norm": 0.8200514316558838
    },
    {
      "bits_per_byte": 1.676567712456564,
      "learning_rate": 0.00021041444601964157,
      "loss_nats": 1.1621081829071045,
      "step": 3370,
      "unclipped_grad_norm": 0.8079729676246643
    },
    {
      "bits_per_byte": 1.7870502272840867,
      "learning_rate": 0.00020988853748208164,
      "loss_nats": 1.2386888265609741,
      "step": 3380,
      "unclipped_grad_norm": 0.9383370876312256
    },
    {
      "bits_per_byte": 1.7544094679941835,
      "learning_rate": 0.00020936186192326782,
      "loss_nats": 1.2160639762878418,
      "step": 3390,
      "unclipped_grad_norm": 0.848736047744751
    },
    {
      "bits_per_byte": 1.758961848762522,
      "learning_rate": 0.00020883442834263225,
      "loss_nats": 1.219219446182251,
      "step": 3400,
      "unclipped_grad_norm": 0.8597305417060852
    },
    {
      "bits_per_byte": 1.7916406162182659,
      "learning_rate": 0.0002083062457525596,
      "loss_nats": 1.241870641708374,
      "step": 3410,
      "unclipped_grad_norm": 0.9396098256111145
    },
    {
      "bits_per_byte": 1.7669203459311984,
      "learning_rate": 0.00020777732317823306,
      "loss_nats": 1.2247358560562134,
      "step": 3420,
      "unclipped_grad_norm": 0.8953562378883362
    },
    {
      "bits_per_byte": 1.7096045477914046,
      "learning_rate": 0.00020724766965748005,
      "loss_nats": 1.1850075721740723,
      "step": 3430,
      "unclipped_grad_norm": 0.8465718626976013
    },
    {
      "bits_per_byte": 1.713755865017931,
      "learning_rate": 0.00020671729424061788,
      "loss_nats": 1.187885046005249,
      "step": 3440,
      "unclipped_grad_norm": 0.8348708152770996
    },
    {
      "bits_per_byte": 1.7182920794170884,
      "learning_rate": 0.00020618620599029906,
      "loss_nats": 1.1910293102264404,
      "step": 3450,
      "unclipped_grad_norm": 0.9096519351005554
    },
    {
      "bits_per_byte": 1.707162050183851,
      "learning_rate": 0.00020565441398135649,
      "loss_nats": 1.183314561843872,
      "step": 3460,
      "unclipped_grad_norm": 0.8655852675437927
    },
    {
      "bits_per_byte": 1.7530255236026904,
      "learning_rate": 0.00020512192730064827,
      "loss_nats": 1.2151046991348267,
      "step": 3470,
      "unclipped_grad_norm": 0.8270760774612427
    },
    {
      "bits_per_byte": 1.8250635526042933,
      "learning_rate": 0.00020458875504690273,
      "loss_nats": 1.2650376558303833,
      "step": 3480,
      "unclipped_grad_norm": 0.9349867701530457
    },
    {
      "bits_per_byte": 1.751566594775421,
      "learning_rate": 0.00020405490633056245,
      "loss_nats": 1.2140934467315674,
      "step": 3490,
      "unclipped_grad_norm": 0.7542157769203186
    },
    {
      "bits_per_byte": 1.6978185767277763,
      "learning_rate": 0.00020352039027362917,
      "loss_nats": 1.1768381595611572,
      "step": 3500,
      "unclipped_grad_norm": 0.8014698028564453
    },
    {
      "bits_per_byte": 1.7200215369541647,
      "learning_rate": 0.0002029852160095075,
      "loss_nats": 1.192228078842163,
      "step": 3510,
      "unclipped_grad_norm": 0.8212472796440125
    },
    {
      "bits_per_byte": 1.7550949908405227,
      "learning_rate": 0.000202449392682849,
      "loss_nats": 1.2165391445159912,
      "step": 3520,
      "unclipped_grad_norm": 0.8387999534606934
    },
    {
      "bits_per_byte": 1.7221352037332687,
      "learning_rate": 0.0002019129294493959,
      "loss_nats": 1.1936931610107422,
      "step": 3530,
      "unclipped_grad_norm": 0.8480421900749207
    },
    {
      "bits_per_byte": 1.728760147427155,
      "learning_rate": 0.00020137583547582478,
      "loss_nats": 1.1982852220535278,
      "step": 3540,
      "unclipped_grad_norm": 0.8651905059814453
    },
    {
      "bits_per_byte": 1.782876724295599,
      "learning_rate": 0.00020083811993958973,
      "loss_nats": 1.2357959747314453,
      "step": 3550,
      "unclipped_grad_norm": 0.9422439932823181
    },
    {
      "bits_per_byte": 1.717021471592514,
      "learning_rate": 0.00020029979202876563,
      "loss_nats": 1.1901485919952393,
      "step": 3560,
      "unclipped_grad_norm": 0.8451233506202698
    },
    {
      "bits_per_byte": 1.7490260670568758,
      "learning_rate": 0.00019976086094189118,
      "loss_nats": 1.2123324871063232,
      "step": 3570,
      "unclipped_grad_norm": 0.869125247001648
    },
    {
      "bits_per_byte": 1.759480720420175,
      "learning_rate": 0.00019922133588781175,
      "loss_nats": 1.2195791006088257,
      "step": 3580,
      "unclipped_grad_norm": 0.8907663226127625
    },
    {
      "bits_per_byte": 1.7336618249593958,
      "learning_rate": 0.0001986812260855219,
      "loss_nats": 1.2016828060150146,
      "step": 3590,
      "unclipped_grad_norm": 0.8389977216720581
    },
    {
      "bits_per_byte": 1.7850502410372031,
      "learning_rate": 0.00019814054076400787,
      "loss_nats": 1.237302541732788,
      "step": 3600,
      "unclipped_grad_norm": 0.8462889790534973
    },
    {
      "bits_per_byte": 1.760744448938715,
      "learning_rate": 0.00019759928916209004,
      "loss_nats": 1.2204550504684448,
      "step": 3610,
      "unclipped_grad_norm": 0.8433146476745605
    },
    {
      "bits_per_byte": 1.7017459725428794,
      "learning_rate": 0.00019705748052826493,
      "loss_nats": 1.1795604228973389,
      "step": 3620,
      "unclipped_grad_norm": 0.7866216897964478
    },
    {
      "bits_per_byte": 1.7213356563894076,
      "learning_rate": 0.00019651512412054723,
      "loss_nats": 1.1931389570236206,
      "step": 3630,
      "unclipped_grad_norm": 0.881050169467926
    },
    {
      "bits_per_byte": 1.6603886165592963,
      "learning_rate": 0.00019597222920631144,
      "loss_nats": 1.1508936882019043,
      "step": 3640,
      "unclipped_grad_norm": 0.8100939989089966
    },
    {
      "bits_per_byte": 1.7286516263744605,
      "learning_rate": 0.0001954288050621338,
      "loss_nats": 1.1982100009918213,
      "step": 3650,
      "unclipped_grad_norm": 0.8479801416397095
    },
    {
      "bits_per_byte": 1.7281064413712255,
      "learning_rate": 0.00019488486097363363,
      "loss_nats": 1.1978321075439453,
      "step": 3660,
      "unclipped_grad_norm": 0.8173186182975769
    },
    {
      "bits_per_byte": 1.7549983365907376,
      "learning_rate": 0.00019434040623531452,
      "loss_nats": 1.2164721488952637,
      "step": 3670,
      "unclipped_grad_norm": 0.8759186267852783
    },
    {
      "bits_per_byte": 1.6966509865110686,
      "learning_rate": 0.0001937954501504058,
      "loss_nats": 1.176028847694397,
      "step": 3680,
      "unclipped_grad_norm": 0.8159555792808533
    },
    {
      "bits_per_byte": 1.7680449404851903,
      "learning_rate": 0.00019325000203070333,
      "loss_nats": 1.225515365600586,
      "step": 3690,
      "unclipped_grad_norm": 0.8133695721626282
    },
    {
      "bits_per_byte": 1.6566674279425777,
      "learning_rate": 0.0001927040711964106,
      "loss_nats": 1.148314356803894,
      "step": 3700,
      "unclipped_grad_norm": 0.8174366354942322
    },
    {
      "bits_per_byte": 1.7250475579429796,
      "learning_rate": 0.00019215766697597923,
      "loss_nats": 1.1957118511199951,
      "step": 3710,
      "unclipped_grad_norm": 0.8113536238670349
    },
    {
      "bits_per_byte": 1.7713626578029849,
      "learning_rate": 0.00019161079870594978,
      "loss_nats": 1.22781503200531,
      "step": 3720,
      "unclipped_grad_norm": 0.8423733115196228
    },
    {
      "bits_per_byte": 1.7265372716647531,
      "learning_rate": 0.0001910634757307922,
      "loss_nats": 1.196744441986084,
      "step": 3730,
      "unclipped_grad_norm": 0.8644699454307556
    },
    {
      "bits_per_byte": 1.7046167069510814,
      "learning_rate": 0.0001905157074027459,
      "loss_nats": 1.1815502643585205,
      "step": 3740,
      "unclipped_grad_norm": 0.8081701993942261
    },
    {
      "bits_per_byte": 1.72418454900095,
      "learning_rate": 0.00018996750308166037,
      "loss_nats": 1.1951136589050293,
      "step": 3750,
      "unclipped_grad_norm": 0.7984616756439209
    },
    {
      "bits_per_byte": 1.7538620472164872,
      "learning_rate": 0.00018941887213483482,
      "loss_nats": 1.2156845331192017,
      "step": 3760,
      "unclipped_grad_norm": 0.8375996351242065
    },
    {
      "bits_per_byte": 1.7548705534811466,
      "learning_rate": 0.00018886982393685845,
      "loss_nats": 1.2163835763931274,
      "step": 3770,
      "unclipped_grad_norm": 0.817222535610199
    },
    {
      "bits_per_byte": 1.8002908276087126,
      "learning_rate": 0.00018832036786945008,
      "loss_nats": 1.2478665113449097,
      "step": 3780,
      "unclipped_grad_norm": 0.8592872023582458
    },
    {
      "bits_per_byte": 1.6997967211780003,
      "learning_rate": 0.0001877705133212979,
      "loss_nats": 1.1782093048095703,
      "step": 3790,
      "unclipped_grad_norm": 0.8284241557121277
    },
    {
      "bits_per_byte": 1.7288383995332977,
      "learning_rate": 0.00018722026968789906,
      "loss_nats": 1.1983394622802734,
      "step": 3800,
      "unclipped_grad_norm": 0.8618725538253784
    },
    {
      "bits_per_byte": 1.6712219757197955,
      "learning_rate": 0.00018666964637139907,
      "loss_nats": 1.1584028005599976,
      "step": 3810,
      "unclipped_grad_norm": 0.9065259099006653
    },
    {
      "bits_per_byte": 1.6792523616365327,
      "learning_rate": 0.00018611865278043115,
      "loss_nats": 1.1639690399169922,
      "step": 3820,
      "unclipped_grad_norm": 0.8587555885314941
    },
    {
      "bits_per_byte": 1.7510613097471863,
      "learning_rate": 0.00018556729832995555,
      "loss_nats": 1.2137432098388672,
      "step": 3830,
      "unclipped_grad_norm": 0.8374478220939636
    },
    {
      "bits_per_byte": 1.6819891215597125,
      "learning_rate": 0.00018501559244109864,
      "loss_nats": 1.1658660173416138,
      "step": 3840,
      "unclipped_grad_norm": 0.8430510759353638
    },
    {
      "bits_per_byte": 1.6493425148596799,
      "learning_rate": 0.00018446354454099178,
      "loss_nats": 1.1432371139526367,
      "step": 3850,
      "unclipped_grad_norm": 0.7972609400749207
    },
    {
      "bits_per_byte": 1.629205754200757,
      "learning_rate": 0.00018391116406261048,
      "loss_nats": 1.129279375076294,
      "step": 3860,
      "unclipped_grad_norm": 0.801466703414917
    },
    {
      "bits_per_byte": 1.6121130864621058,
      "learning_rate": 0.00018335846044461305,
      "loss_nats": 1.117431640625,
      "step": 3870,
      "unclipped_grad_norm": 0.7957305312156677
    },
    {
      "bits_per_byte": 1.7045224604584086,
      "learning_rate": 0.0001828054431311793,
      "loss_nats": 1.1814849376678467,
      "step": 3880,
      "unclipped_grad_norm": 0.8071211576461792
    },
    {
      "bits_per_byte": 1.6850354502544453,
      "learning_rate": 0.00018225212157184934,
      "loss_nats": 1.1679775714874268,
      "step": 3890,
      "unclipped_grad_norm": 0.8125332593917847
    },
    {
      "bits_per_byte": 1.6717119542921035,
      "learning_rate": 0.00018169850522136192,
      "loss_nats": 1.1587424278259277,
      "step": 3900,
      "unclipped_grad_norm": 0.8193724751472473
    },
    {
      "bits_per_byte": 1.7034513525088348,
      "learning_rate": 0.000181144603539493,
      "loss_nats": 1.1807425022125244,
      "step": 3910,
      "unclipped_grad_norm": 0.8426622748374939
    },
    {
      "bits_per_byte": 1.7239871129177595,
      "learning_rate": 0.00018059042599089412,
      "loss_nats": 1.194976806640625,
      "step": 3920,
      "unclipped_grad_norm": 0.8514105081558228
    },
    {
      "bits_per_byte": 1.7258452134776812,
      "learning_rate": 0.00018003598204493048,
      "loss_nats": 1.1962647438049316,
      "step": 3930,
      "unclipped_grad_norm": 0.8463394045829773
    },
    {
      "bits_per_byte": 1.6514083704618439,
      "learning_rate": 0.00017948128117551936,
      "loss_nats": 1.1446690559387207,
      "step": 3940,
      "unclipped_grad_norm": 0.7947182059288025
    },
    {
      "bits_per_byte": 1.7619474675815003,
      "learning_rate": 0.00017892633286096816,
      "loss_nats": 1.2212889194488525,
      "step": 3950,
      "unclipped_grad_norm": 0.8610087633132935
    },
    {
      "bits_per_byte": 1.7314143556779207,
      "learning_rate": 0.00017837114658381247,
      "loss_nats": 1.200124979019165,
      "step": 3960,
      "unclipped_grad_norm": 0.8606525659561157
    },
    {
      "bits_per_byte": 1.6963023776777697,
      "learning_rate": 0.00017781573183065386,
      "loss_nats": 1.1757872104644775,
      "step": 3970,
      "unclipped_grad_norm": 0.8503133654594421
    },
    {
      "bits_per_byte": 1.6634555831721327,
      "learning_rate": 0.0001772600980919981,
      "loss_nats": 1.1530195474624634,
      "step": 3980,
      "unclipped_grad_norm": 0.8032745718955994
    },
    {
      "bits_per_byte": 1.7136887917840944,
      "learning_rate": 0.0001767042548620927,
      "loss_nats": 1.1878385543823242,
      "step": 3990,
      "unclipped_grad_norm": 0.8245929479598999
    },
    {
      "bits_per_byte": 1.6397087346889487,
      "learning_rate": 0.00017614821163876486,
      "loss_nats": 1.1365594863891602,
      "step": 4000,
      "unclipped_grad_norm": 0.8357985615730286
    },
    {
      "bits_per_byte": 1.7015423450842577,
      "learning_rate": 0.00017559197792325906,
      "loss_nats": 1.1794192790985107,
      "step": 4010,
      "unclipped_grad_norm": 0.8669785261154175
    },
    {
      "bits_per_byte": 1.7224059044257267,
      "learning_rate": 0.0001750355632200748,
      "loss_nats": 1.1938807964324951,
      "step": 4020,
      "unclipped_grad_norm": 0.8723159432411194
    },
    {
      "bits_per_byte": 1.6827126525718923,
      "learning_rate": 0.00017447897703680418,
      "loss_nats": 1.166367530822754,
      "step": 4030,
      "unclipped_grad_norm": 0.8873636722564697
    },
    {
      "bits_per_byte": 1.663583022316422,
      "learning_rate": 0.0001739222288839694,
      "loss_nats": 1.1531078815460205,
      "step": 4040,
      "unclipped_grad_norm": 0.867052435874939
    },
    {
      "bits_per_byte": 1.6966081628310037,
      "learning_rate": 0.00017336532827486026,
      "loss_nats": 1.1759991645812988,
      "step": 4050,
      "unclipped_grad_norm": 0.8396835923194885
    },
    {
      "bits_per_byte": 1.6707194424139744,
      "learning_rate": 0.00017280828472537163,
      "loss_nats": 1.1580544710159302,
      "step": 4060,
      "unclipped_grad_norm": 0.7937865853309631
    },
    {
      "bits_per_byte": 1.8029612022286594,
      "learning_rate": 0.0001722511077538408,
      "loss_nats": 1.2497174739837646,
      "step": 4070,
      "unclipped_grad_norm": 0.8503637909889221
    },
    {
      "bits_per_byte": 1.6233989319770252,
      "learning_rate": 0.00017169380688088497,
      "loss_nats": 1.1252543926239014,
      "step": 4080,
      "unclipped_grad_norm": 0.8034883141517639
    },
    {
      "bits_per_byte": 1.6627509702515477,
      "learning_rate": 0.0001711363916292384,
      "loss_nats": 1.1525311470031738,
      "step": 4090,
      "unclipped_grad_norm": 0.8565743565559387
    },
    {
      "bits_per_byte": 1.7695604516045933,
      "learning_rate": 0.00017057887152358974,
      "loss_nats": 1.2265658378601074,
      "step": 4100,
      "unclipped_grad_norm": 0.9050058722496033
    },
    {
      "bits_per_byte": 1.6734001359929729,
      "learning_rate": 0.0001700212560904194,
      "loss_nats": 1.1599125862121582,
      "step": 4110,
      "unclipped_grad_norm": 0.8419733643531799
    },
    {
      "bits_per_byte": 1.686295739119968,
      "learning_rate": 0.00016946355485783656,
      "loss_nats": 1.1688511371612549,
      "step": 4120,
      "unclipped_grad_norm": 0.8891176581382751
    },
    {
      "bits_per_byte": 1.7589159293947416,
      "learning_rate": 0.0001689057773554166,
      "loss_nats": 1.219187617301941,
      "step": 4130,
      "unclipped_grad_norm": 0.8800218105316162
    },
    {
      "bits_per_byte": 1.728608458729094,
      "learning_rate": 0.00016834793311403806,
      "loss_nats": 1.198180079460144,
      "step": 4140,
      "unclipped_grad_norm": 0.9665619730949402
    },
    {
      "bits_per_byte": 1.6689839654841176,
      "learning_rate": 0.00016779003166571981,
      "loss_nats": 1.1568515300750732,
      "step": 4150,
      "unclipped_grad_norm": 0.8343537449836731
    },
    {
      "bits_per_byte": 1.6883550593813994,
      "learning_rate": 0.00016723208254345838,
      "loss_nats": 1.170278549194336,
      "step": 4160,
      "unclipped_grad_norm": 0.8785508275032043
    },
    {
      "bits_per_byte": 1.7045477419080854,
      "learning_rate": 0.00016667409528106481,
      "loss_nats": 1.1815024614334106,
      "step": 4170,
      "unclipped_grad_norm": 0.8620172739028931
    },
    {
      "bits_per_byte": 1.6884666761218092,
      "learning_rate": 0.00016611607941300184,
      "loss_nats": 1.1703559160232544,
      "step": 4180,
      "unclipped_grad_norm": 0.917255699634552
    },
    {
      "bits_per_byte": 1.707792538581914,
      "learning_rate": 0.0001655580444742211,
      "loss_nats": 1.1837515830993652,
      "step": 4190,
      "unclipped_grad_norm": 0.8766590356826782
    },
    {
      "bits_per_byte": 1.696188009214946,
      "learning_rate": 0.00016499999999999997,
      "loss_nats": 1.1757079362869263,
      "step": 4200,
      "unclipped_grad_norm": 0.8971944451332092
    },
    {
      "bits_per_byte": 1.720210545887463,
      "learning_rate": 0.0001644419555257789,
      "loss_nats": 1.1923590898513794,
      "step": 4210,
      "unclipped_grad_norm": 0.9026775360107422
    },
    {
      "bits_per_byte": 1.6943541582087958,
      "learning_rate": 0.0001638839205869981,
      "loss_nats": 1.1744368076324463,
      "step": 4220,
      "unclipped_grad_norm": 0.8253030180931091
    },
    {
      "bits_per_byte": 1.693875014543492,
      "learning_rate": 0.0001633259047189352,
      "loss_nats": 1.1741046905517578,
      "step": 4230,
      "unclipped_grad_norm": 0.9072017669677734
    },
    {
      "bits_per_byte": 1.7273019065304893,
      "learning_rate": 0.0001627679174565416,
      "loss_nats": 1.1972744464874268,
      "step": 4240,
      "unclipped_grad_norm": 0.8421373963356018
    },
    {
      "bits_per_byte": 1.655899869371777,
      "learning_rate": 0.00016220996833428016,
      "loss_nats": 1.147782325744629,
      "step": 4250,
      "unclipped_grad_norm": 0.8799253106117249
    },
    {
      "bits_per_byte": 1.6871874691646918,
      "learning_rate": 0.00016165206688596191,
      "loss_nats": 1.1694692373275757,
      "step": 4260,
      "unclipped_grad_norm": 0.8348552584648132
    },
    {
      "bits_per_byte": 1.6070542167869764,
      "learning_rate": 0.00016109422264458335,
      "loss_nats": 1.1139250993728638,
      "step": 4270,
      "unclipped_grad_norm": 0.9256986975669861
    },
    {
      "bits_per_byte": 1.6846880452997026,
      "learning_rate": 0.00016053644514216336,
      "loss_nats": 1.1677367687225342,
      "step": 4280,
      "unclipped_grad_norm": 0.8782404661178589
    },
    {
      "bits_per_byte": 1.655903652990096,
      "learning_rate": 0.00015997874390958058,
      "loss_nats": 1.147784948348999,
      "step": 4290,
      "unclipped_grad_norm": 0.8491708636283875
    },
    {
      "bits_per_byte": 1.6260650070307003,
      "learning_rate": 0.00015942112847641018,
      "loss_nats": 1.1271023750305176,
      "step": 4300,
      "unclipped_grad_norm": 0.8707545399665833
    },
    {
      "bits_per_byte": 1.6222229146104252,
      "learning_rate": 0.00015886360837076156,
      "loss_nats": 1.1244392395019531,
      "step": 4310,
      "unclipped_grad_norm": 0.8290966153144836
    },
    {
      "bits_per_byte": 1.7449810351085826,
      "learning_rate": 0.00015830619311911495,
      "loss_nats": 1.2095286846160889,
      "step": 4320,
      "unclipped_grad_norm": 0.8566304445266724
    },
    {
      "bits_per_byte": 1.683775849319526,
      "learning_rate": 0.00015774889224615915,
      "loss_nats": 1.1671044826507568,
      "step": 4330,
      "unclipped_grad_norm": 0.8478822708129883
    },
    {
      "bits_per_byte": 1.6416012317790432,
      "learning_rate": 0.00015719171527462834,
      "loss_nats": 1.137871265411377,
      "step": 4340,
      "unclipped_grad_norm": 0.8954984545707703
    },
    {
      "bits_per_byte": 1.6099557360896828,
      "learning_rate": 0.0001566346717251397,
      "loss_nats": 1.115936279296875,
      "step": 4350,
      "unclipped_grad_norm": 0.8437728881835938
    },
    {
      "bits_per_byte": 1.6930343633460747,
      "learning_rate": 0.00015607777111603056,
      "loss_nats": 1.1735219955444336,
      "step": 4360,
      "unclipped_grad_norm": 0.9155061841011047
    },
    {
      "bits_per_byte": 1.666669594951457,
      "learning_rate": 0.00015552102296319576,
      "loss_nats": 1.1552473306655884,
      "step": 4370,
      "unclipped_grad_norm": 0.8761768341064453
    },
    {
      "bits_per_byte": 1.5925817047342201,
      "learning_rate": 0.00015496443677992516,
      "loss_nats": 1.103893518447876,
      "step": 4380,
      "unclipped_grad_norm": 0.8261715769767761
    },
    {
      "bits_per_byte": 1.7381402531878634,
      "learning_rate": 0.0001544080220767409,
      "loss_nats": 1.204787015914917,
      "step": 4390,
      "unclipped_grad_norm": 0.8470990061759949
    },
    {
      "bits_per_byte": 1.684359214471253,
      "learning_rate": 0.00015385178836123511,
      "loss_nats": 1.167508840560913,
      "step": 4400,
      "unclipped_grad_norm": 0.8836275339126587
    },
    {
      "bits_per_byte": 1.6966413554826203,
      "learning_rate": 0.00015329574513790726,
      "loss_nats": 1.1760221719741821,
      "step": 4410,
      "unclipped_grad_norm": 0.868717610836029
    },
    {
      "bits_per_byte": 1.7161705014360449,
      "learning_rate": 0.00015273990190800186,
      "loss_nats": 1.189558744430542,
      "step": 4420,
      "unclipped_grad_norm": 0.9417465329170227
    },
    {
      "bits_per_byte": 1.6679764911153634,
      "learning_rate": 0.0001521842681693461,
      "loss_nats": 1.1561532020568848,
      "step": 4430,
      "unclipped_grad_norm": 0.9034039974212646
    },
    {
      "bits_per_byte": 1.6769285320580742,
      "learning_rate": 0.00015162885341618752,
      "loss_nats": 1.162358283996582,
      "step": 4440,
      "unclipped_grad_norm": 0.8574883341789246
    },
    {
      "bits_per_byte": 1.6926945256279697,
      "learning_rate": 0.00015107366713903178,
      "loss_nats": 1.1732864379882812,
      "step": 4450,
      "unclipped_grad_norm": 0.8669698238372803
    },
    {
      "bits_per_byte": 1.689570116809745,
      "learning_rate": 0.0001505187188244806,
      "loss_nats": 1.1711207628250122,
      "step": 4460,
      "unclipped_grad_norm": 0.9505860805511475
    },
    {
      "bits_per_byte": 1.6610163532349456,
      "learning_rate": 0.00014996401795506952,
      "loss_nats": 1.1513288021087646,
      "step": 4470,
      "unclipped_grad_norm": 0.8719004988670349
    },
    {
      "bits_per_byte": 1.7079321884944145,
      "learning_rate": 0.00014940957400910585,
      "loss_nats": 1.1838483810424805,
      "step": 4480,
      "unclipped_grad_norm": 0.862112283706665
    },
    {
      "bits_per_byte": 1.747874987174651,
      "learning_rate": 0.00014885539646050699,
      "loss_nats": 1.2115346193313599,
      "step": 4490,
      "unclipped_grad_norm": 0.8834897875785828
    },
    {
      "bits_per_byte": 1.6443725602150463,
      "learning_rate": 0.00014830149477863805,
      "loss_nats": 1.1397922039031982,
      "step": 4500,
      "unclipped_grad_norm": 0.8092275261878967
    },
    {
      "bits_per_byte": 1.6392098130187958,
      "learning_rate": 0.00014774787842815066,
      "loss_nats": 1.1362136602401733,
      "step": 4510,
      "unclipped_grad_norm": 0.8292563557624817
    },
    {
      "bits_per_byte": 1.7170035853968242,
      "learning_rate": 0.00014719455686882066,
      "loss_nats": 1.190136194229126,
      "step": 4520,
      "unclipped_grad_norm": 0.929835319519043
    },
    {
      "bits_per_byte": 1.6814485800880505,
      "learning_rate": 0.00014664153955538695,
      "loss_nats": 1.1654913425445557,
      "step": 4530,
      "unclipped_grad_norm": 0.9049495458602905
    },
    {
      "bits_per_byte": 1.6774269377802746,
      "learning_rate": 0.00014608883593738946,
      "loss_nats": 1.1627037525177002,
      "step": 4540,
      "unclipped_grad_norm": 0.9091315865516663
    },
    {
      "bits_per_byte": 1.6939209339112724,
      "learning_rate": 0.00014553645545900817,
      "loss_nats": 1.1741365194320679,
      "step": 4550,
      "unclipped_grad_norm": 0.8590887188911438
    },
    {
      "bits_per_byte": 1.692378077550382,
      "learning_rate": 0.0001449844075589013,
      "loss_nats": 1.1730670928955078,
      "step": 4560,
      "unclipped_grad_norm": 0.8829112648963928
    },
    {
      "bits_per_byte": 1.680521765582551,
      "learning_rate": 0.0001444327016700444,
      "loss_nats": 1.1648489236831665,
      "step": 4570,
      "unclipped_grad_norm": 0.8655811548233032
    },
    {
      "bits_per_byte": 1.6629215770412038,
      "learning_rate": 0.0001438813472195688,
      "loss_nats": 1.1526494026184082,
      "step": 4580,
      "unclipped_grad_norm": 0.8624271154403687
    },
    {
      "bits_per_byte": 1.6390448816566183,
      "learning_rate": 0.00014333035362860093,
      "loss_nats": 1.1360993385314941,
      "step": 4590,
      "unclipped_grad_norm": 0.8392674326896667
    },
    {
      "bits_per_byte": 1.6424092062727966,
      "learning_rate": 0.00014277973031210088,
      "loss_nats": 1.1384313106536865,
      "step": 4600,
      "unclipped_grad_norm": 0.863978385925293
    },
    {
      "bits_per_byte": 1.6535189415532323,
      "learning_rate": 0.00014222948667870207,
      "loss_nats": 1.146131992340088,
      "step": 4610,
      "unclipped_grad_norm": 0.847902238368988
    },
    {
      "bits_per_byte": 1.7059622992114318,
      "learning_rate": 0.00014167963213054991,
      "loss_nats": 1.1824829578399658,
      "step": 4620,
      "unclipped_grad_norm": 0.884921133518219
    },
    {
      "bits_per_byte": 1.687283435483873,
      "learning_rate": 0.00014113017606314152,
      "loss_nats": 1.169535756111145,
      "step": 4630,
      "unclipped_grad_norm": 0.8606892824172974
    },
    {
      "bits_per_byte": 1.71471243252203,
      "learning_rate": 0.00014058112786516518,
      "loss_nats": 1.1885480880737305,
      "step": 4640,
      "unclipped_grad_norm": 0.916991114616394
    },
    {
      "bits_per_byte": 1.7302961244820108,
      "learning_rate": 0.0001400324969183396,
      "loss_nats": 1.1993498802185059,
      "step": 4650,
      "unclipped_grad_norm": 0.8595409989356995
    },
    {
      "bits_per_byte": 1.6534050890383611,
      "learning_rate": 0.00013948429259725406,
      "loss_nats": 1.1460530757904053,
      "step": 4660,
      "unclipped_grad_norm": 0.8465806245803833
    },
    {
      "bits_per_byte": 1.6640274254862515,
      "learning_rate": 0.00013893652426920776,
      "loss_nats": 1.1534159183502197,
      "step": 4670,
      "unclipped_grad_norm": 0.8708443641662598
    },
    {
      "bits_per_byte": 1.6364145789943225,
      "learning_rate": 0.00013838920129405016,
      "loss_nats": 1.1342761516571045,
      "step": 4680,
      "unclipped_grad_norm": 0.8731908202171326
    },
    {
      "bits_per_byte": 1.653688516446983,
      "learning_rate": 0.0001378423330240207,
      "loss_nats": 1.146249532699585,
      "step": 4690,
      "unclipped_grad_norm": 0.8683258891105652
    },
    {
      "bits_per_byte": 1.6763891944649685,
      "learning_rate": 0.00013729592880358936,
      "loss_nats": 1.1619844436645508,
      "step": 4700,
      "unclipped_grad_norm": 0.907905101776123
    },
    {
      "bits_per_byte": 1.661433239180637,
      "learning_rate": 0.00013674999796929659,
      "loss_nats": 1.1516177654266357,
      "step": 4710,
      "unclipped_grad_norm": 0.862278401851654
    },
    {
      "bits_per_byte": 1.68449817645315,
      "learning_rate": 0.00013620454984959417,
      "loss_nats": 1.1676051616668701,
      "step": 4720,
      "unclipped_grad_norm": 0.8695031404495239
    },
    {
      "bits_per_byte": 1.7284700126951493,
      "learning_rate": 0.00013565959376468542,
      "loss_nats": 1.1980841159820557,
      "step": 4730,
      "unclipped_grad_norm": 0.8831793069839478
    },
    {
      "bits_per_byte": 1.66034751270574,
      "learning_rate": 0.00013511513902636634,
      "loss_nats": 1.1508651971817017,
      "step": 4740,
      "unclipped_grad_norm": 0.9113041758537292
    },
    {
      "bits_per_byte": 1.6300096011108898,
      "learning_rate": 0.0001345711949378662,
      "loss_nats": 1.1298365592956543,
      "step": 4750,
      "unclipped_grad_norm": 0.9407509565353394
    },
    {
      "bits_per_byte": 1.6017535395047338,
      "learning_rate": 0.00013402777079368853,
      "loss_nats": 1.1102509498596191,
      "step": 4760,
      "unclipped_grad_norm": 0.8554308414459229
    },
    {
      "bits_per_byte": 1.6115298932930302,
      "learning_rate": 0.0001334848758794528,
      "loss_nats": 1.1170274019241333,
      "step": 4770,
      "unclipped_grad_norm": 0.846508264541626
    },
    {
      "bits_per_byte": 1.6498288817963198,
      "learning_rate": 0.00013294251947173504,
      "loss_nats": 1.1435742378234863,
      "step": 4780,
      "unclipped_grad_norm": 0.8984644412994385
    },
    {
      "bits_per_byte": 1.6369418778018678,
      "learning_rate": 0.00013240071083790996,
      "loss_nats": 1.1346416473388672,
      "step": 4790,
      "unclipped_grad_norm": 0.9358733296394348
    },
    {
      "bits_per_byte": 1.6726428963812239,
      "learning_rate": 0.0001318594592359921,
      "loss_nats": 1.1593877077102661,
      "step": 4800,
      "unclipped_grad_norm": 0.9628081321716309
    },
    {
      "bits_per_byte": 1.6377882044267638,
      "learning_rate": 0.0001313187739144781,
      "loss_nats": 1.1352282762527466,
      "step": 4810,
      "unclipped_grad_norm": 0.874860942363739
    },
    {
      "bits_per_byte": 1.6475561310651679,
      "learning_rate": 0.0001307786641121882,
      "loss_nats": 1.1419988870620728,
      "step": 4820,
      "unclipped_grad_norm": 0.8858349919319153
    },
    {
      "bits_per_byte": 1.624869727607204,
      "learning_rate": 0.00013023913905810876,
      "loss_nats": 1.1262738704681396,
      "step": 4830,
      "unclipped_grad_norm": 0.8662204742431641
    },
    {
      "bits_per_byte": 1.5894763859902412,
      "learning_rate": 0.00012970020797123432,
      "loss_nats": 1.101741075515747,
      "step": 4840,
      "unclipped_grad_norm": 0.836323082447052
    },
    {
      "bits_per_byte": 1.6522122173719767,
      "learning_rate": 0.00012916188006041027,
      "loss_nats": 1.145226240158081,
      "step": 4850,
      "unclipped_grad_norm": 0.9080801010131836
    },
    {
      "bits_per_byte": 1.6432429781641795,
      "learning_rate": 0.00012862416452417516,
      "loss_nats": 1.1390092372894287,
      "step": 4860,
      "unclipped_grad_norm": 0.8762933611869812
    },
    {
      "bits_per_byte": 1.6506518187806984,
      "learning_rate": 0.00012808707055060405,
      "loss_nats": 1.1441446542739868,
      "step": 4870,
      "unclipped_grad_norm": 0.8573883771896362
    },
    {
      "bits_per_byte": 1.6124140561011158,
      "learning_rate": 0.000127550607317151,
      "loss_nats": 1.1176402568817139,
      "step": 4880,
      "unclipped_grad_norm": 0.8266409635543823
    },
    {
      "bits_per_byte": 1.606751183356156,
      "learning_rate": 0.00012701478399049248,
      "loss_nats": 1.1137150526046753,
      "step": 4890,
      "unclipped_grad_norm": 0.9300702810287476
    },
    {
      "bits_per_byte": 1.5799279092143381,
      "learning_rate": 0.0001264796097263708,
      "loss_nats": 1.0951225757598877,
      "step": 4900,
      "unclipped_grad_norm": 0.8544939160346985
    },
    {
      "bits_per_byte": 1.6316014725272758,
      "learning_rate": 0.0001259450936694375,
      "loss_nats": 1.1309399604797363,
      "step": 4910,
      "unclipped_grad_norm": 0.8843865990638733
    },
    {
      "bits_per_byte": 1.6408737451658935,
      "learning_rate": 0.00012541124495309727,
      "loss_nats": 1.1373670101165771,
      "step": 4920,
      "unclipped_grad_norm": 0.8914017677307129
    },
    {
      "bits_per_byte": 1.6335975031731897,
      "learning_rate": 0.00012487807269935167,
      "loss_nats": 1.1323235034942627,
      "step": 4930,
      "unclipped_grad_norm": 0.9028998017311096
    },
    {
      "bits_per_byte": 1.697724846183056,
      "learning_rate": 0.00012434558601864349,
      "loss_nats": 1.176773190498352,
      "step": 4940,
      "unclipped_grad_norm": 0.9006686806678772
    },
    {
      "bits_per_byte": 1.5654046622795774,
      "learning_rate": 0.0001238137940097009,
      "loss_nats": 1.0850558280944824,
      "step": 4950,
      "unclipped_grad_norm": 0.8810124397277832
    },
    {
      "bits_per_byte": 1.6525025240866331,
      "learning_rate": 0.0001232827057593821,
      "loss_nats": 1.1454274654388428,
      "step": 4960,
      "unclipped_grad_norm": 0.9115336537361145
    },
    {
      "bits_per_byte": 1.721798977650832,
      "learning_rate": 0.00012275233034251992,
      "loss_nats": 1.1934601068496704,
      "step": 4970,
      "unclipped_grad_norm": 0.9325219988822937
    },
    {
      "bits_per_byte": 1.6130810048211617,
      "learning_rate": 0.0001222226768217669,
      "loss_nats": 1.1181025505065918,
      "step": 4980,
      "unclipped_grad_norm": 0.9085396528244019
    },
    {
      "bits_per_byte": 1.6527175024002116,
      "learning_rate": 0.00012169375424744034,
      "loss_nats": 1.1455764770507812,
      "step": 4990,
      "unclipped_grad_norm": 0.903756856918335
    },
    {
      "bits_per_byte": 1.663184366531722,
      "learning_rate": 0.00012116557165736772,
      "loss_nats": 1.1528315544128418,
      "step": 5000,
      "unclipped_grad_norm": 0.9185003638267517
    },
    {
      "bits_per_byte": 1.6328528182949538,
      "learning_rate": 0.00012063813807673214,
      "loss_nats": 1.1318073272705078,
      "step": 5010,
      "unclipped_grad_norm": 0.9199827313423157
    },
    {
      "bits_per_byte": 1.6437107709745262,
      "learning_rate": 0.00012011146251791832,
      "loss_nats": 1.1393334865570068,
      "step": 5020,
      "unclipped_grad_norm": 0.8566684722900391
    },
    {
      "bits_per_byte": 1.6317774107791085,
      "learning_rate": 0.0001195855539803584,
      "loss_nats": 1.1310619115829468,
      "step": 5030,
      "unclipped_grad_norm": 0.9012371897697449
    },
    {
      "bits_per_byte": 1.6211884389654854,
      "learning_rate": 0.00011906042145037817,
      "loss_nats": 1.1237221956253052,
      "step": 5040,
      "unclipped_grad_norm": 0.9131078124046326
    },
    {
      "bits_per_byte": 1.6615727171104868,
      "learning_rate": 0.0001185360739010438,
      "loss_nats": 1.1517144441604614,
      "step": 5050,
      "unclipped_grad_norm": 0.9046566486358643
    },
    {
      "bits_per_byte": 1.602131729353981,
      "learning_rate": 0.00011801252029200808,
      "loss_nats": 1.1105130910873413,
      "step": 5060,
      "unclipped_grad_norm": 0.895842969417572
    },
    {
      "bits_per_byte": 1.6828842912574533,
      "learning_rate": 0.00011748976956935777,
      "loss_nats": 1.1664865016937256,
      "step": 5070,
      "unclipped_grad_norm": 0.9286125898361206
    },
    {
      "bits_per_byte": 1.6860735375350533,
      "learning_rate": 0.00011696783066546024,
      "loss_nats": 1.1686971187591553,
      "step": 5080,
      "unclipped_grad_norm": 0.9972010254859924
    },
    {
      "bits_per_byte": 1.6658614484750525,
      "learning_rate": 0.00011644671249881147,
      "loss_nats": 1.1546871662139893,
      "step": 5090,
      "unclipped_grad_norm": 0.9360038042068481
    },
    {
      "bits_per_byte": 1.5959157604038465,
      "learning_rate": 0.00011592642397388293,
      "loss_nats": 1.1062045097351074,
      "step": 5100,
      "unclipped_grad_norm": 0.8976820707321167
    },
    {
      "bits_per_byte": 1.6220873922815453,
      "learning_rate": 0.00011540697398097015,
      "loss_nats": 1.124345302581787,
      "step": 5110,
      "unclipped_grad_norm": 0.8928531408309937
    },
    {
      "bits_per_byte": 1.5977783325126909,
      "learning_rate": 0.00011488837139604016,
      "loss_nats": 1.1074955463409424,
      "step": 5120,
      "unclipped_grad_norm": 0.8989635705947876
    },
    {
      "bits_per_byte": 1.6638141669991817,
      "learning_rate": 0.00011437062508058039,
      "loss_nats": 1.1532680988311768,
      "step": 5130,
      "unclipped_grad_norm": 0.9084221720695496
    },
    {
      "bits_per_byte": 1.5980395741593514,
      "learning_rate": 0.00011385374388144673,
      "loss_nats": 1.10767662525177,
      "step": 5140,
      "unclipped_grad_norm": 0.896472156047821
    },
    {
      "bits_per_byte": 1.590172399778283,
      "learning_rate": 0.00011333773663071286,
      "loss_nats": 1.102223515510559,
      "step": 5150,
      "unclipped_grad_norm": 0.8930872678756714
    },
    {
      "bits_per_byte": 1.6326397317905348,
      "learning_rate": 0.00011282261214551897,
      "loss_nats": 1.1316596269607544,
      "step": 5160,
      "unclipped_grad_norm": 0.9028102159500122
    },
    {
      "bits_per_byte": 1.6614396025387188,
      "learning_rate": 0.00011230837922792112,
      "loss_nats": 1.1516221761703491,
      "step": 5170,
      "unclipped_grad_norm": 0.9357115030288696
    },
    {
      "bits_per_byte": 1.6782567540706879,
      "learning_rate": 0.0001117950466647412,
      "loss_nats": 1.1632789373397827,
      "step": 5180,
      "unclipped_grad_norm": 0.8988713622093201
    },
    {
      "bits_per_byte": 1.6215858908716294,
      "learning_rate": 0.00011128262322741627,
      "loss_nats": 1.123997688293457,
      "step": 5190,
      "unclipped_grad_norm": 0.8884453177452087
    },
    {
      "bits_per_byte": 1.5925564232845433,
      "learning_rate": 0.00011077111767184913,
      "loss_nats": 1.103875994682312,
      "step": 5200,
      "unclipped_grad_norm": 0.9432880282402039
    },
    {
      "bits_per_byte": 1.6471872282790672,
      "learning_rate": 0.00011026053873825837,
      "loss_nats": 1.1417431831359863,
      "step": 5210,
      "unclipped_grad_norm": 0.932002067565918
    },
    {
      "bits_per_byte": 1.634469455213064,
      "learning_rate": 0.0001097508951510293,
      "loss_nats": 1.1329278945922852,
      "step": 5220,
      "unclipped_grad_norm": 0.8859370946884155
    },
    {
      "bits_per_byte": 1.684877914146255,
      "learning_rate": 0.0001092421956185646,
      "loss_nats": 1.1678683757781982,
      "step": 5230,
      "unclipped_grad_norm": 0.9864789843559265
    },
    {
      "bits_per_byte": 1.612974375577627,
      "learning_rate": 0.00010873444883313581,
      "loss_nats": 1.1180286407470703,
      "step": 5240,
      "unclipped_grad_norm": 0.9477531313896179
    },
    {
      "bits_per_byte": 1.5751990742462143,
      "learning_rate": 0.00010822766347073447,
      "loss_nats": 1.0918447971343994,
      "step": 5250,
      "unclipped_grad_norm": 0.8995106220245361
    },
    {
      "bits_per_byte": 1.5678148271487689,
      "learning_rate": 0.00010772184819092424,
      "loss_nats": 1.086726427078247,
      "step": 5260,
      "unclipped_grad_norm": 0.9177948236465454
    },
    {
      "bits_per_byte": 1.6615842399480945,
      "learning_rate": 0.0001072170116366926,
      "loss_nats": 1.1517224311828613,
      "step": 5270,
      "unclipped_grad_norm": 0.9501162171363831
    },
    {
      "bits_per_byte": 1.6530886409607735,
      "learning_rate": 0.00010671316243430345,
      "loss_nats": 1.1458337306976318,
      "step": 5280,
      "unclipped_grad_norm": 0.9527469277381897
    },
    {
      "bits_per_byte": 1.6249691335794025,
      "learning_rate": 0.00010621030919314944,
      "loss_nats": 1.1263427734375,
      "step": 5290,
      "unclipped_grad_norm": 0.9307261109352112
    },
    {
      "bits_per_byte": 1.5681051338634253,
      "learning_rate": 0.000105708460505605,
      "loss_nats": 1.0869276523590088,
      "step": 5300,
      "unclipped_grad_norm": 0.9039522409439087
    },
    {
      "bits_per_byte": 1.6362377808292354,
      "learning_rate": 0.00010520762494687964,
      "loss_nats": 1.1341536045074463,
      "step": 5310,
      "unclipped_grad_norm": 0.9477429986000061
    },
    {
      "bits_per_byte": 1.60507762018061,
      "learning_rate": 0.00010470781107487111,
      "loss_nats": 1.1125550270080566,
      "step": 5320,
      "unclipped_grad_norm": 0.926771342754364
    },
    {
      "bits_per_byte": 1.6965259551238914,
      "learning_rate": 0.00010420902743001955,
      "loss_nats": 1.1759421825408936,
      "step": 5330,
      "unclipped_grad_norm": 0.9526844620704651
    },
    {
      "bits_per_byte": 1.5996515675458884,
      "learning_rate": 0.00010371128253516117,
      "loss_nats": 1.1087939739227295,
      "step": 5340,
      "unclipped_grad_norm": 0.9416574835777283
    },
    {
      "bits_per_byte": 1.628412054267025,
      "learning_rate": 0.000103214584895383,
      "loss_nats": 1.128729224205017,
      "step": 5350,
      "unclipped_grad_norm": 0.9132721424102783
    },
    {
      "bits_per_byte": 1.5519294776191745,
      "learning_rate": 0.00010271894299787715,
      "loss_nats": 1.0757155418395996,
      "step": 5360,
      "unclipped_grad_norm": 0.8944959044456482
    },
    {
      "bits_per_byte": 1.5965125402023406,
      "learning_rate": 0.00010222436531179622,
      "loss_nats": 1.1066181659698486,
      "step": 5370,
      "unclipped_grad_norm": 0.9166159629821777
    },
    {
      "bits_per_byte": 1.6489008634122642,
      "learning_rate": 0.0001017308602881082,
      "loss_nats": 1.1429309844970703,
      "step": 5380,
      "unclipped_grad_norm": 0.9673669934272766
    },
    {
      "bits_per_byte": 1.5567627060563722,
      "learning_rate": 0.00010123843635945242,
      "loss_nats": 1.0790656805038452,
      "step": 5390,
      "unclipped_grad_norm": 0.8981189131736755
    },
    {
      "bits_per_byte": 1.6274061277421286,
      "learning_rate": 0.00010074710193999505,
      "loss_nats": 1.1280319690704346,
      "step": 5400,
      "unclipped_grad_norm": 0.95365971326828
    },
    {
      "bits_per_byte": 1.5903326876088872,
      "learning_rate": 0.0001002568654252858,
      "loss_nats": 1.1023346185684204,
      "step": 5410,
      "unclipped_grad_norm": 0.9384826421737671
    },
    {
      "bits_per_byte": 1.5563373929607884,
      "learning_rate": 9.976773519211404e-05,
      "loss_nats": 1.0787708759307861,
      "step": 5420,
      "unclipped_grad_norm": 0.8839442133903503
    },
    {
      "bits_per_byte": 1.6100503265476573,
      "learning_rate": 9.92797195983659e-05,
      "loss_nats": 1.116001844406128,
      "step": 5430,
      "unclipped_grad_norm": 0.9247893691062927
    },
    {
      "bits_per_byte": 1.5906383007794704,
      "learning_rate": 9.879282698288142e-05,
      "loss_nats": 1.1025464534759521,
      "step": 5440,
      "unclipped_grad_norm": 0.9033803343772888
    },
    {
      "bits_per_byte": 1.6134788006926075,
      "learning_rate": 9.830706566531197e-05,
      "loss_nats": 1.1183782815933228,
      "step": 5450,
      "unclipped_grad_norm": 0.9274419546127319
    },
    {
      "bits_per_byte": 1.588039986890235,
      "learning_rate": 9.782244394597826e-05,
      "loss_nats": 1.100745439529419,
      "step": 5460,
      "unclipped_grad_norm": 0.93131422996521
    },
    {
      "bits_per_byte": 1.6180344491313123,
      "learning_rate": 9.733897010572829e-05,
      "loss_nats": 1.1215360164642334,
      "step": 5470,
      "unclipped_grad_norm": 0.9031676054000854
    },
    {
      "bits_per_byte": 1.5764667583657241,
      "learning_rate": 9.685665240579612e-05,
      "loss_nats": 1.0927234888076782,
      "step": 5480,
      "unclipped_grad_norm": 0.9524510502815247
    },
    {
      "bits_per_byte": 1.667538279320965,
      "learning_rate": 9.637549908766041e-05,
      "loss_nats": 1.1558494567871094,
      "step": 5490,
      "unclipped_grad_norm": 1.0016353130340576
    },
    {
      "bits_per_byte": 1.6287254066568972,
      "learning_rate": 9.589551837290389e-05,
      "loss_nats": 1.1289464235305786,
      "step": 5500,
      "unclipped_grad_norm": 0.9340007305145264
    },
    {
      "bits_per_byte": 1.5745544832707805,
      "learning_rate": 9.541671846307257e-05,
      "loss_nats": 1.091398000717163,
      "step": 5510,
      "unclipped_grad_norm": 0.919238805770874
    },
    {
      "bits_per_byte": 1.54915986900968,
      "learning_rate": 9.493910753953597e-05,
      "loss_nats": 1.0737957954406738,
      "step": 5520,
      "unclipped_grad_norm": 0.9040963649749756
    },
    {
      "bits_per_byte": 1.5578374256416139,
      "learning_rate": 9.446269376334689e-05,
      "loss_nats": 1.079810619354248,
      "step": 5530,
      "unclipped_grad_norm": 0.8987631797790527
    },
    {
      "bits_per_byte": 1.6786796594091595,
      "learning_rate": 9.39874852751024e-05,
      "loss_nats": 1.163572072982788,
      "step": 5540,
      "unclipped_grad_norm": 0.9474910497665405
    },
    {
      "bits_per_byte": 1.6142570221877617,
      "learning_rate": 9.351349019480433e-05,
      "loss_nats": 1.11891770362854,
      "step": 5550,
      "unclipped_grad_norm": 0.9275277256965637
    },
    {
      "bits_per_byte": 1.6535560898058186,
      "learning_rate": 9.304071662172086e-05,
      "loss_nats": 1.1461577415466309,
      "step": 5560,
      "unclipped_grad_norm": 0.9428616166114807
    },
    {
      "bits_per_byte": 1.600082040120998,
      "learning_rate": 9.256917263424787e-05,
      "loss_nats": 1.109092354774475,
      "step": 5570,
      "unclipped_grad_norm": 0.8811440467834473
    },
    {
      "bits_per_byte": 1.625784675309794,
      "learning_rate": 9.209886628977114e-05,
      "loss_nats": 1.1269080638885498,
      "step": 5580,
      "unclipped_grad_norm": 0.9328234791755676
    },
    {
      "bits_per_byte": 1.6288614449337298,
      "learning_rate": 9.162980562452843e-05,
      "loss_nats": 1.1290407180786133,
      "step": 5590,
      "unclipped_grad_norm": 0.9723306894302368
    },
    {
      "bits_per_byte": 1.6197528997787336,
      "learning_rate": 9.116199865347238e-05,
      "loss_nats": 1.1227271556854248,
      "step": 5600,
      "unclipped_grad_norm": 1.0117963552474976
    },
    {
      "bits_per_byte": 1.5616874292638288,
      "learning_rate": 9.069545337013333e-05,
      "loss_nats": 1.0824792385101318,
      "step": 5610,
      "unclipped_grad_norm": 0.901840329170227
    },
    {
      "bits_per_byte": 1.6073563903045422,
      "learning_rate": 9.0230177746483e-05,
      "loss_nats": 1.1141345500946045,
      "step": 5620,
      "unclipped_grad_norm": 0.9225190281867981
    },
    {
      "bits_per_byte": 1.5324476268947373,
      "learning_rate": 8.976617973279797e-05,
      "loss_nats": 1.0622117519378662,
      "step": 5630,
      "unclipped_grad_norm": 0.8962357640266418
    },
    {
      "bits_per_byte": 1.5325635432014189,
      "learning_rate": 8.930346725752419e-05,
      "loss_nats": 1.0622920989990234,
      "step": 5640,
      "unclipped_grad_norm": 0.8800385594367981
    },
    {
      "bits_per_byte": 1.6207170345194706,
      "learning_rate": 8.88420482271411e-05,
      "loss_nats": 1.1233954429626465,
      "step": 5650,
      "unclipped_grad_norm": 0.9619011282920837
    },
    {
      "bits_per_byte": 1.5448582389462995,
      "learning_rate": 8.838193052602691e-05,
      "loss_nats": 1.0708141326904297,
      "step": 5660,
      "unclipped_grad_norm": 0.9848581552505493
    },
    {
      "bits_per_byte": 1.6303492668463437,
      "learning_rate": 8.792312201632352e-05,
      "loss_nats": 1.130071997642517,
      "step": 5670,
      "unclipped_grad_norm": 0.942057728767395
    },
    {
      "bits_per_byte": 1.6277949805157295,
      "learning_rate": 8.746563053780261e-05,
      "loss_nats": 1.1283015012741089,
      "step": 5680,
      "unclipped_grad_norm": 1.0017359256744385
    },
    {
      "bits_per_byte": 1.6320572265520623,
      "learning_rate": 8.700946390773114e-05,
      "loss_nats": 1.131255865097046,
      "step": 5690,
      "unclipped_grad_norm": 0.9255995154380798
    },
    {
      "bits_per_byte": 1.6423192593463953,
      "learning_rate": 8.655462992073833e-05,
      "loss_nats": 1.1383689641952515,
      "step": 5700,
      "unclipped_grad_norm": 0.9918621182441711
    },
    {
      "bits_per_byte": 1.6211487109731362,
      "learning_rate": 8.610113634868199e-05,
      "loss_nats": 1.123694658279419,
      "step": 5710,
      "unclipped_grad_norm": 1.0109361410140991
    },
    {
      "bits_per_byte": 1.6355581053930257,
      "learning_rate": 8.564899094051613e-05,
      "loss_nats": 1.1336824893951416,
      "step": 5720,
      "unclipped_grad_norm": 0.9666855335235596
    },
    {
      "bits_per_byte": 1.5611546270114558,
      "learning_rate": 8.519820142215816e-05,
      "loss_nats": 1.0821099281311035,
      "step": 5730,
      "unclipped_grad_norm": 0.9373083114624023
    },
    {
      "bits_per_byte": 1.6439162902423072,
      "learning_rate": 8.474877549635728e-05,
      "loss_nats": 1.13947594165802,
      "step": 5740,
      "unclipped_grad_norm": 0.923280656337738
    },
    {
      "bits_per_byte": 1.5432997321641806,
      "learning_rate": 8.430072084256247e-05,
      "loss_nats": 1.0697338581085205,
      "step": 5750,
      "unclipped_grad_norm": 0.9531146287918091
    },
    {
      "bits_per_byte": 1.5628747974853856,
      "learning_rate": 8.385404511679161e-05,
      "loss_nats": 1.0833022594451904,
      "step": 5760,
      "unclipped_grad_norm": 0.9156620502471924
    },
    {
      "bits_per_byte": 1.5744538734200257,
      "learning_rate": 8.340875595150034e-05,
      "loss_nats": 1.0913282632827759,
      "step": 5770,
      "unclipped_grad_norm": 0.9679534435272217
    },
    {
      "bits_per_byte": 1.6685440338632107,
      "learning_rate": 8.296486095545203e-05,
      "loss_nats": 1.1565465927124023,
      "step": 5780,
      "unclipped_grad_norm": 0.9836651682853699
    },
    {
      "bits_per_byte": 1.5013796489469744,
      "learning_rate": 8.252236771358729e-05,
      "loss_nats": 1.0406770706176758,
      "step": 5790,
      "unclipped_grad_norm": 0.9446811676025391
    },
    {
      "bits_per_byte": 1.6245429605705646,
      "learning_rate": 8.208128378689484e-05,
      "loss_nats": 1.1260473728179932,
      "step": 5800,
      "unclipped_grad_norm": 0.9269219040870667
    },
    {
      "bits_per_byte": 1.5340637478648953,
      "learning_rate": 8.16416167122819e-05,
      "loss_nats": 1.063331961631775,
      "step": 5810,
      "unclipped_grad_norm": 0.87967449426651
    },
    {
      "bits_per_byte": 1.5414010436986552,
      "learning_rate": 8.120337400244578e-05,
      "loss_nats": 1.0684177875518799,
      "step": 5820,
      "unclipped_grad_norm": 0.9367535710334778
    },
    {
      "bits_per_byte": 1.6061841565562613,
      "learning_rate": 8.076656314574521e-05,
      "loss_nats": 1.1133220195770264,
      "step": 5830,
      "unclipped_grad_norm": 0.9435480833053589
    },
    {
      "bits_per_byte": 1.5806048329281341,
      "learning_rate": 8.03311916060725e-05,
      "loss_nats": 1.0955917835235596,
      "step": 5840,
      "unclipped_grad_norm": 0.9508634805679321
    },
    {
      "bits_per_byte": 1.5486303344276733,
      "learning_rate": 7.989726682272606e-05,
      "loss_nats": 1.073428750038147,
      "step": 5850,
      "unclipped_grad_norm": 0.9543959498405457
    },
    {
      "bits_per_byte": 1.563623781929893,
      "learning_rate": 7.946479621028318e-05,
      "loss_nats": 1.083821415901184,
      "step": 5860,
      "unclipped_grad_norm": 0.900787353515625
    },
    {
      "bits_per_byte": 1.6201161271373559,
      "learning_rate": 7.903378715847343e-05,
      "loss_nats": 1.122978925704956,
      "step": 5870,
      "unclipped_grad_norm": 0.9706017374992371
    },
    {
      "bits_per_byte": 1.5594879431419442,
      "learning_rate": 7.860424703205224e-05,
      "loss_nats": 1.080954670906067,
      "step": 5880,
      "unclipped_grad_norm": 0.9371631741523743
    },
    {
      "bits_per_byte": 1.6214689426690427,
      "learning_rate": 7.817618317067534e-05,
      "loss_nats": 1.1239166259765625,
      "step": 5890,
      "unclipped_grad_norm": 1.0151838064193726
    },
    {
      "bits_per_byte": 1.6434954486956461,
      "learning_rate": 7.774960288877295e-05,
      "loss_nats": 1.1391842365264893,
      "step": 5900,
      "unclipped_grad_norm": 1.0114811658859253
    },
    {
      "bits_per_byte": 1.5747577667641004,
      "learning_rate": 7.732451347542522e-05,
      "loss_nats": 1.091538906097412,
      "step": 5910,
      "unclipped_grad_norm": 0.9295753836631775
    },
    {
      "bits_per_byte": 1.5403519495283922,
      "learning_rate": 7.690092219423722e-05,
      "loss_nats": 1.0676906108856201,
      "step": 5920,
      "unclipped_grad_norm": 0.9814701080322266
    },
    {
      "bits_per_byte": 1.6536778535226295,
      "learning_rate": 7.647883628321536e-05,
      "loss_nats": 1.1462421417236328,
      "step": 5930,
      "unclipped_grad_norm": 0.9983100891113281
    },
    {
      "bits_per_byte": 1.5856325737434576,
      "learning_rate": 7.605826295464316e-05,
      "loss_nats": 1.099076747894287,
      "step": 5940,
      "unclipped_grad_norm": 0.964913010597229
    },
    {
      "bits_per_byte": 1.5932624120663352,
      "learning_rate": 7.563920939495851e-05,
      "loss_nats": 1.104365348815918,
      "step": 5950,
      "unclipped_grad_norm": 0.9470323920249939
    },
    {
      "bits_per_byte": 1.5376077943512252,
      "learning_rate": 7.522168276463047e-05,
      "loss_nats": 1.0657885074615479,
      "step": 5960,
      "unclipped_grad_norm": 0.97988361120224
    },
    {
      "bits_per_byte": 1.5721553252912444,
      "learning_rate": 7.480569019803719e-05,
      "loss_nats": 1.0897350311279297,
      "step": 5970,
      "unclipped_grad_norm": 0.9889833927154541
    },
    {
      "bits_per_byte": 1.552403117839651,
      "learning_rate": 7.439123880334388e-05,
      "loss_nats": 1.0760438442230225,
      "step": 5980,
      "unclipped_grad_norm": 0.9984597563743591
    },
    {
      "bits_per_byte": 1.542794103170644,
      "learning_rate": 7.39783356623813e-05,
      "loss_nats": 1.0693833827972412,
      "step": 5990,
      "unclipped_grad_norm": 0.9274381399154663
    },
    {
      "bits_per_byte": 1.648261775881658,
      "learning_rate": 7.356698783052497e-05,
      "loss_nats": 1.1424880027770996,
      "step": 6000,
      "unclipped_grad_norm": 0.9919959902763367
    },
    {
      "bits_per_byte": 1.595717292424751,
      "learning_rate": 7.315720233657431e-05,
      "loss_nats": 1.1060669422149658,
      "step": 6010,
      "unclipped_grad_norm": 1.009833574295044
    },
    {
      "bits_per_byte": 1.5605256864572503,
      "learning_rate": 7.274898618263286e-05,
      "loss_nats": 1.0816739797592163,
      "step": 6020,
      "unclipped_grad_norm": 0.9331597685813904
    },
    {
      "bits_per_byte": 1.5469388850564376,
      "learning_rate": 7.234234634398831e-05,
      "loss_nats": 1.072256326675415,
      "step": 6030,
      "unclipped_grad_norm": 0.9432128667831421
    },
    {
      "bits_per_byte": 1.5694490062972672,
      "learning_rate": 7.193728976899362e-05,
      "loss_nats": 1.0878591537475586,
      "step": 6040,
      "unclipped_grad_norm": 0.9656267166137695
    },
    {
      "bits_per_byte": 1.5872409554943265,
      "learning_rate": 7.153382337894797e-05,
      "loss_nats": 1.100191593170166,
      "step": 6050,
      "unclipped_grad_norm": 0.9372153878211975
    },
    {
      "bits_per_byte": 1.5733865490887713,
      "learning_rate": 7.113195406797887e-05,
      "loss_nats": 1.0905884504318237,
      "step": 6060,
      "unclipped_grad_norm": 0.9818388819694519
    },
    {
      "bits_per_byte": 1.4709371719003996,
      "learning_rate": 7.073168870292397e-05,
      "loss_nats": 1.0195759534835815,
      "step": 6070,
      "unclipped_grad_norm": 0.9397172331809998
    },
    {
      "bits_per_byte": 1.5520641400348003,
      "learning_rate": 7.033303412321404e-05,
      "loss_nats": 1.0758088827133179,
      "step": 6080,
      "unclipped_grad_norm": 1.003225564956665
    },
    {
      "bits_per_byte": 1.548305115234892,
      "learning_rate": 6.993599714075586e-05,
      "loss_nats": 1.0732033252716064,
      "step": 6090,
      "unclipped_grad_norm": 0.9615646600723267
    },
    {
      "bits_per_byte": 1.6580690865471095,
      "learning_rate": 6.954058453981609e-05,
      "loss_nats": 1.149285912513733,
      "step": 6100,
      "unclipped_grad_norm": 1.030853271484375
    },
    {
      "bits_per_byte": 1.5849845431150065,
      "learning_rate": 6.914680307690504e-05,
      "loss_nats": 1.0986275672912598,
      "step": 6110,
      "unclipped_grad_norm": 1.0033804178237915
    },
    {
      "bits_per_byte": 1.5391577020008007,
      "learning_rate": 6.875465948066139e-05,
      "loss_nats": 1.0668628215789795,
      "step": 6120,
      "unclipped_grad_norm": 0.9861558079719543
    },
    {
      "bits_per_byte": 1.6089128332948508,
      "learning_rate": 6.836416045173733e-05,
      "loss_nats": 1.115213394165039,
      "step": 6130,
      "unclipped_grad_norm": 0.9338588714599609
    },
    {
      "bits_per_byte": 1.5704974125369269,
      "learning_rate": 6.797531266268377e-05,
      "loss_nats": 1.0885858535766602,
      "step": 6140,
      "unclipped_grad_norm": 0.9351722598075867
    },
    {
      "bits_per_byte": 1.5301674809095982,
      "learning_rate": 6.758812275783663e-05,
      "loss_nats": 1.060631275177002,
      "step": 6150,
      "unclipped_grad_norm": 0.9382454752922058
    },
    {
      "bits_per_byte": 1.594941134721407,
      "learning_rate": 6.720259735320298e-05,
      "loss_nats": 1.1055289506912231,
      "step": 6160,
      "unclipped_grad_norm": 0.9520907998085022
    },
    {
      "bits_per_byte": 1.53183020917814,
      "learning_rate": 6.681874303634842e-05,
      "loss_nats": 1.061783790588379,
      "step": 6170,
      "unclipped_grad_norm": 0.9805156588554382
    },
    {
      "bits_per_byte": 1.5374516341042417,
      "learning_rate": 6.643656636628402e-05,
      "loss_nats": 1.0656802654266357,
      "step": 6180,
      "unclipped_grad_norm": 0.9368215799331665
    },
    {
      "bits_per_byte": 1.5636041759076946,
      "learning_rate": 6.605607387335466e-05,
      "loss_nats": 1.0838078260421753,
      "step": 6190,
      "unclipped_grad_norm": 1.028733491897583
    },
    {
      "bits_per_byte": 1.544540243024854,
      "learning_rate": 6.567727205912724e-05,
      "loss_nats": 1.0705937147140503,
      "step": 6200,
      "unclipped_grad_norm": 0.9240782856941223
    },
    {
      "bits_per_byte": 1.5709191139968426,
      "learning_rate": 6.530016739627959e-05,
      "loss_nats": 1.0888781547546387,
      "step": 6210,
      "unclipped_grad_norm": 0.9899703860282898
    },
    {
      "bits_per_byte": 1.604423398176728,
      "learning_rate": 6.492476632848993e-05,
      "loss_nats": 1.1121015548706055,
      "step": 6220,
      "unclipped_grad_norm": 0.9920126795768738
    },
    {
      "bits_per_byte": 1.531685743751415,
      "learning_rate": 6.455107527032678e-05,
      "loss_nats": 1.0616836547851562,
      "step": 6230,
      "unclipped_grad_norm": 0.9534332156181335
    },
    {
      "bits_per_byte": 1.5783843649228444,
      "learning_rate": 6.417910060713926e-05,
      "loss_nats": 1.0940526723861694,
      "step": 6240,
      "unclipped_grad_norm": 0.9837333559989929
    },
    {
      "bits_per_byte": 1.5384501653751512,
      "learning_rate": 6.380884869494799e-05,
      "loss_nats": 1.0663723945617676,
      "step": 6250,
      "unclipped_grad_norm": 0.9426394104957581
    },
    {
      "bits_per_byte": 1.5118525324712653,
      "learning_rate": 6.34403258603367e-05,
      "loss_nats": 1.0479363203048706,
      "step": 6260,
      "unclipped_grad_norm": 0.9544605612754822
    },
    {
      "bits_per_byte": 1.5631276119821538,
      "learning_rate": 6.30735384003437e-05,
      "loss_nats": 1.08347749710083,
      "step": 6270,
      "unclipped_grad_norm": 0.9525503516197205
    },
    {
      "bits_per_byte": 1.563399516553168,
      "learning_rate": 6.270849258235484e-05,
      "loss_nats": 1.0836659669876099,
      "step": 6280,
      "unclipped_grad_norm": 0.9565526247024536
    },
    {
      "bits_per_byte": 1.5618869291388295,
      "learning_rate": 6.234519464399581e-05,
      "loss_nats": 1.0826175212860107,
      "step": 6290,
      "unclipped_grad_norm": 0.9895926713943481
    },
    {
      "bits_per_byte": 1.5244160370994442,
      "learning_rate": 6.198365079302614e-05,
      "loss_nats": 1.0566446781158447,
      "step": 6300,
      "unclipped_grad_norm": 0.9698922038078308
    },
    {
      "bits_per_byte": 1.5852545558768611,
      "learning_rate": 6.16238672072326e-05,
      "loss_nats": 1.0988147258758545,
      "step": 6310,
      "unclipped_grad_norm": 1.004137396812439
    },
    {
      "bits_per_byte": 1.55847651317222,
      "learning_rate": 6.126585003432406e-05,
      "loss_nats": 1.0802536010742188,
      "step": 6320,
      "unclipped_grad_norm": 0.9893865585327148
    },
    {
      "bits_per_byte": 1.5695085122944659,
      "learning_rate": 6.090960539182614e-05,
      "loss_nats": 1.0879004001617432,
      "step": 6330,
      "unclipped_grad_norm": 0.9685529470443726
    },
    {
      "bits_per_byte": 1.4838095573695282,
      "learning_rate": 6.0555139366976935e-05,
      "loss_nats": 1.0284984111785889,
      "step": 6340,
      "unclipped_grad_norm": 0.916477382183075
    },
    {
      "bits_per_byte": 1.5924496220583575,
      "learning_rate": 6.020245801662274e-05,
      "loss_nats": 1.103801965713501,
      "step": 6350,
      "unclipped_grad_norm": 0.9532552361488342
    },
    {
      "bits_per_byte": 1.6071991981616538,
      "learning_rate": 5.985156736711483e-05,
      "loss_nats": 1.114025592803955,
      "step": 6360,
      "unclipped_grad_norm": 0.9765400290489197
    },
    {
      "bits_per_byte": 1.53600216432277,
      "learning_rate": 5.950247341420618e-05,
      "loss_nats": 1.0646755695343018,
      "step": 6370,
      "unclipped_grad_norm": 1.0022367238998413
    },
    {
      "bits_per_byte": 1.5289498437414897,
      "learning_rate": 5.915518212294935e-05,
      "loss_nats": 1.0597872734069824,
      "step": 6380,
      "unclipped_grad_norm": 0.9823477864265442
    },
    {
      "bits_per_byte": 1.6382666601614642,
      "learning_rate": 5.880969942759425e-05,
      "loss_nats": 1.1355599164962769,
      "step": 6390,
      "unclipped_grad_norm": 0.9935681223869324
    },
    {
      "bits_per_byte": 1.5996092598137763,
      "learning_rate": 5.846603123148688e-05,
      "loss_nats": 1.1087646484375,
      "step": 6400,
      "unclipped_grad_norm": 1.055086374282837
    },
    {
      "bits_per_byte": 1.61815156931655,
      "learning_rate": 5.812418340696858e-05,
      "loss_nats": 1.1216171979904175,
      "step": 6410,
      "unclipped_grad_norm": 1.0302053689956665
    },
    {
      "bits_per_byte": 1.538316190890129,
      "learning_rate": 5.778416179527541e-05,
      "loss_nats": 1.0662795305252075,
      "step": 6420,
      "unclipped_grad_norm": 1.0392645597457886
    },
    {
      "bits_per_byte": 1.5869190039719114,
      "learning_rate": 5.7445972206438635e-05,
      "loss_nats": 1.099968433380127,
      "step": 6430,
      "unclipped_grad_norm": 0.9931192994117737
    },
    {
      "bits_per_byte": 1.5112646957706162,
      "learning_rate": 5.710962041918516e-05,
      "loss_nats": 1.047528862953186,
      "step": 6440,
      "unclipped_grad_norm": 0.9700445532798767
    },
    {
      "bits_per_byte": 1.5261810950452492,
      "learning_rate": 5.677511218083908e-05,
      "loss_nats": 1.0578681230545044,
      "step": 6450,
      "unclipped_grad_norm": 0.9057909846305847
    },
    {
      "bits_per_byte": 1.548995453595455,
      "learning_rate": 5.644245320722316e-05,
      "loss_nats": 1.0736818313598633,
      "step": 6460,
      "unclipped_grad_norm": 0.9796255826950073
    },
    {
      "bits_per_byte": 1.5656127612871213,
      "learning_rate": 5.61116491825615e-05,
      "loss_nats": 1.0852000713348389,
      "step": 6470,
      "unclipped_grad_norm": 1.0005474090576172
    },
    {
      "bits_per_byte": 1.5309916217725328,
      "learning_rate": 5.5782705759382104e-05,
      "loss_nats": 1.0612025260925293,
      "step": 6480,
      "unclipped_grad_norm": 0.9810837507247925
    },
    {
      "bits_per_byte": 1.6052196778502228,
      "learning_rate": 5.5455628558420546e-05,
      "loss_nats": 1.1126534938812256,
      "step": 6490,
      "unclipped_grad_norm": 1.013913869857788
    },
    {
      "bits_per_byte": 1.648418280093943,
      "learning_rate": 5.513042316852368e-05,
      "loss_nats": 1.1425964832305908,
      "step": 6500,
      "unclipped_grad_norm": 1.0267438888549805
    },
    {
      "bits_per_byte": 1.6007087449007422,
      "learning_rate": 5.480709514655443e-05,
      "loss_nats": 1.1095267534255981,
      "step": 6510,
      "unclipped_grad_norm": 1.015332818031311
    },
    {
      "bits_per_byte": 1.5803760960024866,
      "learning_rate": 5.448565001729653e-05,
      "loss_nats": 1.095433235168457,
      "step": 6520,
      "unclipped_grad_norm": 1.0154883861541748
    },
    {
      "bits_per_byte": 1.5618320666732044,
      "learning_rate": 5.41660932733604e-05,
      "loss_nats": 1.082579493522644,
      "step": 6530,
      "unclipped_grad_norm": 0.9579733610153198
    },
    {
      "bits_per_byte": 1.5301038473287791,
      "learning_rate": 5.384843037508902e-05,
      "loss_nats": 1.0605871677398682,
      "step": 6540,
      "unclipped_grad_norm": 0.9741249084472656
    },
    {
      "bits_per_byte": 1.5804765338705906,
      "learning_rate": 5.353266675046498e-05,
      "loss_nats": 1.0955028533935547,
      "step": 6550,
      "unclipped_grad_norm": 0.9633719325065613
    },
    {
      "bits_per_byte": 1.594095152061813,
      "learning_rate": 5.321880779501729e-05,
      "loss_nats": 1.1049425601959229,
      "step": 6560,
      "unclipped_grad_norm": 1.0173745155334473
    },
    {
      "bits_per_byte": 1.5579932419232956,
      "learning_rate": 5.290685887172962e-05,
      "loss_nats": 1.079918622970581,
      "step": 6570,
      "unclipped_grad_norm": 0.988563060760498
    },
    {
      "bits_per_byte": 1.5634815522776295,
      "learning_rate": 5.2596825310948295e-05,
      "loss_nats": 1.0837228298187256,
      "step": 6580,
      "unclipped_grad_norm": 0.980972409248352
    },
    {
      "bits_per_byte": 1.5783241709950424,
      "learning_rate": 5.228871241029154e-05,
      "loss_nats": 1.0940109491348267,
      "step": 6590,
      "unclipped_grad_norm": 1.0017447471618652
    },
    {
      "bits_per_byte": 1.4742578129232586,
      "learning_rate": 5.198252543455865e-05,
      "loss_nats": 1.021877646446228,
      "step": 6600,
      "unclipped_grad_norm": 0.9747391939163208
    },
    {
      "bits_per_byte": 1.5148066784651357,
      "learning_rate": 5.167826961564027e-05,
      "loss_nats": 1.0499839782714844,
      "step": 6610,
      "unclipped_grad_norm": 0.9631054401397705
    },
    {
      "bits_per_byte": 1.6442318784066405,
      "learning_rate": 5.137595015242884e-05,
      "loss_nats": 1.1396946907043457,
      "step": 6620,
      "unclipped_grad_norm": 1.0293588638305664
    },
    {
      "bits_per_byte": 1.593316414618706,
      "learning_rate": 5.1075572210729925e-05,
      "loss_nats": 1.104402780532837,
      "step": 6630,
      "unclipped_grad_norm": 0.9871839880943298
    },
    {
      "bits_per_byte": 1.541715771949734,
      "learning_rate": 5.07771409231737e-05,
      "loss_nats": 1.0686359405517578,
      "step": 6640,
      "unclipped_grad_norm": 0.9633845090866089
    },
    {
      "bits_per_byte": 1.535648567992596,
      "learning_rate": 5.048066138912756e-05,
      "loss_nats": 1.0644304752349854,
      "step": 6650,
      "unclipped_grad_norm": 0.9820731282234192
    },
    {
      "bits_per_byte": 1.568001256342304,
      "learning_rate": 5.018613867460863e-05,
      "loss_nats": 1.0868556499481201,
      "step": 6660,
      "unclipped_grad_norm": 1.0003972053527832
    },
    {
      "bits_per_byte": 1.5410072034281794,
      "learning_rate": 4.989357781219755e-05,
      "loss_nats": 1.0681447982788086,
      "step": 6670,
      "unclipped_grad_norm": 1.0044373273849487
    },
    {
      "bits_per_byte": 1.5651386051186926,
      "learning_rate": 4.960298380095222e-05,
      "loss_nats": 1.0848714113235474,
      "step": 6680,
      "unclipped_grad_norm": 1.0404566526412964
    },
    {
      "bits_per_byte": 1.5524736307265046,
      "learning_rate": 4.931436160632256e-05,
      "loss_nats": 1.0760927200317383,
      "step": 6690,
      "unclipped_grad_norm": 1.033932089805603
    },
    {
      "bits_per_byte": 1.565877786552101,
      "learning_rate": 4.9027716160065485e-05,
      "loss_nats": 1.0853837728500366,
      "step": 6700,
      "unclipped_grad_norm": 0.9689918756484985
    },
    {
      "bits_per_byte": 1.5892436934636238,
      "learning_rate": 4.874305236016084e-05,
      "loss_nats": 1.1015797853469849,
      "step": 6710,
      "unclipped_grad_norm": 1.0171167850494385
    },
    {
      "bits_per_byte": 1.6393943504031716,
      "learning_rate": 4.846037507072752e-05,
      "loss_nats": 1.1363415718078613,
      "step": 6720,
      "unclipped_grad_norm": 1.0878738164901733
    },
    {
      "bits_per_byte": 1.4992198908174392,
      "learning_rate": 4.817968912194049e-05,
      "loss_nats": 1.039180040359497,
      "step": 6730,
      "unclipped_grad_norm": 0.971998393535614
    },
    {
      "bits_per_byte": 1.5310298019210244,
      "learning_rate": 4.790099930994813e-05,
      "loss_nats": 1.0612289905548096,
      "step": 6740,
      "unclipped_grad_norm": 0.973441481590271
    },
    {
      "bits_per_byte": 1.5191564636707577,
      "learning_rate": 4.762431039679047e-05,
      "loss_nats": 1.0529990196228027,
      "step": 6750,
      "unclipped_grad_norm": 0.9789940714836121
    },
    {
      "bits_per_byte": 1.5175835103459665,
      "learning_rate": 4.734962711031755e-05,
      "loss_nats": 1.0519087314605713,
      "step": 6760,
      "unclipped_grad_norm": 0.9706876277923584
    },
    {
      "bits_per_byte": 1.5403151452411075,
      "learning_rate": 4.707695414410892e-05,
      "loss_nats": 1.0676651000976562,
      "step": 6770,
      "unclipped_grad_norm": 1.0128401517868042
    },
    {
      "bits_per_byte": 1.5160401380371238,
      "learning_rate": 4.680629615739322e-05,
      "loss_nats": 1.0508389472961426,
      "step": 6780,
      "unclipped_grad_norm": 0.994110107421875
    },
    {
      "bits_per_byte": 1.5638086632795705,
      "learning_rate": 4.653765777496863e-05,
      "loss_nats": 1.0839495658874512,
      "step": 6790,
      "unclipped_grad_norm": 0.9864916801452637
    },
    {
      "bits_per_byte": 1.5414822195098625,
      "learning_rate": 4.6271043587123986e-05,
      "loss_nats": 1.0684740543365479,
      "step": 6800,
      "unclipped_grad_norm": 1.023742914199829
    },
    {
      "bits_per_byte": 1.5324796156677978,
      "learning_rate": 4.600645814956006e-05,
      "loss_nats": 1.0622339248657227,
      "step": 6810,
      "unclipped_grad_norm": 1.0296359062194824
    },
    {
      "bits_per_byte": 1.5483856031154957,
      "learning_rate": 4.574390598331204e-05,
      "loss_nats": 1.0732591152191162,
      "step": 6820,
      "unclipped_grad_norm": 0.9808952212333679
    },
    {
      "bits_per_byte": 1.549455679169164,
      "learning_rate": 4.548339157467199e-05,
      "loss_nats": 1.0740008354187012,
      "step": 6830,
      "unclipped_grad_norm": 0.996810793876648
    },
    {
      "bits_per_byte": 1.5259360197677698,
      "learning_rate": 4.52249193751124e-05,
      "loss_nats": 1.0576982498168945,
      "step": 6840,
      "unclipped_grad_norm": 0.9733325242996216
    },
    {
      "bits_per_byte": 1.6203460679415593,
      "learning_rate": 4.4968493801209985e-05,
      "loss_nats": 1.1231383085250854,
      "step": 6850,
      "unclipped_grad_norm": 1.0648897886276245
    },
    {
      "bits_per_byte": 1.568673364541876,
      "learning_rate": 4.471411923457034e-05,
      "loss_nats": 1.0873215198516846,
      "step": 6860,
      "unclipped_grad_norm": 1.0356100797653198
    },
    {
      "bits_per_byte": 1.4980275350990075,
      "learning_rate": 4.446180002175291e-05,
      "loss_nats": 1.0383535623550415,
      "step": 6870,
      "unclipped_grad_norm": 0.9937211275100708
    },
    {
      "bits_per_byte": 1.521422507078526,
      "learning_rate": 4.421154047419693e-05,
      "loss_nats": 1.0545697212219238,
      "step": 6880,
      "unclipped_grad_norm": 1.069592833518982
    },
    {
      "bits_per_byte": 1.5148947335823775,
      "learning_rate": 4.396334486814753e-05,
      "loss_nats": 1.0500450134277344,
      "step": 6890,
      "unclipped_grad_norm": 1.0049850940704346
    },
    {
      "bits_per_byte": 1.5822056474423651,
      "learning_rate": 4.371721744458285e-05,
      "loss_nats": 1.0967013835906982,
      "step": 6900,
      "unclipped_grad_norm": 1.0225300788879395
    },
    {
      "bits_per_byte": 1.5606532975841907,
      "learning_rate": 4.347316240914145e-05,
      "loss_nats": 1.081762433052063,
      "step": 6910,
      "unclipped_grad_norm": 1.021737813949585
    },
    {
      "bits_per_byte": 1.5130916674707318,
      "learning_rate": 4.3231183932050546e-05,
      "loss_nats": 1.048795223236084,
      "step": 6920,
      "unclipped_grad_norm": 1.0000710487365723
    },
    {
      "bits_per_byte": 1.5005744261756349,
      "learning_rate": 4.299128614805464e-05,
      "loss_nats": 1.040118932723999,
      "step": 6930,
      "unclipped_grad_norm": 1.0489689111709595
    },
    {
      "bits_per_byte": 1.568557792200496,
      "learning_rate": 4.2753473156344965e-05,
      "loss_nats": 1.0872414112091064,
      "step": 6940,
      "unclipped_grad_norm": 1.0565794706344604
    },
    {
      "bits_per_byte": 1.484857619643886,
      "learning_rate": 4.251774902048942e-05,
      "loss_nats": 1.0292248725891113,
      "step": 6950,
      "unclipped_grad_norm": 1.0331164598464966
    },
    {
      "bits_per_byte": 1.498764136792653,
      "learning_rate": 4.228411776836306e-05,
      "loss_nats": 1.0388641357421875,
      "step": 6960,
      "unclipped_grad_norm": 1.0262477397918701
    },
    {
      "bits_per_byte": 1.4731473209466375,
      "learning_rate": 4.2052583392079427e-05,
      "loss_nats": 1.0211079120635986,
      "step": 6970,
      "unclipped_grad_norm": 0.9956484436988831
    },
    {
      "bits_per_byte": 1.5915678670073838,
      "learning_rate": 4.182314984792214e-05,
      "loss_nats": 1.1031907796859741,
      "step": 6980,
      "unclipped_grad_norm": 1.0641905069351196
    },
    {
      "bits_per_byte": 1.5488172795691613,
      "learning_rate": 4.159582105627748e-05,
      "loss_nats": 1.0735583305358887,
      "step": 6990,
      "unclipped_grad_norm": 1.0450271368026733
    },
    {
      "bits_per_byte": 1.5813185609292149,
      "learning_rate": 4.137060090156724e-05,
      "loss_nats": 1.0960865020751953,
      "step": 7000,
      "unclipped_grad_norm": 1.0753706693649292
    },
    {
      "bits_per_byte": 1.4760245906955722,
      "learning_rate": 4.114749323218249e-05,
      "loss_nats": 1.0231022834777832,
      "step": 7010,
      "unclipped_grad_norm": 0.977759838104248
    },
    {
      "bits_per_byte": 1.5697365612895098,
      "learning_rate": 4.0926501860417706e-05,
      "loss_nats": 1.0880584716796875,
      "step": 7020,
      "unclipped_grad_norm": 1.0638940334320068
    },
    {
      "bits_per_byte": 1.5177874817698898,
      "learning_rate": 4.070763056240571e-05,
      "loss_nats": 1.0520501136779785,
      "step": 7030,
      "unclipped_grad_norm": 1.0119246244430542
    },
    {
      "bits_per_byte": 1.5230066392756236,
      "learning_rate": 4.049088307805306e-05,
      "loss_nats": 1.055667757987976,
      "step": 7040,
      "unclipped_grad_norm": 1.0224649906158447
    },
    {
      "bits_per_byte": 1.5100723400521843,
      "learning_rate": 4.027626311097629e-05,
      "loss_nats": 1.0467023849487305,
      "step": 7050,
      "unclipped_grad_norm": 1.0288535356521606
    },
    {
      "bits_per_byte": 1.4969820525644124,
      "learning_rate": 4.006377432843846e-05,
      "loss_nats": 1.0376288890838623,
      "step": 7060,
      "unclipped_grad_norm": 0.9655771851539612
    },
    {
      "bits_per_byte": 1.5399431467672913,
      "learning_rate": 3.985342036128656e-05,
      "loss_nats": 1.067407250404358,
      "step": 7070,
      "unclipped_grad_norm": 0.986509382724762
    },
    {
      "bits_per_byte": 1.5402030125527448,
      "learning_rate": 3.964520480388956e-05,
      "loss_nats": 1.0675873756408691,
      "step": 7080,
      "unclipped_grad_norm": 1.0742125511169434
    },
    {
      "bits_per_byte": 1.5012656244494524,
      "learning_rate": 3.943913121407678e-05,
      "loss_nats": 1.0405980348587036,
      "step": 7090,
      "unclipped_grad_norm": 1.0513901710510254
    },
    {
      "bits_per_byte": 1.5043721470719873,
      "learning_rate": 3.923520311307735e-05,
      "loss_nats": 1.0427513122558594,
      "step": 7100,
      "unclipped_grad_norm": 1.0241730213165283
    },
    {
      "bits_per_byte": 1.4832691878805173,
      "learning_rate": 3.9033423985459816e-05,
      "loss_nats": 1.0281238555908203,
      "step": 7110,
      "unclipped_grad_norm": 0.991193950176239
    },
    {
      "bits_per_byte": 1.5470104298391967,
      "learning_rate": 3.88337972790728e-05,
      "loss_nats": 1.0723059177398682,
      "step": 7120,
      "unclipped_grad_norm": 1.0775213241577148
    },
    {
      "bits_per_byte": 1.564362103450047,
      "learning_rate": 3.8636326404985845e-05,
      "loss_nats": 1.0843331813812256,
      "step": 7130,
      "unclipped_grad_norm": 1.0102075338363647
    },
    {
      "bits_per_byte": 1.530404644985138,
      "learning_rate": 3.844101473743143e-05,
      "loss_nats": 1.0607956647872925,
      "step": 7140,
      "unclipped_grad_norm": 1.0219485759735107
    },
    {
      "bits_per_byte": 1.5796152447550695,
      "learning_rate": 3.824786561374701e-05,
      "loss_nats": 1.0949058532714844,
      "step": 7150,
      "unclipped_grad_norm": 1.0659542083740234
    },
    {
      "bits_per_byte": 1.568367407405991,
      "learning_rate": 3.805688233431824e-05,
      "loss_nats": 1.0871094465255737,
      "step": 7160,
      "unclipped_grad_norm": 1.0356900691986084
    },
    {
      "bits_per_byte": 1.5848232233884971,
      "learning_rate": 3.786806816252236e-05,
      "loss_nats": 1.0985157489776611,
      "step": 7170,
      "unclipped_grad_norm": 1.0675727128982544
    },
    {
      "bits_per_byte": 1.5187308066098724,
      "learning_rate": 3.768142632467269e-05,
      "loss_nats": 1.0527039766311646,
      "step": 7180,
      "unclipped_grad_norm": 0.9808175563812256
    },
    {
      "bits_per_byte": 1.5693248348233442,
      "learning_rate": 3.7496960009963236e-05,
      "loss_nats": 1.087773084640503,
      "step": 7190,
      "unclipped_grad_norm": 1.0627371072769165
    },
    {
      "bits_per_byte": 1.5123168856285947,
      "learning_rate": 3.7314672370414325e-05,
      "loss_nats": 1.0482581853866577,
      "step": 7200,
      "unclipped_grad_norm": 1.0047924518585205
    },
    {
      "bits_per_byte": 1.5017243021793036,
      "learning_rate": 3.713456652081882e-05,
      "loss_nats": 1.0409159660339355,
      "step": 7210,
      "unclipped_grad_norm": 1.0128343105316162
    },
    {
      "bits_per_byte": 1.5058675362212395,
      "learning_rate": 3.695664553868873e-05,
      "loss_nats": 1.0437878370285034,
      "step": 7220,
      "unclipped_grad_norm": 1.020466685295105
    },
    {
      "bits_per_byte": 1.5492668422185167,
      "learning_rate": 3.678091246420277e-05,
      "loss_nats": 1.0738699436187744,
      "step": 7230,
      "unclipped_grad_norm": 1.0896223783493042
    },
    {
      "bits_per_byte": 1.5598331123222258,
      "learning_rate": 3.660737030015427e-05,
      "loss_nats": 1.0811939239501953,
      "step": 7240,
      "unclipped_grad_norm": 1.0040799379348755
    },
    {
      "bits_per_byte": 1.5635771746315092,
      "learning_rate": 3.6436022011900056e-05,
      "loss_nats": 1.0837891101837158,
      "step": 7250,
      "unclipped_grad_norm": 1.0716665983200073
    },
    {
      "bits_per_byte": 1.530704410745592,
      "learning_rate": 3.626687052730954e-05,
      "loss_nats": 1.0610034465789795,
      "step": 7260,
      "unclipped_grad_norm": 1.0284863710403442
    },
    {
      "bits_per_byte": 1.5407076096503765,
      "learning_rate": 3.6099918736714955e-05,
      "loss_nats": 1.0679371356964111,
      "step": 7270,
      "unclipped_grad_norm": 1.0729715824127197
    },
    {
      "bits_per_byte": 1.5375348737072594,
      "learning_rate": 3.5935169492861716e-05,
      "loss_nats": 1.0657379627227783,
      "step": 7280,
      "unclipped_grad_norm": 1.0802364349365234
    },
    {
      "bits_per_byte": 1.5291964668628268,
      "learning_rate": 3.5772625610859886e-05,
      "loss_nats": 1.0599582195281982,
      "step": 7290,
      "unclipped_grad_norm": 1.0581231117248535
    },
    {
      "bits_per_byte": 1.5223175047936162,
      "learning_rate": 3.5612289868135887e-05,
      "loss_nats": 1.055190086364746,
      "step": 7300,
      "unclipped_grad_norm": 1.0030993223190308
    },
    {
      "bits_per_byte": 1.518025505758684,
      "learning_rate": 3.545416500438522e-05,
      "loss_nats": 1.0522150993347168,
      "step": 7310,
      "unclipped_grad_norm": 1.0415269136428833
    },
    {
      "bits_per_byte": 1.5002450793992326,
      "learning_rate": 3.52982537215255e-05,
      "loss_nats": 1.0398906469345093,
      "step": 7320,
      "unclipped_grad_norm": 1.0742172002792358
    },
    {
      "bits_per_byte": 1.5556656287265183,
      "learning_rate": 3.5144558683650396e-05,
      "loss_nats": 1.0783052444458008,
      "step": 7330,
      "unclipped_grad_norm": 1.098557472229004
    },
    {
      "bits_per_byte": 1.5089695872948519,
      "learning_rate": 3.499308251698404e-05,
      "loss_nats": 1.0459380149841309,
      "step": 7340,
      "unclipped_grad_norm": 1.0178015232086182
    },
    {
      "bits_per_byte": 1.5128952632834465,
      "learning_rate": 3.484382780983616e-05,
      "loss_nats": 1.048659086227417,
      "step": 7350,
      "unclipped_grad_norm": 1.0542628765106201
    },
    {
      "bits_per_byte": 1.5165268489390655,
      "learning_rate": 3.469679711255795e-05,
      "loss_nats": 1.0511763095855713,
      "step": 7360,
      "unclipped_grad_norm": 1.0132694244384766
    },
    {
      "bits_per_byte": 1.4757258568310236,
      "learning_rate": 3.4551992937498284e-05,
      "loss_nats": 1.0228952169418335,
      "step": 7370,
      "unclipped_grad_norm": 1.0333343744277954
    },
    {
      "bits_per_byte": 1.4706363742440405,
      "learning_rate": 3.440941775896104e-05,
      "loss_nats": 1.0193674564361572,
      "step": 7380,
      "unclipped_grad_norm": 1.0362695455551147
    },
    {
      "bits_per_byte": 1.4849742238811712,
      "learning_rate": 3.4269074013162596e-05,
      "loss_nats": 1.0293056964874268,
      "step": 7390,
      "unclipped_grad_norm": 1.0089579820632935
    },
    {
      "bits_per_byte": 1.4991997688472884,
      "learning_rate": 3.41309640981904e-05,
      "loss_nats": 1.0391660928726196,
      "step": 7400,
      "unclipped_grad_norm": 1.0394701957702637
    },
    {
      "bits_per_byte": 1.5501245196983695,
      "learning_rate": 3.3995090373961795e-05,
      "loss_nats": 1.0744644403457642,
      "step": 7410,
      "unclipped_grad_norm": 1.044879674911499
    },
    {
      "bits_per_byte": 1.5165729402894967,
      "learning_rate": 3.3861455162183876e-05,
      "loss_nats": 1.051208257675171,
      "step": 7420,
      "unclipped_grad_norm": 1.0228228569030762
    },
    {
      "bits_per_byte": 1.4806290822071222,
      "learning_rate": 3.3730060746313664e-05,
      "loss_nats": 1.0262938737869263,
      "step": 7430,
      "unclipped_grad_norm": 1.033545970916748
    },
    {
      "bits_per_byte": 1.4643294264715996,
      "learning_rate": 3.3600909371519225e-05,
      "loss_nats": 1.014995813369751,
      "step": 7440,
      "unclipped_grad_norm": 1.0252758264541626
    },
    {
      "bits_per_byte": 1.5693753977226979,
      "learning_rate": 3.3474003244641194e-05,
      "loss_nats": 1.0878081321716309,
      "step": 7450,
      "unclipped_grad_norm": 1.039492130279541
    },
    {
      "bits_per_byte": 1.4916665847741957,
      "learning_rate": 3.334934453415514e-05,
      "loss_nats": 1.0339444875717163,
      "step": 7460,
      "unclipped_grad_norm": 1.036054253578186
    },
    {
      "bits_per_byte": 1.466042717639495,
      "learning_rate": 3.3226935370134486e-05,
      "loss_nats": 1.0161833763122559,
      "step": 7470,
      "unclipped_grad_norm": 1.0448886156082153
    },
    {
      "bits_per_byte": 1.5608368030726611,
      "learning_rate": 3.31067778442141e-05,
      "loss_nats": 1.0818896293640137,
      "step": 7480,
      "unclipped_grad_norm": 1.0651273727416992
    },
    {
      "bits_per_byte": 1.5231578120257319,
      "learning_rate": 3.298887400955457e-05,
      "loss_nats": 1.0557725429534912,
      "step": 7490,
      "unclipped_grad_norm": 1.077128291130066
    },
    {
      "bits_per_byte": 1.5418131141301226,
      "learning_rate": 3.287322588080714e-05,
      "loss_nats": 1.0687034130096436,
      "step": 7500,
      "unclipped_grad_norm": 1.0931757688522339
    },
    {
      "bits_per_byte": 1.528281175194935,
      "learning_rate": 3.275983543407924e-05,
      "loss_nats": 1.059323787689209,
      "step": 7510,
      "unclipped_grad_norm": 1.0170495510101318
    },
    {
      "bits_per_byte": 1.541195696413525,
      "learning_rate": 3.2648704606900735e-05,
      "loss_nats": 1.0682754516601562,
      "step": 7520,
      "unclipped_grad_norm": 1.072600245475769
    },
    {
      "bits_per_byte": 1.5347975978361268,
      "learning_rate": 3.253983529819086e-05,
      "loss_nats": 1.063840627670288,
      "step": 7530,
      "unclipped_grad_norm": 1.0730668306350708
    },
    {
      "bits_per_byte": 1.5420986053305548,
      "learning_rate": 3.243322936822571e-05,
      "loss_nats": 1.0689013004302979,
      "step": 7540,
      "unclipped_grad_norm": 1.0506013631820679
    },
    {
      "bits_per_byte": 1.5157248938380923,
      "learning_rate": 3.232888863860648e-05,
      "loss_nats": 1.050620436668396,
      "step": 7550,
      "unclipped_grad_norm": 1.036077857017517
    },
    {
      "bits_per_byte": 1.443441617576359,
      "learning_rate": 3.222681489222838e-05,
      "loss_nats": 1.00051748752594,
      "step": 7560,
      "unclipped_grad_norm": 1.0049386024475098
    },
    {
      "bits_per_byte": 1.4652058500603964,
      "learning_rate": 3.212700987325009e-05,
      "loss_nats": 1.0156033039093018,
      "step": 7570,
      "unclipped_grad_norm": 1.0102496147155762
    },
    {
      "bits_per_byte": 1.4617899306489595,
      "learning_rate": 3.202947528706403e-05,
      "loss_nats": 1.0132355690002441,
      "step": 7580,
      "unclipped_grad_norm": 1.0245665311813354
    },
    {
      "bits_per_byte": 1.5365071053857031,
      "learning_rate": 3.1934212800267116e-05,
      "loss_nats": 1.0650255680084229,
      "step": 7590,
      "unclipped_grad_norm": 1.048311471939087
    },
    {
      "bits_per_byte": 1.567407228266224,
      "learning_rate": 3.1841224040632484e-05,
      "loss_nats": 1.0864439010620117,
      "step": 7600,
      "unclipped_grad_norm": 1.0463460683822632
    },
    {
      "bits_per_byte": 1.5308137917115408,
      "learning_rate": 3.175051059708143e-05,
      "loss_nats": 1.0610792636871338,
      "step": 7610,
      "unclipped_grad_norm": 1.0807271003723145
    },
    {
      "bits_per_byte": 1.5336785067269625,
      "learning_rate": 3.1662074019656414e-05,
      "loss_nats": 1.0630649328231812,
      "step": 7620,
      "unclipped_grad_norm": 1.048875331878662
    },
    {
      "bits_per_byte": 1.510936380890119,
      "learning_rate": 3.157591581949455e-05,
      "loss_nats": 1.0473012924194336,
      "step": 7630,
      "unclipped_grad_norm": 1.0328357219696045
    },
    {
      "bits_per_byte": 1.4914947741059836,
      "learning_rate": 3.149203746880175e-05,
      "loss_nats": 1.033825397491455,
      "step": 7640,
      "unclipped_grad_norm": 1.0975395441055298
    },
    {
      "bits_per_byte": 1.462546310347454,
      "learning_rate": 3.141044040082763e-05,
      "loss_nats": 1.0137598514556885,
      "step": 7650,
      "unclipped_grad_norm": 1.0314407348632812
    },
    {
      "bits_per_byte": 1.4939462148113822,
      "learning_rate": 3.133112600984091e-05,
      "loss_nats": 1.035524606704712,
      "step": 7660,
      "unclipped_grad_norm": 1.1010568141937256
    },
    {
      "bits_per_byte": 1.5304318182439745,
      "learning_rate": 3.125409565110571e-05,
      "loss_nats": 1.0608144998550415,
      "step": 7670,
      "unclipped_grad_norm": 1.0511856079101562
    },
    {
      "bits_per_byte": 1.5306942637691912,
      "learning_rate": 3.117935064085835e-05,
      "loss_nats": 1.060996413230896,
      "step": 7680,
      "unclipped_grad_norm": 1.062692403793335
    },
    {
      "bits_per_byte": 1.5040376408160592,
      "learning_rate": 3.110689225628479e-05,
      "loss_nats": 1.042519450187683,
      "step": 7690,
      "unclipped_grad_norm": 1.0179529190063477
    },
    {
      "bits_per_byte": 1.5428006385113768,
      "learning_rate": 3.1036721735498926e-05,
      "loss_nats": 1.0693879127502441,
      "step": 7700,
      "unclipped_grad_norm": 1.0779086351394653
    },
    {
      "bits_per_byte": 1.4839846357081066,
      "learning_rate": 3.096884027752132e-05,
      "loss_nats": 1.0286197662353516,
      "step": 7710,
      "unclipped_grad_norm": 1.0490714311599731
    },
    {
      "bits_per_byte": 1.496933553456869,
      "learning_rate": 3.090324904225885e-05,
      "loss_nats": 1.037595272064209,
      "step": 7720,
      "unclipped_grad_norm": 0.9670684337615967
    },
    {
      "bits_per_byte": 1.4854798528747077,
      "learning_rate": 3.083994915048472e-05,
      "loss_nats": 1.029656171798706,
      "step": 7730,
      "unclipped_grad_norm": 0.9940106868743896
    },
    {
      "bits_per_byte": 1.5407578285844283,
      "learning_rate": 3.077894168381942e-05,
      "loss_nats": 1.06797194480896,
      "step": 7740,
      "unclipped_grad_norm": 1.0928970575332642
    },
    {
      "bits_per_byte": 1.4753865350608713,
      "learning_rate": 3.0720227684712254e-05,
      "loss_nats": 1.0226600170135498,
      "step": 7750,
      "unclipped_grad_norm": 1.013372778892517
    },
    {
      "bits_per_byte": 1.4801974057534566,
      "learning_rate": 3.0663808156423456e-05,
      "loss_nats": 1.0259946584701538,
      "step": 7760,
      "unclipped_grad_norm": 1.0438604354858398
    },
    {
      "bits_per_byte": 1.538979355991856,
      "learning_rate": 3.060968406300711e-05,
      "loss_nats": 1.0667392015457153,
      "step": 7770,
      "unclipped_grad_norm": 1.1242064237594604
    },
    {
      "bits_per_byte": 1.51728718423853,
      "learning_rate": 3.055785632929463e-05,
      "loss_nats": 1.0517033338546753,
      "step": 7780,
      "unclipped_grad_norm": 1.1380839347839355
    },
    {
      "bits_per_byte": 1.5213089985289567,
      "learning_rate": 3.0508325840878984e-05,
      "loss_nats": 1.0544910430908203,
      "step": 7790,
      "unclipped_grad_norm": 1.054468035697937
    },
    {
      "bits_per_byte": 1.5100510142034773,
      "learning_rate": 3.046109344409957e-05,
      "loss_nats": 1.0466876029968262,
      "step": 7800,
      "unclipped_grad_norm": 1.0544580221176147
    },
    {
      "bits_per_byte": 1.498489308516574,
      "learning_rate": 3.0416159946027727e-05,
      "loss_nats": 1.0386736392974854,
      "step": 7810,
      "unclipped_grad_norm": 1.0938669443130493
    },
    {
      "bits_per_byte": 1.5401904578192318,
      "learning_rate": 3.0373526114452946e-05,
      "loss_nats": 1.067578673362732,
      "step": 7820,
      "unclipped_grad_norm": 1.1261787414550781
    },
    {
      "bits_per_byte": 1.5279041892242438,
      "learning_rate": 3.0333192677869803e-05,
      "loss_nats": 1.0590624809265137,
      "step": 7830,
      "unclipped_grad_norm": 1.0831990242004395
    },
    {
      "bits_per_byte": 1.5007298984920148,
      "learning_rate": 3.029516032546544e-05,
      "loss_nats": 1.040226697921753,
      "step": 7840,
      "unclipped_grad_norm": 1.0495885610580444
    },
    {
      "bits_per_byte": 1.485421722738716,
      "learning_rate": 3.02594297071078e-05,
      "loss_nats": 1.029615879058838,
      "step": 7850,
      "unclipped_grad_norm": 1.0680148601531982
    },
    {
      "bits_per_byte": 1.5337288976436654,
      "learning_rate": 3.0226001433334603e-05,
      "loss_nats": 1.0630998611450195,
      "step": 7860,
      "unclipped_grad_norm": 1.059203028678894
    },
    {
      "bits_per_byte": 1.528788867980282,
      "learning_rate": 3.019487607534278e-05,
      "loss_nats": 1.059675693511963,
      "step": 7870,
      "unclipped_grad_norm": 1.0786525011062622
    },
    {
      "bits_per_byte": 1.4810511276323397,
      "learning_rate": 3.016605416497886e-05,
      "loss_nats": 1.0265864133834839,
      "step": 7880,
      "unclipped_grad_norm": 1.0475893020629883
    },
    {
      "bits_per_byte": 1.50947349646188,
      "learning_rate": 3.0139536194729734e-05,
      "loss_nats": 1.0462872982025146,
      "step": 7890,
      "unclipped_grad_norm": 1.0943796634674072
    },
    {
      "bits_per_byte": 1.4635482812713807,
      "learning_rate": 3.0115322617714362e-05,
      "loss_nats": 1.0144543647766113,
      "step": 7900,
      "unclipped_grad_norm": 1.0396785736083984
    },
    {
      "bits_per_byte": 1.530891011921778,
      "learning_rate": 3.0093413847675965e-05,
      "loss_nats": 1.061132788658142,
      "step": 7910,
      "unclipped_grad_norm": 1.1283469200134277
    },
    {
      "bits_per_byte": 1.5486817572402813,
      "learning_rate": 3.0073810258974985e-05,
      "loss_nats": 1.0734643936157227,
      "step": 7920,
      "unclipped_grad_norm": 1.0839927196502686
    },
    {
      "bits_per_byte": 1.5089941808139253,
      "learning_rate": 3.0056512186582658e-05,
      "loss_nats": 1.0459550619125366,
      "step": 7930,
      "unclipped_grad_norm": 1.0844900608062744
    },
    {
      "bits_per_byte": 1.5398654106091012,
      "learning_rate": 3.0041519926075316e-05,
      "loss_nats": 1.067353367805481,
      "step": 7940,
      "unclipped_grad_norm": 1.0818886756896973
    },
    {
      "bits_per_byte": 1.473885126518839,
      "learning_rate": 3.0028833733629313e-05,
      "loss_nats": 1.0216193199157715,
      "step": 7950,
      "unclipped_grad_norm": 1.0441930294036865
    },
    {
      "bits_per_byte": 1.5289575829607784,
      "learning_rate": 3.0018453826016686e-05,
      "loss_nats": 1.0597926378250122,
      "step": 7960,
      "unclipped_grad_norm": 1.1338133811950684
    },
    {
      "bits_per_byte": 1.498011712695128,
      "learning_rate": 3.001038038060143e-05,
      "loss_nats": 1.0383425951004028,
      "step": 7970,
      "unclipped_grad_norm": 1.0597225427627563
    },
    {
      "bits_per_byte": 1.5385745088317249,
      "learning_rate": 3.0004613535336433e-05,
      "loss_nats": 1.0664585828781128,
      "step": 7980,
      "unclipped_grad_norm": 1.1261745691299438
    },
    {
      "bits_per_byte": 1.500032680825417,
      "learning_rate": 3.000115338876117e-05,
      "loss_nats": 1.039743423461914,
      "step": 7990,
      "unclipped_grad_norm": 1.0900131464004517
    },
    {
      "bits_per_byte": 1.5133369147308622,
      "learning_rate": 3e-05,
      "loss_nats": 1.0489652156829834,
      "step": 8000,
      "unclipped_grad_norm": 1.0884610414505005
    }
  ]
}