{
  "architecture": "baseline",
  "batch_size": 32,
  "canonical_sha256_without_self": "6574e479d5e2794b43a6244a723ba3a2b034f8ef3bfeae24909948fb4a990d99",
  "depth": 32,
  "diagnostics": [
    {
      "activation_grad_rms_by_block": [
        0.0002768727717921138,
        0.0002554995589889586,
        0.00023783760843798518,
        0.00022115583124104887,
        0.00020655356638599187,
        0.00019376208365429193,
        0.00018488570640329272,
        0.00017590694187674671,
        0.0001684112212387845,
        0.0001647331955609843,
        0.00015843863366171718,
        0.0001538397918920964,
        0.00015132295084185898,
        0.00014730033581145108,
        0.00014253577683120966,
        0.0001394751889165491,
        0.0001364792260574177,
        0.0001337067224085331,
        0.00013106675760354847,
        0.00012915431580040604,
        0.00012709009752143174,
        0.00012574320135172457,
        0.0001242366706719622,
        0.0001228937617270276,
        0.00012132258416386321,
        0.00011988351616309956,
        0.00011879197700181976,
        0.00011764531518565491,
        0.00011639683361863717,
        0.00011549647024367005,
        0.00011481995898066089,
        0.00011415599146857858
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.0002190592585975537,
        "first_to_last_ratio": 1.8672887091156776,
        "imbalance_abs_log_ratio": 0.6244874905881238,
        "last_quartile_mean": 0.00011731408085324802,
        "mean": 0.0001546067051094724,
        "normalized": [
          1.7908199492128654,
          1.6525774791465027,
          1.538339545297078,
          1.4304413969915148,
          1.3359935860462033,
          1.2532579587482624,
          1.1958453307208163,
          1.1377704592578457,
          1.0892879524179597,
          1.065498391187767,
          1.0247850088360109,
          0.9950395863049215,
          0.978760596021445,
          0.9527422223192203,
          0.9219249367631522,
          0.9021289782858439,
          0.8827510162691898,
          0.8648183939620171,
          0.8477430361816715,
          0.835373315206192,
          0.8220219002238166,
          0.8133101424203405,
          0.80356586464988,
          0.7948798963150414,
          0.7847174809007043,
          0.775409553409819,
          0.7683494510649245,
          0.7609328140222236,
          0.7528576043077825,
          0.7470340316863389,
          0.7426583400723811,
          0.7383637817502684
        ],
        "population_cv": 0.27888268733798954
      },
      "activation_output_rms_by_block": [
        0.028270306065678596,
        0.0284538883715868,
        0.02867252007126808,
        0.02881420962512493,
        0.029061447829008102,
        0.02929086796939373,
        0.029533835127949715,
        0.029811719432473183,
        0.030115574598312378,
        0.030315518379211426,
        0.030812423676252365,
        0.03120259940624237,
        0.03179442882537842,
        0.03198441118001938,
        0.03240228444337845,
        0.0329107940196991,
        0.033366311341524124,
        0.033811621367931366,
        0.03420914337038994,
        0.03470747172832489,
        0.03500960022211075,
        0.03567802160978317,
        0.035978782922029495,
        0.036836784332990646,
        0.03769209608435631,
        0.038124579936265945,
        0.038542140275239944,
        0.03904740884900093,
        0.0400773324072361,
        0.04093807935714722,
        0.04163828864693642,
        0.042313966900110245
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.028988599311560392,
        "first_to_last_ratio": 0.728416493900548,
        "imbalance_abs_log_ratio": 0.31688228737002583,
        "last_quartile_mean": 0.03979673655703664,
        "mean": 0.03379432682413608,
        "normalized": [
          0.8365399971656723,
          0.8419723381282118,
          0.8484418174825138,
          0.8526345198433033,
          0.8599504875549783,
          0.8667392051281828,
          0.873928789339177,
          0.8821515986281313,
          0.8911429055838948,
          0.8970593951159853,
          0.9117632032322649,
          0.9233088017589188,
          0.9408214873116122,
          0.9464432106153376,
          0.9588084003566117,
          0.9738555879797565,
          0.9873346942272493,
          1.0005117587897308,
          1.0122747391422395,
          1.0270206567197142,
          1.0359608701276664,
          1.055739970659861,
          1.0646397281194921,
          1.0900286466627183,
          1.1153379761196018,
          1.1281355043603702,
          1.1404914344295491,
          1.1554427183059932,
          1.1859189447920195,
          1.2113891059344621,
          1.2321088348236646,
          1.2521026715611152
        ],
        "population_cv": 0.12455242949918699
      },
      "bits_per_byte": 7.95915895791699,
      "branch_output_rms_by_sublayer": [
        0.0022265268489718437,
        0.002717922907322645,
        0.0021652730647474527,
        0.002733128145337105,
        0.0025650423485785723,
        0.002695153234526515,
        0.002537536434829235,
        0.002696225419640541,
        0.0027307753916829824,
        0.0027451792266219854,
        0.0029185698367655277,
        0.0026901187375187874,
        0.002936159959062934,
        0.0027115964330732822,
        0.0030124466866254807,
        0.0027617362793534994,
        0.0031113612931221724,
        0.002723218873143196,
        0.0034121612552553415,
        0.0027921765577048063,
        0.0034701526165008545,
        0.002744668396189809,
        0.0039346772246062756,
        0.0027058592531830072,
        0.004295050632208586,
        0.0026920263189822435,
        0.004176756367087364,
        0.0027182905469089746,
        0.004023758694529533,
        0.0027150253299623728,
        0.004447397775948048,
        0.0027387896552681923,
        0.004260339308530092,
        0.00275799329392612,
        0.005327676888555288,
        0.002721488941460848,
        0.00474865036085248,
        0.0027454239316284657,
        0.00498866755515337,
        0.002719792304560542,
        0.0052231852896511555,
        0.0027601351030170918,
        0.005184618756175041,
        0.002755985129624605,
        0.0052438946440815926,
        0.0027470553759485483,
        0.0059704589657485485,
        0.0027777766808867455,
        0.0060942270793020725,
        0.0026787330862134695,
        0.005698176100850105,
        0.002644060645252466,
        0.006003713235259056,
        0.002661966485902667,
        0.006405353546142578,
        0.002804658142849803,
        0.006667613051831722,
        0.0026854604948312044,
        0.006290186662226915,
        0.0027102255262434483,
        0.0072579155676066875,
        0.0027289679273962975,
        0.006407012697309256,
        0.0026884053368121386
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.005888019831616423,
        0.005779358340731243,
        0.0060348395662513914,
        0.005586875351280001,
        0.005559071516712478,
        0.0054082748429522395,
        0.005481135467826832,
        0.0053805923812947415,
        0.004971746645687392,
        0.005439263395635174,
        0.005139424262259116,
        0.005696496994849179,
        0.005782818048228185,
        0.0054294510739720785,
        0.005067071208887471,
        0.005218073376917496,
        0.005181036474299639,
        0.005605812513830226,
        0.005170579144406511,
        0.004834742615880465,
        0.005150211089068095,
        0.00476651480071292,
        0.004776402486441839,
        0.00519066249960807,
        0.0053994045182471796,
        0.004795508114414784,
        0.004895792813412353,
        0.00496559115560922,
        0.00502742614163479,
        0.004657478744424888,
        0.005287606581362069,
        0.004870555415551657
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.005639770912333169,
        "first_to_last_ratio": 1.130799174678946,
        "imbalance_abs_log_ratio": 0.12292461698112994,
        "last_quartile_mean": 0.0049874204355821175,
        "mean": 0.005263682419187692,
        "normalized": [
          1.1186122875029152,
          1.0979686615711766,
          1.146505257280075,
          1.061400537941684,
          1.0561183357962487,
          1.0274698228824493,
          1.0413119621059315,
          1.0222106792919112,
          0.9445377303850805,
          1.0333570611721248,
          0.976393302818647,
          1.0822265747043835,
          1.0986259404914112,
          1.0314929058371969,
          0.962647592570647,
          0.9913351455049914,
          0.9842988352437859,
          1.0649982402804865,
          0.9823121405573801,
          0.9185095586801336,
          0.9784425956805525,
          0.9055475655859389,
          0.9074260386664719,
          0.9861275977985596,
          1.0257846291342996,
          0.9110557462459602,
          0.9301079403205877,
          0.9433683038148654,
          0.9551157804103686,
          0.884832779319472,
          1.0045451378463044,
          0.9253133125579595
        ],
        "population_cv": 0.06750436074287791
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.028027428314089775,
        0.028122643008828163,
        0.028270306065678596,
        0.028331812471151352,
        0.0284538883715868,
        0.02856677956879139,
        0.02867252007126808,
        0.02870391495525837,
        0.02881420962512493,
        0.028933145105838776,
        0.029061447829008102,
        0.029177924618124962,
        0.02929086796939373,
        0.029417123645544052,
        0.029533835127949715,
        0.02969702146947384,
        0.029811719432473183,
        0.02999095991253853,
        0.030115574598312378,
        0.0302139800041914,
        0.030315518379211426,
        0.03069150820374489,
        0.030812423676252365,
        0.03109365701675415,
        0.03120259940624237,
        0.03165328875184059,
        0.03179442882537842,
        0.03191109746694565,
        0.03198441118001938,
        0.03227011114358902,
        0.03240228444337845,
        0.03283532336354256,
        0.0329107940196991,
        0.03322411701083183,
        0.033366311341524124,
        0.0337243415415287,
        0.033811621367931366,
        0.03413747623562813,
        0.03420914337038994,
        0.03456577658653259,
        0.03470747172832489,
        0.03486170619726181,
        0.03500960022211075,
        0.03554223105311394,
        0.03567802160978317,
        0.03587620332837105,
        0.035978782922029495,
        0.036742135882377625,
        0.036836784332990646,
        0.03761643171310425,
        0.03769209608435631,
        0.03803960233926773,
        0.038124579936265945,
        0.03846931457519531,
        0.038542140275239944,
        0.03895273432135582,
        0.03904740884900093,
        0.03994588926434517,
        0.0400773324072361,
        0.04087882116436958,
        0.04093807935714722,
        0.04153488203883171,
        0.04163828864693642,
        0.042190924286842346
      ],
      "loss_nats": 5.516868591308594,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 0,
      "stream_state_rms_by_sublayer": [
        0.028122643008828163,
        0.028270306065678596,
        0.028331812471151352,
        0.0284538883715868,
        0.02856677956879139,
        0.02867252007126808,
        0.02870391495525837,
        0.02881420962512493,
        0.028933145105838776,
        0.029061447829008102,
        0.029177924618124962,
        0.02929086796939373,
        0.029417123645544052,
        0.029533835127949715,
        0.02969702146947384,
        0.029811719432473183,
        0.02999095991253853,
        0.030115574598312378,
        0.0302139800041914,
        0.030315518379211426,
        0.03069150820374489,
        0.030812423676252365,
        0.03109365701675415,
        0.03120259940624237,
        0.03165328875184059,
        0.03179442882537842,
        0.03191109746694565,
        0.03198441118001938,
        0.03227011114358902,
        0.03240228444337845,
        0.03283532336354256,
        0.0329107940196991,
        0.03322411701083183,
        0.033366311341524124,
        0.0337243415415287,
        0.033811621367931366,
        0.03413747623562813,
        0.03420914337038994,
        0.03456577658653259,
        0.03470747172832489,
        0.03486170619726181,
        0.03500960022211075,
        0.03554223105311394,
        0.03567802160978317,
        0.03587620332837105,
        0.035978782922029495,
        0.036742135882377625,
        0.036836784332990646,
        0.03761643171310425,
        0.03769209608435631,
        0.03803960233926773,
        0.038124579936265945,
        0.03846931457519531,
        0.038542140275239944,
        0.03895273432135582,
        0.03904740884900093,
        0.03994588926434517,
        0.0400773324072361,
        0.04087882116436958,
        0.04093807935714722,
        0.04153488203883171,
        0.04163828864693642,
        0.042190924286842346,
        0.042313966900110245
      ]
    },
    {
      "activation_grad_rms_by_block": [
        7.172436016844586e-05,
        6.834617670392618e-05,
        6.473411485785618e-05,
        6.156626477604732e-05,
        5.8372206694912165e-05,
        5.494635479408316e-05,
        5.167145354789682e-05,
        4.809222446056083e-05,
        4.4674157834379e-05,
        4.1754625272005796e-05,
        3.8718964788131416e-05,
        3.608527549658902e-05,
        3.353153806529008e-05,
        3.1093299185158685e-05,
        2.904756365751382e-05,
        2.7216165108256973e-05,
        2.5600058506825007e-05,
        2.4208331524278037e-05,
        2.304886766069103e-05,
        2.1986410501995124e-05,
        2.1054642274975777e-05,
        2.025439971475862e-05,
        1.956623418664094e-05,
        1.9002633052878082e-05,
        1.8555338101577945e-05,
        1.8124846974387765e-05,
        1.7765629308996722e-05,
        1.744573273754213e-05,
        1.7174377717310563e-05,
        1.6944877643254586e-05,
        1.6726755347917788e-05,
        1.656309177633375e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 5.9931644500466064e-05,
        "first_to_last_ratio": 3.4418587232383575,
        "imbalance_abs_log_ratio": 1.2360116519842717,
        "last_quartile_mean": 1.7412581200915156e-05,
        "mean": 3.3612405388794286e-05,
        "normalized": [
          2.1338657361414946,
          2.033361668508004,
          1.9258995037420696,
          1.831653047852752,
          1.7366268798594315,
          1.634704632367703,
          1.537273305799265,
          1.4307879458276982,
          1.3290973174229443,
          1.242238536309156,
          1.1519248426368067,
          1.0735701619444094,
          0.9975941226886675,
          0.9250542716447356,
          0.8641917566303572,
          0.8097059640167944,
          0.761625305023828,
          0.7202201462305526,
          0.6857250290208349,
          0.654115950574753,
          0.626394988143127,
          0.602587035366146,
          0.582113477459246,
          0.5653458249439413,
          0.5520383884148895,
          0.5392308811207612,
          0.5285438249212427,
          0.5190266074607738,
          0.510953545830914,
          0.5041257073765888,
          0.49763636831222313,
          0.4927672264078892
        ],
        "population_cv": 0.513984504821457
      },
      "activation_output_rms_by_block": [
        0.029439672827720642,
        0.030752120539546013,
        0.03267423436045647,
        0.034982286393642426,
        0.03797784447669983,
        0.04108294099569321,
        0.044510453939437866,
        0.04866119846701622,
        0.05275992304086685,
        0.05702463164925575,
        0.061875153332948685,
        0.06669966131448746,
        0.071797676384449,
        0.07734881341457367,
        0.08311311900615692,
        0.08836977183818817,
        0.09436330199241638,
        0.10090246796607971,
        0.10806354880332947,
        0.11583591252565384,
        0.1238454207777977,
        0.13231202960014343,
        0.1417466551065445,
        0.15185882151126862,
        0.16391249001026154,
        0.1746179163455963,
        0.18763093650341034,
        0.20137503743171692,
        0.21650369465351105,
        0.23270079493522644,
        0.2497490495443344,
        0.2673082947731018
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.037510094000026584,
        "first_to_last_ratio": 0.17716440452291277,
        "imbalance_abs_log_ratio": 1.7306771383999247,
        "last_quartile_mean": 0.21172477677464485,
        "mean": 0.11005612107692286,
        "normalized": [
          0.2674969146617844,
          0.27942217332966024,
          0.29688702491721536,
          0.3178586165979067,
          0.3450770761778485,
          0.37329083192909013,
          0.4044341514483109,
          0.4421489508339555,
          0.4793910827003499,
          0.5181413908763769,
          0.5622145567869099,
          0.6060513550888119,
          0.6523733135594211,
          0.702812462021184,
          0.7551885183020911,
          0.8029519028425762,
          0.857410756158323,
          0.9168274056792781,
          0.9818949436515149,
          1.0525167650120182,
          1.1252933463940358,
          1.2022232685055743,
          1.2879488548162785,
          1.3798307629352853,
          1.4893536897933755,
          1.586626119809806,
          1.7048659780791944,
          1.8297486360705681,
          1.9672117510137168,
          2.114383031658748,
          2.269288133185925,
          2.428836235162865
        ],
        "population_cv": 0.6268498746310547
      },
      "bits_per_byte": 5.1517691996807775,
      "branch_output_rms_by_sublayer": [
        0.0028861628379672766,
        0.004992613103240728,
        0.002704753540456295,
        0.005112498998641968,
        0.002895323559641838,
        0.005543495528399944,
        0.0029729304369539022,
        0.005716898012906313,
        0.00318311620503664,
        0.005760234780609608,
        0.003378538880497217,
        0.0057550352066755295,
        0.0039320578798651695,
        0.005740197841078043,
        0.0034711055923253298,
        0.006492512300610542,
        0.0037959336768835783,
        0.006124566774815321,
        0.004324590787291527,
        0.0062123509123921394,
        0.004316650331020355,
        0.006645897403359413,
        0.0046909768134355545,
        0.006303280591964722,
        0.0053819348104298115,
        0.006548910401761532,
        0.005383738782256842,
        0.006836008746176958,
        0.00522694643586874,
        0.006793942768126726,
        0.005513017997145653,
        0.006699956953525543,
        0.005531978793442249,
        0.0068945568054914474,
        0.006940840277820826,
        0.0070524453185498714,
        0.006634449120610952,
        0.007447477430105209,
        0.0070588430389761925,
        0.007686031982302666,
        0.007738462649285793,
        0.007927682250738144,
        0.008234621025621891,
        0.008245103061199188,
        0.008838997222483158,
        0.008133068680763245,
        0.009296931326389313,
        0.008390788920223713,
        0.013193055056035519,
        0.008053022436797619,
        0.00934307835996151,
        0.008179600350558758,
        0.012704792432487011,
        0.008694498799741268,
        0.012802425771951675,
        0.009316418319940567,
        0.014256707392632961,
        0.008604929782450199,
        0.015807906165719032,
        0.009046261198818684,
        0.0164310485124588,
        0.009124966338276863,
        0.01581309363245964,
        0.009515758603811264
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0004018231655400003,
        0.0003405186139403606,
        0.0003221632759320079,
        0.0003173401597627506,
        0.0002956526561473367,
        0.0002913580859417252,
        0.00030034368266542127,
        0.00026684124508166604,
        0.0002497190139849492,
        0.00026578754174316435,
        0.0002404550275593826,
        0.00024493210822064133,
        0.0002532169075836586,
        0.00023865218150919594,
        0.0002121496513808785,
        0.0002091708005241373,
        0.0001980500112256327,
        0.00020258502349527826,
        0.00019483821933596074,
        0.0001851504169444552,
        0.00018439346409065922,
        0.00017525536186022104,
        0.000171573793780652,
        0.0001670148315601783,
        0.00018406810665917416,
        0.00014791927954418835,
        0.00017020274695916728,
        0.00015649953794173187,
        0.00016066786897129027,
        0.00016871534344057424,
        0.00016946426752177343,
        0.0001617385113992835
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00031700511062640856,
        "first_to_last_ratio": 1.9222979375866573,
        "imbalance_abs_log_ratio": 0.653521312887717,
        "last_quartile_mean": 0.0001649094578046479,
        "mean": 0.00022650815319523427,
        "normalized": [
          1.773989853661721,
          1.5033393241560593,
          1.4223032212634112,
          1.401009876625613,
          1.305262755343313,
          1.286302862972861,
          1.3259729437047942,
          1.1780646361619815,
          1.1024725179305521,
          1.1734126917456873,
          1.061573387833537,
          1.0813390368757583,
          1.117915200895233,
          1.0536140891294732,
          0.9366093378458668,
          0.9234581518301755,
          0.8743615116358631,
          0.8943829201621014,
          0.8601819254074433,
          0.8174117104953322,
          0.814069875585118,
          0.7737265055936556,
          0.7574729269580301,
          0.7373457829406393,
          0.8126334706394442,
          0.6530417446682029,
          0.7514199579935843,
          0.6909223166321972,
          0.7093248817088087,
          0.7448532914184036,
          0.7481596804849096,
          0.7140516097002307
        ],
        "population_cv": 0.27857383691000986
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.0285640936344862,
        0.02861291915178299,
        0.029439672827720642,
        0.02950597181916237,
        0.030752120539546013,
        0.030888807028532028,
        0.03267423436045647,
        0.03270668536424637,
        0.034982286393642426,
        0.03526782616972923,
        0.03797784447669983,
        0.03822308033704758,
        0.04108294099569321,
        0.041396427899599075,
        0.044510453939437866,
        0.04485016688704491,
        0.04866119846701622,
        0.04912792518734932,
        0.05275992304086685,
        0.05317053198814392,
        0.05702463164925575,
        0.05767740681767464,
        0.061875153332948685,
        0.06268688291311264,
        0.06669966131448746,
        0.06753430515527725,
        0.071797676384449,
        0.07279384881258011,
        0.07734881341457367,
        0.078476682305336,
        0.08311311900615692,
        0.08398299664258957,
        0.08836977183818817,
        0.08972389250993729,
        0.09436330199241638,
        0.09631941467523575,
        0.10090246796607971,
        0.10291995853185654,
        0.10806354880332947,
        0.11053454130887985,
        0.11583591252565384,
        0.11846786737442017,
        0.1238454207777977,
        0.12666596472263336,
        0.13231202960014343,
        0.13596534729003906,
        0.1417466551065445,
        0.1458624303340912,
        0.15185882151126862,
        0.1581486165523529,
        0.16391249001026154,
        0.16865970194339752,
        0.1746179163455963,
        0.18122267723083496,
        0.18763093650341034,
        0.19437597692012787,
        0.20137503743171692,
        0.2100502997636795,
        0.21650369465351105,
        0.22599515318870544,
        0.23270079493522644,
        0.2427441030740738,
        0.2497490495443344,
        0.2599930763244629
      ],
      "loss_nats": 3.570934295654297,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 100,
      "stream_state_rms_by_sublayer": [
        0.02861291915178299,
        0.029439672827720642,
        0.02950597181916237,
        0.030752120539546013,
        0.030888807028532028,
        0.03267423436045647,
        0.03270668536424637,
        0.034982286393642426,
        0.03526782616972923,
        0.03797784447669983,
        0.03822308033704758,
        0.04108294099569321,
        0.041396427899599075,
        0.044510453939437866,
        0.04485016688704491,
        0.04866119846701622,
        0.04912792518734932,
        0.05275992304086685,
        0.05317053198814392,
        0.05702463164925575,
        0.05767740681767464,
        0.061875153332948685,
        0.06268688291311264,
        0.06669966131448746,
        0.06753430515527725,
        0.071797676384449,
        0.07279384881258011,
        0.07734881341457367,
        0.078476682305336,
        0.08311311900615692,
        0.08398299664258957,
        0.08836977183818817,
        0.08972389250993729,
        0.09436330199241638,
        0.09631941467523575,
        0.10090246796607971,
        0.10291995853185654,
        0.10806354880332947,
        0.11053454130887985,
        0.11583591252565384,
        0.11846786737442017,
        0.1238454207777977,
        0.12666596472263336,
        0.13231202960014343,
        0.13596534729003906,
        0.1417466551065445,
        0.1458624303340912,
        0.15185882151126862,
        0.1581486165523529,
        0.16391249001026154,
        0.16865970194339752,
        0.1746179163455963,
        0.18122267723083496,
        0.18763093650341034,
        0.19437597692012787,
        0.20137503743171692,
        0.2100502997636795,
        0.21650369465351105,
        0.22599515318870544,
        0.23270079493522644,
        0.2427441030740738,
        0.2497490495443344,
        0.2599930763244629,
        0.2673082947731018
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.00014451092283707112,
        0.00014379274216480553,
        0.00014628663484472781,
        0.0001445338421035558,
        0.00013843078340869397,
        0.0001292457600357011,
        0.00012031046207994223,
        0.00011329296830808744,
        0.0001077206979971379,
        0.0001003394863801077,
        9.358100214740261e-05,
        8.73711978783831e-05,
        8.221827738452703e-05,
        7.71504346630536e-05,
        7.201205153251067e-05,
        6.728569132974371e-05,
        6.256035703700036e-05,
        5.796365803689696e-05,
        5.370515646063723e-05,
        4.9705678975442424e-05,
        4.572505713440478e-05,
        4.197436646791175e-05,
        3.853226371575147e-05,
        3.558541357051581e-05,
        3.2570587791269645e-05,
        2.9896507840021513e-05,
        2.8078287868993357e-05,
        2.6684616386773996e-05,
        2.568897070887033e-05,
        2.5203555196640082e-05,
        2.4927356207626872e-05,
        2.5036461011040956e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00013505051447282312,
        "first_to_last_ratio": 4.954020049421058,
        "imbalance_abs_log_ratio": 1.6001993781766728,
        "last_quartile_mean": 2.7260792876404594e-05,
        "mean": 7.412253904703903e-05,
        "normalized": [
          1.9496218652920512,
          1.9399327655728709,
          1.9735783032458267,
          1.9499310730820074,
          1.8675936521932173,
          1.743676912547211,
          1.6231292617160862,
          1.5284550389752622,
          1.4532785760182485,
          1.3536973728926245,
          1.2625174926619152,
          1.1787399487615544,
          1.1092210067487078,
          1.040849863684419,
          0.9715270477554875,
          0.90776290443914,
          0.8440126018524382,
          0.7819977402570161,
          0.7245455586263081,
          0.670587915828121,
          0.616884657787925,
          0.5662834410094119,
          0.5198454371793504,
          0.4800889719648283,
          0.4394154357097774,
          0.4033389603808477,
          0.3788090401379066,
          0.36000677701879086,
          0.34657434889767885,
          0.34002552422881266,
          0.3362992758762309,
          0.3377712276579264
        ],
        "population_cv": 0.5690574657123064
      },
      "activation_output_rms_by_block": [
        0.049516160041093826,
        0.057937320321798325,
        0.06586314737796783,
        0.0719481110572815,
        0.076947420835495,
        0.08064281940460205,
        0.08439230173826218,
        0.08712083846330643,
        0.0897950753569603,
        0.09305135160684586,
        0.09555979818105698,
        0.09745962917804718,
        0.10008277744054794,
        0.10209457576274872,
        0.1030229851603508,
        0.10483334213495255,
        0.10759729146957397,
        0.11058001965284348,
        0.11383581161499023,
        0.1180567741394043,
        0.1237289309501648,
        0.13110940158367157,
        0.14003770053386688,
        0.15068796277046204,
        0.16247524321079254,
        0.1767880618572235,
        0.1922861486673355,
        0.20968382060527802,
        0.22965215146541595,
        0.2505997121334076,
        0.27601101994514465,
        0.307341605424881
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.07179601490497589,
        "first_to_last_ratio": 0.31823808816289667,
        "imbalance_abs_log_ratio": 1.1449554714268964,
        "last_quartile_mean": 0.22560472041368484,
        "mean": 0.13002310344018042,
        "normalized": [
          0.3808258588741859,
          0.44559250463094413,
          0.506549571847971,
          0.5533486676879896,
          0.5917980635718029,
          0.6202191554495797,
          0.6490562023624398,
          0.6700412169702441,
          0.6906086147856962,
          0.7156524428726306,
          0.7349447571448028,
          0.7495562450013764,
          0.7697307231756155,
          0.785203345109504,
          0.7923436868875267,
          0.8062670353287107,
          0.8275244062227459,
          0.8504643923048483,
          0.8755044957634206,
          0.9079676689436854,
          0.9515918915678598,
          1.008354654786339,
          1.0770216740619014,
          1.1589322111496045,
          1.2495874880077935,
          1.359666529868349,
          1.4788613990882042,
          1.6126658651994643,
          1.7662411170724883,
          1.9273475674936549,
          2.1227844332459633,
          2.363746113522658
        ],
        "population_cv": 0.4886061625493153
      },
      "bits_per_byte": 3.0864900833624125,
      "branch_output_rms_by_sublayer": [
        0.01111105177551508,
        0.01691170036792755,
        0.01229762565344572,
        0.013536608777940273,
        0.012528475373983383,
        0.011229105293750763,
        0.017575321719050407,
        0.009298570454120636,
        0.015520633198320866,
        0.008989781141281128,
        0.012922349385917187,
        0.008074384182691574,
        0.013718407601118088,
        0.0077475146390497684,
        0.012880085967481136,
        0.00855444185435772,
        0.019964123144745827,
        0.007989896461367607,
        0.02377571351826191,
        0.008533375337719917,
        0.02730517089366913,
        0.0086820675060153,
        0.02310827188193798,
        0.008458662778139114,
        0.019315505400300026,
        0.008968821726739407,
        0.031517889350652695,
        0.009528025984764099,
        0.018633879721164703,
        0.008953212760388851,
        0.016267014667391777,
        0.008700341917574406,
        0.016277894377708435,
        0.010046470910310745,
        0.022248439490795135,
        0.010587109252810478,
        0.016605904325842857,
        0.012151478789746761,
        0.01819506660103798,
        0.012493276968598366,
        0.017910068854689598,
        0.013204623945057392,
        0.017647475004196167,
        0.015671590343117714,
        0.02343275025486946,
        0.016670797020196915,
        0.03074510768055916,
        0.019418122246861458,
        0.017775433138012886,
        0.019835535436868668,
        0.017164910212159157,
        0.021999742835760117,
        0.01125776395201683,
        0.023901214823126793,
        0.007435314357280731,
        0.02528192289173603,
        0.006950905080884695,
        0.027673469856381416,
        0.007996064610779285,
        0.026783082634210587,
        0.010837847366929054,
        0.030296670272946358,
        0.021106943488121033,
        0.028989506885409355
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0006755996472372422,
        0.0003902215048764158,
        0.00033650809592979206,
        0.00029675556364760884,
        0.00026186200490798707,
        0.0002455230107742763,
        0.00021966924842671773,
        0.00022418062520577193,
        0.00022548545903395049,
        0.00022091596273961857,
        0.00024014950731391624,
        0.00021478395214610848,
        0.0001956916368008953,
        0.00020741213919383669,
        0.0001699546796737774,
        0.00016808908215382032,
        0.00015850476160987511,
        0.00015061268581609982,
        0.00013300713992511992,
        0.0001284985461118983,
        0.0001312583398227583,
        0.00012527717278626568,
        0.0001234119340024874,
        0.00012326054887191204,
        0.00010914072106764207,
        0.00010077507233480893,
        9.59134879196041e-05,
        8.362219426315277e-05,
        8.733675859901561e-05,
        7.698995894839292e-05,
        8.92787325176121e-05,
        9.631544502550806e-05
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.0003312899626257265,
        "first_to_last_ratio": 3.584553340265607,
        "imbalance_abs_log_ratio": 1.276633875010675,
        "last_quartile_mean": 9.242154633446708e-05,
        "mean": 0.00019081267561512152,
        "normalized": [
          3.540643435030279,
          2.045050223306504,
          1.763552105985586,
          1.5552193411206041,
          1.3723512028947988,
          1.2867227503769625,
          1.1512298526215383,
          1.17487281430902,
          1.1817111117332983,
          1.1577635606620644,
          1.2585616051960276,
          1.1256272753039647,
          1.0255693767201026,
          1.0869935056735807,
          0.8906886249872848,
          0.8809115097409158,
          0.8306825586869656,
          0.7893222257408777,
          0.697056102254977,
          0.6734277253733716,
          0.6878910921385158,
          0.6565453389425546,
          0.6467701037399387,
          0.6459767333305183,
          0.5719783589628198,
          0.5281361524329609,
          0.5026578428839092,
          0.43824234419217944,
          0.45770941745598764,
          0.4034845101364514,
          0.4678868017012227,
          0.5047643963642175
        ],
        "population_cv": 0.6065148734399883
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.043906208127737045,
        0.04462560638785362,
        0.049516160041093826,
        0.052572574466466904,
        0.057937320321798325,
        0.060727521777153015,
        0.06586314737796783,
        0.06803189963102341,
        0.0719481110572815,
        0.07307156920433044,
        0.076947420835495,
        0.07774755358695984,
        0.08064281940460205,
        0.08214583247900009,
        0.08439230173826218,
        0.08468613773584366,
        0.08712083846330643,
        0.08735909312963486,
        0.0897950753569603,
        0.09098032861948013,
        0.09305135160684586,
        0.09323208779096603,
        0.09555979818105698,
        0.09555087983608246,
        0.09745962917804718,
        0.09809984266757965,
        0.10008277744054794,
        0.09976384043693542,
        0.10209457576274872,
        0.10044334083795547,
        0.1030229851603508,
        0.1023804098367691,
        0.10483334213495255,
        0.10451660305261612,
        0.10759729146957397,
        0.10710933804512024,
        0.11058001965284348,
        0.10916344821453094,
        0.11383581161499023,
        0.11319266259670258,
        0.1180567741394043,
        0.11788330227136612,
        0.1237289309501648,
        0.12335778772830963,
        0.13110940158367157,
        0.13146334886550903,
        0.14003770053386688,
        0.13964787125587463,
        0.15068796277046204,
        0.15031130611896515,
        0.16247524321079254,
        0.16256581246852875,
        0.1767880618572235,
        0.1762392520904541,
        0.1922861486673355,
        0.19244804978370667,
        0.20968382060527802,
        0.21054796874523163,
        0.22965215146541595,
        0.23175573348999023,
        0.2505997121334076,
        0.2547222673892975,
        0.27601101994514465,
        0.28623583912849426
      ],
      "loss_nats": 2.1393918991088867,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 500,
      "stream_state_rms_by_sublayer": [
        0.04462560638785362,
        0.049516160041093826,
        0.052572574466466904,
        0.057937320321798325,
        0.060727521777153015,
        0.06586314737796783,
        0.06803189963102341,
        0.0719481110572815,
        0.07307156920433044,
        0.076947420835495,
        0.07774755358695984,
        0.08064281940460205,
        0.08214583247900009,
        0.08439230173826218,
        0.08468613773584366,
        0.08712083846330643,
        0.08735909312963486,
        0.0897950753569603,
        0.09098032861948013,
        0.09305135160684586,
        0.09323208779096603,
        0.09555979818105698,
        0.09555087983608246,
        0.09745962917804718,
        0.09809984266757965,
        0.10008277744054794,
        0.09976384043693542,
        0.10209457576274872,
        0.10044334083795547,
        0.1030229851603508,
        0.1023804098367691,
        0.10483334213495255,
        0.10451660305261612,
        0.10759729146957397,
        0.10710933804512024,
        0.11058001965284348,
        0.10916344821453094,
        0.11383581161499023,
        0.11319266259670258,
        0.1180567741394043,
        0.11788330227136612,
        0.1237289309501648,
        0.12335778772830963,
        0.13110940158367157,
        0.13146334886550903,
        0.14003770053386688,
        0.13964787125587463,
        0.15068796277046204,
        0.15031130611896515,
        0.16247524321079254,
        0.16256581246852875,
        0.1767880618572235,
        0.1762392520904541,
        0.1922861486673355,
        0.19244804978370667,
        0.20968382060527802,
        0.21054796874523163,
        0.22965215146541595,
        0.23175573348999023,
        0.2505997121334076,
        0.2547222673892975,
        0.27601101994514465,
        0.28623583912849426,
        0.307341605424881
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.00010485891834832728,
        0.00010495351307326928,
        0.00011622957390500233,
        0.00011138651461806148,
        0.00010787966311909258,
        0.000104747356090229,
        0.00010223284334642813,
        9.973083797376603e-05,
        9.735774074215442e-05,
        9.589978435542434e-05,
        9.408760524820536e-05,
        9.229916759068146e-05,
        9.143509669229388e-05,
        8.827183046378195e-05,
        8.612661622464657e-05,
        8.33196536405012e-05,
        8.087792957667261e-05,
        7.82960414653644e-05,
        8.221158350352198e-05,
        7.83153300289996e-05,
        7.789687515469268e-05,
        7.347232894971967e-05,
        6.715180643368512e-05,
        6.132270209491253e-05,
        5.5257889471249655e-05,
        5.0392085540806875e-05,
        4.541657472145744e-05,
        4.046381582156755e-05,
        3.4924181818496436e-05,
        2.9514063498936594e-05,
        2.4251618015114218e-05,
        2.1234091036603786e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00010650240255927201,
        "first_to_last_ratio": 2.8263626166920495,
        "imbalance_abs_log_ratio": 1.038990590500678,
        "last_quartile_mean": 3.768178999052907e-05,
        "mean": 7.755673851761458e-05,
        "normalized": [
          1.3520284678359944,
          1.353248151989171,
          1.4986392688316106,
          1.4361938981325721,
          1.3909773048874547,
          1.3505900079390127,
          1.3181684183793927,
          1.2859080961883835,
          1.255309887999515,
          1.236511310150616,
          1.213145460298544,
          1.1900857276214447,
          1.1789445822496427,
          1.1381581040019337,
          1.1104981703825207,
          1.0743057951254327,
          1.0428227272386361,
          1.0095324140994133,
          1.060018575753417,
          1.009781116713833,
          1.0043856490561534,
          0.947336496531926,
          0.8658410309303259,
          0.7906818062106236,
          0.7124834092746121,
          0.6497447740064765,
          0.585591601574923,
          0.5217317875271078,
          0.4503049314092096,
          0.380548022816011,
          0.3126951761851902,
          0.2737878286589002
        ],
        "population_cv": 0.34367475536074504
      },
      "activation_output_rms_by_block": [
        0.09794864058494568,
        0.13700027763843536,
        0.20758503675460815,
        0.22207145392894745,
        0.22574232518672943,
        0.2265917956829071,
        0.22746813297271729,
        0.22716933488845825,
        0.22722382843494415,
        0.22613705694675446,
        0.22211141884326935,
        0.21741008758544922,
        0.21311214566230774,
        0.20640577375888824,
        0.19932369887828827,
        0.19335229694843292,
        0.19345322251319885,
        0.19645750522613525,
        0.20112650096416473,
        0.2038118690252304,
        0.20720544457435608,
        0.21031972765922546,
        0.2129739224910736,
        0.21647654473781586,
        0.22099705040454865,
        0.23361735045909882,
        0.2512694001197815,
        0.2773953676223755,
        0.3133004307746887,
        0.35808902978897095,
        0.4303363263607025,
        0.54518723487854
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.1964471247047186,
        "first_to_last_ratio": 0.5975141297159507,
        "imbalance_abs_log_ratio": 0.5149773473959137,
        "last_quartile_mean": 0.32877402380108833,
        "mean": 0.2358959447592497,
        "normalized": [
          0.4152196880065486,
          0.5807657176059338,
          0.879985609614718,
          0.9413958097312304,
          0.9569572101678865,
          0.9605582491643161,
          0.964273180723248,
          0.9630065286637394,
          0.9632375353753703,
          0.958630540162719,
          0.9415652272867661,
          0.9216355448896473,
          0.9034158933075572,
          0.8749865283591098,
          0.8449644994182233,
          0.8196507877478102,
          0.8200786270854848,
          0.832814253872128,
          0.8526068609167068,
          0.8639905583507864,
          0.8783764586789548,
          0.8915783943376951,
          0.9028299435517223,
          0.9176781099766134,
          0.9368412442617167,
          0.9903406804959011,
          1.065170494457722,
          1.1759225785143497,
          1.32812978660755,
          1.5179956999872501,
          1.824263349672647,
          2.311134409007948
        ],
        "population_cv": 0.335075401661643
      },
      "bits_per_byte": 2.08044695422856,
      "branch_output_rms_by_sublayer": [
        0.018880587071180344,
        0.05959044396877289,
        0.060257185250520706,
        0.016382697969675064,
        0.09490419179201126,
        0.0035255588591098785,
        0.0323294997215271,
        0.004731281660497189,
        0.02052305079996586,
        0.00493080448359251,
        0.01820369064807892,
        0.005357450805604458,
        0.019246362149715424,
        0.006479416508227587,
        0.024384424090385437,
        0.009427462704479694,
        0.017510300502181053,
        0.009214801713824272,
        0.01695968210697174,
        0.0109269879758358,
        0.019068937748670578,
        0.012604814022779465,
        0.019973911345005035,
        0.013778526335954666,
        0.022227400913834572,
        0.013526024296879768,
        0.027813225984573364,
        0.017681727185845375,
        0.027307361364364624,
        0.018625596538186073,
        0.03342962637543678,
        0.019818320870399475,
        0.030618051066994667,
        0.021072659641504288,
        0.025097575038671494,
        0.022579019889235497,
        0.025198329240083694,
        0.01933942921459675,
        0.03349718078970909,
        0.022255338728427887,
        0.03462665155529976,
        0.024717653170228004,
        0.03921883553266525,
        0.025107676163315773,
        0.043981269001960754,
        0.0301958154886961,
        0.038896940648555756,
        0.03620966896414757,
        0.04658674821257591,
        0.046204980462789536,
        0.033179834485054016,
        0.04984369874000549,
        0.02075173333287239,
        0.05490582063794136,
        0.011197025887668133,
        0.06379986554384232,
        0.012507086619734764,
        0.07197272032499313,
        0.011961712501943111,
        0.07922235131263733,
        0.010508585721254349,
        0.11548902839422226,
        0.021868467330932617,
        0.1684120148420334
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00040070945427075937,
        0.00021125547826331077,
        0.00020379424936248455,
        0.00011927406183950472,
        9.446000613801463e-05,
        0.00010295610627477164,
        9.818733926828671e-05,
        0.00012673421109679544,
        0.00011573807851122185,
        0.00011146965874368272,
        0.0001232427596852727,
        0.00012517422740451753,
        0.000137325623762558,
        0.0001763842480315643,
        0.00017420922346804142,
        0.00019637547238758496,
        0.00020814888113781,
        0.0002138469295217413,
        0.0002110578549773679,
        0.0002602958562879386,
        0.0002719585176535882,
        0.00027346390406242154,
        0.00028937535779988523,
        0.00028561131716422746,
        0.00028061956919165,
        0.00023874030969169658,
        0.00021716224392712217,
        0.0001981302869037355,
        0.00018664421118786767,
        0.0001673158872138231,
        0.0001540361578610053,
        0.00014575183058398037
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00016967136331424098,
        "first_to_last_ratio": 0.8545520537501935,
        "imbalance_abs_log_ratio": 0.1571778610785998,
        "last_quartile_mean": 0.0001985500620701101,
        "mean": 0.00019123279105231976,
        "normalized": [
          2.0954013799920355,
          1.104703210682696,
          1.0656867383519393,
          0.6237113477409442,
          0.4939529754192163,
          0.5383810261212141,
          0.5134440528111281,
          0.6627221743687356,
          0.605220882225982,
          0.582900339059453,
          0.6444645764311125,
          0.6545646628682571,
          0.71810709348998,
          0.9223535726323577,
          0.9109798717542076,
          1.0268922568507521,
          1.0884581038241612,
          1.1182545019867147,
          1.1036697933233854,
          1.3611465630741315,
          1.4221332866452938,
          1.4300052964640568,
          1.5132099270606496,
          1.493526897727945,
          1.4674239059496585,
          1.2484276800958218,
          1.1355910392360917,
          1.0360685832877303,
          0.976005266465014,
          0.8749330399515364,
          0.8054902980465429,
          0.7621696560612549
        ],
        "population_cv": 0.3661316865844204
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.0565960556268692,
        0.06250853836536407,
        0.09794864058494568,
        0.12857383489608765,
        0.13700027763843536,
        0.2065405696630478,
        0.20758503675460815,
        0.22102168202400208,
        0.22207145392894745,
        0.2248813658952713,
        0.22574232518672943,
        0.2264578640460968,
        0.2265917956829071,
        0.22759772837162018,
        0.22746813297271729,
        0.22716368734836578,
        0.22716933488845825,
        0.22757723927497864,
        0.22722382843494415,
        0.22673146426677704,
        0.22613705694675446,
        0.2244347184896469,
        0.22211141884326935,
        0.2183273881673813,
        0.21741008758544922,
        0.21451692283153534,
        0.21311214566230774,
        0.20762479305267334,
        0.20640577375888824,
        0.20071867108345032,
        0.19932369887828827,
        0.19245882332324982,
        0.19335229694843292,
        0.19078245759010315,
        0.19345322251319885,
        0.1916658580303192,
        0.19645750522613525,
        0.19844752550125122,
        0.20112650096416473,
        0.2010338455438614,
        0.2038118690252304,
        0.20468150079250336,
        0.20720544457435608,
        0.20727655291557312,
        0.21031972765922546,
        0.20860347151756287,
        0.2129739224910736,
        0.21107539534568787,
        0.21647654473781586,
        0.2111099362373352,
        0.22099705040454865,
        0.2196389138698578,
        0.23361735045909882,
        0.23286758363246918,
        0.2512694001197815,
        0.25084227323532104,
        0.2773953676223755,
        0.27732276916503906,
        0.3133004307746887,
        0.31342777609825134,
        0.35808902978897095,
        0.35860511660575867,
        0.4303363263607025,
        0.4337502121925354
      ],
      "loss_nats": 1.4420559406280518,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 2000,
      "stream_state_rms_by_sublayer": [
        0.06250853836536407,
        0.09794864058494568,
        0.12857383489608765,
        0.13700027763843536,
        0.2065405696630478,
        0.20758503675460815,
        0.22102168202400208,
        0.22207145392894745,
        0.2248813658952713,
        0.22574232518672943,
        0.2264578640460968,
        0.2265917956829071,
        0.22759772837162018,
        0.22746813297271729,
        0.22716368734836578,
        0.22716933488845825,
        0.22757723927497864,
        0.22722382843494415,
        0.22673146426677704,
        0.22613705694675446,
        0.2244347184896469,
        0.22211141884326935,
        0.2183273881673813,
        0.21741008758544922,
        0.21451692283153534,
        0.21311214566230774,
        0.20762479305267334,
        0.20640577375888824,
        0.20071867108345032,
        0.19932369887828827,
        0.19245882332324982,
        0.19335229694843292,
        0.19078245759010315,
        0.19345322251319885,
        0.1916658580303192,
        0.19645750522613525,
        0.19844752550125122,
        0.20112650096416473,
        0.2010338455438614,
        0.2038118690252304,
        0.20468150079250336,
        0.20720544457435608,
        0.20727655291557312,
        0.21031972765922546,
        0.20860347151756287,
        0.2129739224910736,
        0.21107539534568787,
        0.21647654473781586,
        0.2111099362373352,
        0.22099705040454865,
        0.2196389138698578,
        0.23361735045909882,
        0.23286758363246918,
        0.2512694001197815,
        0.25084227323532104,
        0.2773953676223755,
        0.27732276916503906,
        0.3133004307746887,
        0.31342777609825134,
        0.35808902978897095,
        0.35860511660575867,
        0.4303363263607025,
        0.4337502121925354,
        0.54518723487854
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.00010598741209832951,
        0.00010775360715342686,
        0.00011871542665176094,
        0.00011485091090435162,
        0.00011244894267292693,
        0.00011016445205314085,
        0.00010846564691746607,
        0.00010656306403689086,
        0.0001046601973939687,
        0.00010342278255848214,
        0.00010179441596847028,
        0.00010021102934842929,
        9.965883509721607e-05,
        9.537018195260316e-05,
        9.32025650399737e-05,
        9.038933058036491e-05,
        8.758632611716166e-05,
        8.443772094324231e-05,
        8.744140359340236e-05,
        8.20954010123387e-05,
        8.16298124846071e-05,
        7.812595140421763e-05,
        7.176945655373856e-05,
        6.528423546114936e-05,
        5.917211092310026e-05,
        5.3287756600184366e-05,
        4.791216633748263e-05,
        4.251671634847298e-05,
        3.64475381502416e-05,
        3.0126682759146206e-05,
        2.3703445549472235e-05,
        1.9818620785372332e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.0001106186828110367,
        "first_to_last_ratio": 2.827449739094469,
        "imbalance_abs_log_ratio": 1.0393751531099071,
        "last_quartile_mean": 3.912312968168408e-05,
        "mean": 8.203169204534788e-05,
        "normalized": [
          1.292030061256553,
          1.3135607040003474,
          1.4471897835062812,
          1.4000797501636437,
          1.3707987714158585,
          1.342949892978447,
          1.3222407610159408,
          1.2990474946924393,
          1.275850769189444,
          1.2607661743867122,
          1.2409157172108236,
          1.2216135843331344,
          1.2148821097353903,
          1.1626016674126585,
          1.1361775274420824,
          1.1018830445481589,
          1.067713269509827,
          1.0293304799389529,
          1.06594660445458,
          1.000776638460116,
          0.9951009231832177,
          0.9523874182801066,
          0.8748991367149143,
          0.7958416294163435,
          0.7213323222735595,
          0.6495996275528049,
          0.5840689755734447,
          0.5182962253779837,
          0.444310451747028,
          0.36725662982170226,
          0.28895473150022005,
          0.24159712290728336
        ],
        "population_cv": 0.3450366052372569
      },
      "activation_output_rms_by_block": [
        0.11079900711774826,
        0.18173734843730927,
        0.25566670298576355,
        0.2647266387939453,
        0.2669948637485504,
        0.2675777077674866,
        0.2675747275352478,
        0.26725223660469055,
        0.2656837999820709,
        0.265005499124527,
        0.25976043939590454,
        0.2535160183906555,
        0.24569463729858398,
        0.23344992101192474,
        0.22194796800613403,
        0.21299737691879272,
        0.21236176788806915,
        0.21684354543685913,
        0.22108736634254456,
        0.22472712397575378,
        0.22933396697044373,
        0.23535853624343872,
        0.2425893247127533,
        0.2497948557138443,
        0.26040923595428467,
        0.27719539403915405,
        0.2986113131046295,
        0.3278995454311371,
        0.36809566617012024,
        0.4197922646999359,
        0.5092319846153259,
        0.6714946031570435
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.23529115412384272,
        "first_to_last_ratio": 0.6008590681870453,
        "imbalance_abs_log_ratio": 0.5093948674743478,
        "last_quartile_mean": 0.39159125089645386,
        "mean": 0.2751628558617085,
        "normalized": [
          0.40266701975732394,
          0.6604719516672225,
          0.9291468580855864,
          0.962072580490267,
          0.9703157895800326,
          0.9724339679842758,
          0.972423137190165,
          0.9712511369594387,
          0.9655511066348229,
          0.9630860178952118,
          0.9440243617999629,
          0.9213308155155495,
          0.8929062628353637,
          0.8484063747661208,
          0.806605845513105,
          0.7740775049443485,
          0.7717675673303741,
          0.7880552945919432,
          0.803478237097738,
          0.8167058859452209,
          0.8334481274816485,
          0.8553426860845104,
          0.881620900295765,
          0.9078073235269309,
          0.9463822256778773,
          1.007386673506787,
          1.0852166516788362,
          1.191656280802424,
          1.3377374828349577,
          1.5256138528774221,
          1.850656706627787,
          2.4403533720209807
        ],
        "population_cv": 0.3576152457433685
      },
      "bits_per_byte": 1.9501283044158226,
      "branch_output_rms_by_sublayer": [
        0.02630079723894596,
        0.06842244416475296,
        0.10497251898050308,
        0.019157659262418747,
        0.09477359801530838,
        0.0031024799682199955,
        0.0338861346244812,
        0.0047894855961203575,
        0.022547677159309387,
        0.004550074227154255,
        0.021251287311315536,
        0.004262411035597324,
        0.01840241253376007,
        0.006118505261838436,
        0.02267448604106903,
        0.009554358199238777,
        0.022321712225675583,
        0.00963375624269247,
        0.01962294802069664,
        0.011137282475829124,
        0.019759519025683403,
        0.013955196365714073,
        0.021449215710163116,
        0.01607326976954937,
        0.026240134611725807,
        0.015075869858264923,
        0.039021123200654984,
        0.02189606800675392,
        0.031211141496896744,
        0.022710295394062996,
        0.03785466402769089,
        0.026124922558665276,
        0.0325845405459404,
        0.026591502130031586,
        0.03150996193289757,
        0.029410794377326965,
        0.02998937852680683,
        0.024189060553908348,
        0.04107306897640228,
        0.029402343556284904,
        0.040071673691272736,
        0.032556869089603424,
        0.04498095437884331,
        0.03419904410839081,
        0.05116673558950424,
        0.041954249143600464,
        0.04688612371683121,
        0.050006624311208725,
        0.052477724850177765,
        0.06379573792219162,
        0.04486159235239029,
        0.06914625316858292,
        0.032891806215047836,
        0.0747767761349678,
        0.015299716033041477,
        0.08387137204408646,
        0.017307085916399956,
        0.09213023632764816,
        0.017144370824098587,
        0.10062393546104431,
        0.017122838646173477,
        0.15051087737083435,
        0.027784937992691994,
        0.24076412618160248
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00032837002715831827,
        0.0002079177996867109,
        0.00018809406473136247,
        0.00010899326506499365,
        9.474430680287842e-05,
        8.318366756728252e-05,
        9.647881676005136e-05,
        0.00013404889675397192,
        0.00012865721885494213,
        0.0001321695767247016,
        0.00014056789256563647,
        0.00015176892928157556,
        0.00016698296163968515,
        0.00022933649843535653,
        0.000217708018553423,
        0.0002507549770035681,
        0.00025689149388127776,
        0.0002828493196393162,
        0.00027498157120012506,
        0.0003306027661240732,
        0.00032889440594708014,
        0.00033897343531885953,
        0.000372930739966371,
        0.0003793341532875068,
        0.0003649148263870391,
        0.0003190518951646816,
        0.00028800578855572547,
        0.0002568521570555004,
        0.00024004963937954607,
        0.00021095610212654392,
        0.00019368945551758278,
        0.00017448279084866967
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00015522885556569618,
        "first_to_last_ratio": 0.6063619309634984,
        "imbalance_abs_log_ratio": 0.5002782253787689,
        "last_quartile_mean": 0.00025600033187941115,
        "mean": 0.00022728867056201116,
        "normalized": [
          1.4447267712304608,
          0.9147741467825811,
          0.8275558313851142,
          0.4795367265578555,
          0.41684570800914306,
          0.36598246345317736,
          0.42447701648080627,
          0.5897737728084399,
          0.56605205414249,
          0.581505344713791,
          0.6184553423540983,
          0.6677364468114501,
          0.7346734935216545,
          1.0090098106002459,
          0.9578480881387607,
          1.1032445056919573,
          1.1302432859766764,
          1.244449707677561,
          1.2098340428503753,
          1.4545501335662696,
          1.4470338760565185,
          1.491378494496396,
          1.6407801543483633,
          1.668953196609125,
          1.605512608634311,
          1.4037298664107178,
          1.2671365794149818,
          1.1300702160841907,
          1.05614432424626,
          0.928141739774877,
          0.8521738238806694,
          0.7676704272906798
        ],
        "population_cv": 0.3864254083322711
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.06033600866794586,
        0.06945198029279709,
        0.11079900711774826,
        0.17341101169586182,
        0.18173734843730927,
        0.25461602210998535,
        0.25566670298576355,
        0.26339152455329895,
        0.2647266387939453,
        0.2662918269634247,
        0.2669948637485504,
        0.2675604224205017,
        0.2675777077674866,
        0.2677975296974182,
        0.2675747275352478,
        0.2675320506095886,
        0.26725223660469055,
        0.26636025309562683,
        0.2656837999820709,
        0.2656107544898987,
        0.265005499124527,
        0.26352381706237793,
        0.25976043939590454,
        0.2553655505180359,
        0.2535160183906555,
        0.24811074137687683,
        0.24569463729858398,
        0.23600345849990845,
        0.23344992101192474,
        0.22498825192451477,
        0.22194796800613403,
        0.21307076513767242,
        0.21299737691879272,
        0.20943206548690796,
        0.21236176788806915,
        0.21104833483695984,
        0.21684354543685913,
        0.21818861365318298,
        0.22108736634254456,
        0.22058506309986115,
        0.22472712397575378,
        0.22517690062522888,
        0.22933396697044373,
        0.2308174967765808,
        0.23535853624343872,
        0.23589646816253662,
        0.2425893247127533,
        0.241608664393425,
        0.2497948557138443,
        0.24810902774333954,
        0.26040923595428467,
        0.26088401675224304,
        0.27719539403915405,
        0.2782979905605316,
        0.2986113131046295,
        0.2985531687736511,
        0.3278995454311371,
        0.3279455602169037,
        0.36809566617012024,
        0.3681766986846924,
        0.4197922646999359,
        0.4200114607810974,
        0.5092319846153259,
        0.5118712186813354
      ],
      "loss_nats": 1.3517259359359741,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 4000,
      "stream_state_rms_by_sublayer": [
        0.06945198029279709,
        0.11079900711774826,
        0.17341101169586182,
        0.18173734843730927,
        0.25461602210998535,
        0.25566670298576355,
        0.26339152455329895,
        0.2647266387939453,
        0.2662918269634247,
        0.2669948637485504,
        0.2675604224205017,
        0.2675777077674866,
        0.2677975296974182,
        0.2675747275352478,
        0.2675320506095886,
        0.26725223660469055,
        0.26636025309562683,
        0.2656837999820709,
        0.2656107544898987,
        0.265005499124527,
        0.26352381706237793,
        0.25976043939590454,
        0.2553655505180359,
        0.2535160183906555,
        0.24811074137687683,
        0.24569463729858398,
        0.23600345849990845,
        0.23344992101192474,
        0.22498825192451477,
        0.22194796800613403,
        0.21307076513767242,
        0.21299737691879272,
        0.20943206548690796,
        0.21236176788806915,
        0.21104833483695984,
        0.21684354543685913,
        0.21818861365318298,
        0.22108736634254456,
        0.22058506309986115,
        0.22472712397575378,
        0.22517690062522888,
        0.22933396697044373,
        0.2308174967765808,
        0.23535853624343872,
        0.23589646816253662,
        0.2425893247127533,
        0.241608664393425,
        0.2497948557138443,
        0.24810902774333954,
        0.26040923595428467,
        0.26088401675224304,
        0.27719539403915405,
        0.2782979905605316,
        0.2986113131046295,
        0.2985531687736511,
        0.3278995454311371,
        0.3279455602169037,
        0.36809566617012024,
        0.3681766986846924,
        0.4197922646999359,
        0.4200114607810974,
        0.5092319846153259,
        0.5118712186813354,
        0.6714946031570435
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.00018891454965341836,
        0.0001931450387928635,
        0.0002085021696984768,
        0.00020282011246308684,
        0.00019955210154876113,
        0.00019673256610985845,
        0.00019503913063090295,
        0.0001924382959259674,
        0.00019001582404598594,
        0.00018850783817470074,
        0.00018595026631373912,
        0.0001838041644077748,
        0.00018427071336191148,
        0.00017771514831110835,
        0.0001728432544041425,
        0.0001671368081588298,
        0.0001615467481315136,
        0.00015400326810777187,
        0.000160209892783314,
        0.00015067463391460478,
        0.00014840754738543183,
        0.00014097621897235513,
        0.00012854386295657605,
        0.00011642893514363095,
        0.00010425484651932493,
        9.334496280644089e-05,
        8.331386925419793e-05,
        7.313644891837612e-05,
        6.15216777077876e-05,
        4.904071465716697e-05,
        3.525121428538114e-05,
        2.4962779207271524e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00019714299560291693,
        "first_to_last_ratio": 3.0050767723956184,
        "imbalance_abs_log_ratio": 1.1003031158789967,
        "last_quartile_mean": 6.560331416949339e-05,
        "mean": 0.00014728142508602104,
        "normalized": [
          1.282677360998382,
          1.311401208129643,
          1.4156718647765612,
          1.3770922731405397,
          1.3549033860326305,
          1.3357595229334251,
          1.324261566025645,
          1.3066026202121013,
          1.2901547084773621,
          1.2799159029361715,
          1.262550699826086,
          1.247979263511487,
          1.2511470014245618,
          1.206636534154339,
          1.1735577266664268,
          1.1348125404219307,
          1.0968575842959203,
          1.045639448544342,
          1.0877807075110928,
          1.023038946198422,
          1.0076460578701834,
          0.9571894004285788,
          0.8727771535446431,
          0.7905201560592559,
          0.707861473084158,
          0.6337863905914947,
          0.5656780493910737,
          0.4965761899415362,
          0.41771511697320496,
          0.33297284181304126,
          0.23934596141225786,
          0.1694903426735027
        ],
        "population_cv": 0.361085575415688
      },
      "activation_output_rms_by_block": [
        0.1016591340303421,
        0.18086770176887512,
        0.2325568050146103,
        0.2376081645488739,
        0.23870401084423065,
        0.23941391706466675,
        0.23885947465896606,
        0.2381371706724167,
        0.23675362765789032,
        0.23536528646945953,
        0.22870509326457977,
        0.22131286561489105,
        0.21254637837409973,
        0.19929367303848267,
        0.18746964633464813,
        0.17704539000988007,
        0.17596758902072906,
        0.1784123033285141,
        0.18117663264274597,
        0.18366514146327972,
        0.1872515082359314,
        0.19141052663326263,
        0.1969655156135559,
        0.2033752053976059,
        0.21314463019371033,
        0.22718842327594757,
        0.2457481175661087,
        0.269707053899765,
        0.30305343866348267,
        0.347743958234787,
        0.4299577474594116,
        0.6035616993904114
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.2134757973253727,
        "first_to_last_ratio": 0.6468706108937196,
        "imbalance_abs_log_ratio": 0.4356089876432565,
        "last_quartile_mean": 0.33001313358545303,
        "mean": 0.23576961969956756,
        "normalized": [
          0.4311799550759871,
          0.7671374369579427,
          0.986373076017804,
          1.0077980566438083,
          1.0124460104249322,
          1.015457026947505,
          1.0131053990897376,
          1.010041798327817,
          1.0041735994636487,
          0.9982850494876174,
          0.9700363157730422,
          0.9386827102529232,
          0.9015002808459363,
          0.8452898778580343,
          0.7951391132306771,
          0.7509253746750002,
          0.7463539587711004,
          0.7567230398719659,
          0.7684477451913126,
          0.7790025775895867,
          0.7942138960674365,
          0.8118540755045961,
          0.8354151644496934,
          0.8626014057992686,
          0.9040377232033228,
          0.9636034683579899,
          1.0423230858973958,
          1.143943202875089,
          1.2853795117863462,
          1.4749311581276,
          1.8236350722679655,
          2.5599638331669174
        ],
        "population_cv": 0.3675345504070441
      },
      "bits_per_byte": 1.8720084529317582,
      "branch_output_rms_by_sublayer": [
        0.03573180362582207,
        0.052011821419000626,
        0.10664675384759903,
        0.01823282428085804,
        0.06781794875860214,
        0.0026424091774970293,
        0.02450474537909031,
        0.003992235753685236,
        0.01725216954946518,
        0.0036646369844675064,
        0.018646538257598877,
        0.003319106763228774,
        0.015910238027572632,
        0.004927239380776882,
        0.018212314695119858,
        0.008043181151151657,
        0.016784843057394028,
        0.007957165129482746,
        0.01594853773713112,
        0.009037249721586704,
        0.0176989808678627,
        0.012198803015053272,
        0.019605962559580803,
        0.013432997278869152,
        0.022475004196166992,
        0.012972000986337662,
        0.03453320264816284,
        0.018862294033169746,
        0.026713354513049126,
        0.01990535296499729,
        0.0319337360560894,
        0.023401649668812752,
        0.02801610343158245,
        0.023707188665866852,
        0.028282491490244865,
        0.026515409350395203,
        0.02569115161895752,
        0.022723721340298653,
        0.03440641239285469,
        0.027487577870488167,
        0.034427426755428314,
        0.03072489984333515,
        0.039456624537706375,
        0.03320617973804474,
        0.043320003896951675,
        0.04054662957787514,
        0.04175209999084473,
        0.04858461394906044,
        0.043093886226415634,
        0.060716040432453156,
        0.03683279827237129,
        0.0668661892414093,
        0.03015786223113537,
        0.0722465068101883,
        0.015251493081450462,
        0.07915013283491135,
        0.017382971942424774,
        0.08651599287986755,
        0.015752796083688736,
        0.09494592994451523,
        0.018999194726347923,
        0.14218401908874512,
        0.03338775411248207,
        0.24767111241817474
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0004570302696620844,
        0.00031493934518203267,
        0.0002143615651481701,
        0.00013786409436507824,
        0.0001291171736265367,
        0.00013682107280354788,
        0.00015837516984797755,
        0.00019563631023119442,
        0.00019674727625245423,
        0.000191343414336242,
        0.00021955065404037036,
        0.0002332172506517699,
        0.00027919484030541734,
        0.0003858941801364767,
        0.0003704235701666674,
        0.00042305543536477747,
        0.0004472346343569547,
        0.0004940425030518999,
        0.0004673224693273912,
        0.0005594278228367747,
        0.0005764247107952261,
        0.0005972083894683241,
        0.0006512089060709392,
        0.0006542639335938355,
        0.0006197665059356166,
        0.0005424206044372668,
        0.0004994517842756415,
        0.0004394000632885711,
        0.0004055610719669202,
        0.0003512581615295656,
        0.00030757820471853463,
        0.00025263519960351045
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00021801812510832775,
        "first_to_last_ratio": 0.5102716406035511,
        "imbalance_abs_log_ratio": 0.6728120664256351,
        "last_quartile_mean": 0.0004272589494694534,
        "mean": 0.0003721492683555553,
        "normalized": [
          1.2280832142478726,
          0.8462715688618158,
          0.5760096374645209,
          0.3704537562958782,
          0.34694995961452973,
          0.36765105950128485,
          0.42556894051626687,
          0.5256931206546978,
          0.5286783905872947,
          0.5141577066152674,
          0.5899532061705127,
          0.6266766334979106,
          0.7502227306239702,
          1.0369338675356186,
          0.9953628870573538,
          1.1367896468913272,
          1.2017614231332092,
          1.3275385579419885,
          1.2557393203871798,
          1.5032350468100115,
          1.5489072794427852,
          1.6047549740115168,
          1.7498594285795221,
          1.7580685741634856,
          1.6653707494152192,
          1.4575350553128925,
          1.3420738041017997,
          1.1807091956143891,
          1.0897806510785424,
          0.9438636359052839,
          0.8264914937967075,
          0.6788544841693472
        ],
        "population_cv": 0.43653500885793567
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.06193641945719719,
        0.07578708976507187,
        0.1016591340303421,
        0.17375877499580383,
        0.18086770176887512,
        0.23162810504436493,
        0.2325568050146103,
        0.2364615947008133,
        0.2376081645488739,
        0.23820650577545166,
        0.23870401084423065,
        0.23957808315753937,
        0.23941391706466675,
        0.2393639236688614,
        0.23885947465896606,
        0.23879873752593994,
        0.2381371706724167,
        0.23766019940376282,
        0.23675362765789032,
        0.23612910509109497,
        0.23536528646945953,
        0.2328052520751953,
        0.22870509326457977,
        0.22353777289390564,
        0.22131286561489105,
        0.21547429263591766,
        0.21254637837409973,
        0.20247745513916016,
        0.19929367303848267,
        0.19119344651699066,
        0.18746964633464813,
        0.1784159243106842,
        0.17704539000988007,
        0.17436346411705017,
        0.17596758902072906,
        0.17439287900924683,
        0.1784123033285141,
        0.1789977103471756,
        0.18117663264274597,
        0.18044494092464447,
        0.18366514146327972,
        0.1840001940727234,
        0.1872515082359314,
        0.18770679831504822,
        0.19141052663326263,
        0.19118990004062653,
        0.1969655156135559,
        0.19665203988552094,
        0.2033752053976059,
        0.20245759189128876,
        0.21314463019371033,
        0.21310319006443024,
        0.22718842327594757,
        0.2287222445011139,
        0.2457481175661087,
        0.2456687092781067,
        0.269707053899765,
        0.26972076296806335,
        0.30305343866348267,
        0.30293965339660645,
        0.347743958234787,
        0.3480367064476013,
        0.4299577474594116,
        0.43200185894966125
      ],
      "loss_nats": 1.2975773811340332,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 8000,
      "stream_state_rms_by_sublayer": [
        0.07578708976507187,
        0.1016591340303421,
        0.17375877499580383,
        0.18086770176887512,
        0.23162810504436493,
        0.2325568050146103,
        0.2364615947008133,
        0.2376081645488739,
        0.23820650577545166,
        0.23870401084423065,
        0.23957808315753937,
        0.23941391706466675,
        0.2393639236688614,
        0.23885947465896606,
        0.23879873752593994,
        0.2381371706724167,
        0.23766019940376282,
        0.23675362765789032,
        0.23612910509109497,
        0.23536528646945953,
        0.2328052520751953,
        0.22870509326457977,
        0.22353777289390564,
        0.22131286561489105,
        0.21547429263591766,
        0.21254637837409973,
        0.20247745513916016,
        0.19929367303848267,
        0.19119344651699066,
        0.18746964633464813,
        0.1784159243106842,
        0.17704539000988007,
        0.17436346411705017,
        0.17596758902072906,
        0.17439287900924683,
        0.1784123033285141,
        0.1789977103471756,
        0.18117663264274597,
        0.18044494092464447,
        0.18366514146327972,
        0.1840001940727234,
        0.1872515082359314,
        0.18770679831504822,
        0.19141052663326263,
        0.19118990004062653,
        0.1969655156135559,
        0.19665203988552094,
        0.2033752053976059,
        0.20245759189128876,
        0.21314463019371033,
        0.21310319006443024,
        0.22718842327594757,
        0.2287222445011139,
        0.2457481175661087,
        0.2456687092781067,
        0.269707053899765,
        0.26972076296806335,
        0.30305343866348267,
        0.30293965339660645,
        0.347743958234787,
        0.3480367064476013,
        0.4299577474594116,
        0.43200185894966125,
        0.6035616993904114
      ]
    }
  ],
  "environment": {
    "autocast": "cuda-bfloat16-forward-fp32-cross-entropy",
    "compile": false,
    "compute_capability": [
      12,
      0
    ],
    "cublas_workspace_config": ":4096:8",
    "cuda": "12.8",
    "deterministic_algorithms": true,
    "gpu": "NVIDIA GeForce RTX 5090",
    "python": "3.10.14",
    "torch": "2.11.0+cu128"
  },
  "evaluations": [
    {
      "bits_per_byte": 7.95552135286859,
      "cross_entropy_nats": 5.514347195625305,
      "step": 0
    },
    {
      "bits_per_byte": 5.166060485015187,
      "cross_entropy_nats": 3.5808402597904205,
      "step": 100
    },
    {
      "bits_per_byte": 3.1277970484292927,
      "cross_entropy_nats": 2.168023705482483,
      "step": 500
    },
    {
      "bits_per_byte": 1.9613801618593651,
      "cross_entropy_nats": 1.359525129199028,
      "step": 2000
    },
    {
      "bits_per_byte": 1.7971937855298066,
      "cross_entropy_nats": 1.2457198053598404,
      "step": 4000
    },
    {
      "bits_per_byte": 1.7178972072507075,
      "cross_entropy_nats": 1.1907556056976318,
      "step": 8000
    }
  ],
  "gradient_gate": null,
  "hashes": {
    "final_mixer_parameters": null,
    "final_model_state": "b93ef93bca883c40ec5d076dddd2b9f0759523f4f41313e8f1ca34c1bdbdcf6c",
    "final_optimizer_state": "06d1b38e173978179caca1f5bf426eaad5609e9e9a1b567ef88c8b18eeab7e4d",
    "final_public_parameters": "b93ef93bca883c40ec5d076dddd2b9f0759523f4f41313e8f1ca34c1bdbdcf6c",
    "initial_mixer_parameters": null,
    "initial_public_parameter_elements": 18985152,
    "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670",
    "initial_public_parameter_tensors": 227,
    "initial_public_parameters": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58"
  },
  "manifest": {
    "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716",
    "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371",
    "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
    "input_gate_tensor_hashes": {
      "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434",
      "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94",
      "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc"
    },
    "path": "experiments/k3/attnres_gradient/manifest.json",
    "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338"
  },
  "model": {
    "attnres_aggregation_groups": 8,
    "context": 256,
    "d_ff": 768,
    "d_head": 32,
    "d_model": 192,
    "heads": 6,
    "layers": 32,
    "parameters": {
      "core": 18985152,
      "embedding": 98304,
      "mixer": 0,
      "total": 18985152
    },
    "sublayers": 64,
    "sublayers_per_attnres_group": 8,
    "transformer_blocks_per_attnres_group": 4,
    "vocabulary": 256
  },
  "optimizer": {
    "betas": [
      0.9,
      0.95
    ],
    "epsilon": 1e-08,
    "grad_clip": 1.0,
    "min_lr": 3e-05,
    "name": "AdamW",
    "peak_lr": 0.0003,
    "warmup_steps": 400,
    "weight_decay_ndim_ge_2": 0.1
  },
  "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
  "run_kind": "formal",
  "schema_version": 1,
  "seed": 2026073001,
  "steps": 8000,
  "target_bytes_seen": 65536000,
  "timing": {
    "mean_ms": 42.00224438344842,
    "measured_steps": 7980,
    "median_ms": 41.91726399585605,
    "p95_ms": 42.462232409161516,
    "peak_allocated_bytes": 5938562048,
    "peak_reserved_bytes": 6325010432,
    "warmup_steps_excluded": 20
  },
  "training_history": [
    {
      "bits_per_byte": 7.958522622108798,
      "learning_rate": 7.499999999999999e-07,
      "loss_nats": 5.516427516937256,
      "step": 1,
      "unclipped_grad_norm": 22.09486961364746
    },
    {
      "bits_per_byte": 7.238571600789322,
      "learning_rate": 7.499999999999999e-06,
      "loss_nats": 5.017395496368408,
      "step": 10,
      "unclipped_grad_norm": 11.172341346740723
    },
    {
      "bits_per_byte": 6.729949421197048,
      "learning_rate": 1.4999999999999999e-05,
      "loss_nats": 4.6648454666137695,
      "step": 20,
      "unclipped_grad_norm": 3.462282180786133
    },
    {
      "bits_per_byte": 6.60904492970989,
      "learning_rate": 2.2499999999999998e-05,
      "loss_nats": 4.581040859222412,
      "step": 30,
      "unclipped_grad_norm": 3.06864070892334
    },
    {
      "bits_per_byte": 6.370936325451538,
      "learning_rate": 2.9999999999999997e-05,
      "loss_nats": 4.415996551513672,
      "step": 40,
      "unclipped_grad_norm": 3.234333038330078
    },
    {
      "bits_per_byte": 6.086515545038246,
      "learning_rate": 3.75e-05,
      "loss_nats": 4.218851089477539,
      "step": 50,
      "unclipped_grad_norm": 2.8305625915527344
    },
    {
      "bits_per_byte": 5.8833778930063385,
      "learning_rate": 4.4999999999999996e-05,
      "loss_nats": 4.078046798706055,
      "step": 60,
      "unclipped_grad_norm": 2.670325517654419
    },
    {
      "bits_per_byte": 5.715139992383416,
      "learning_rate": 5.2499999999999995e-05,
      "loss_nats": 3.961433172225952,
      "step": 70,
      "unclipped_grad_norm": 3.2375030517578125
    },
    {
      "bits_per_byte": 5.566718964688812,
      "learning_rate": 5.9999999999999995e-05,
      "loss_nats": 3.858555555343628,
      "step": 80,
      "unclipped_grad_norm": 2.6123173236846924
    },
    {
      "bits_per_byte": 5.355997221719156,
      "learning_rate": 6.749999999999999e-05,
      "loss_nats": 3.712494373321533,
      "step": 90,
      "unclipped_grad_norm": 2.440842628479004
    },
    {
      "bits_per_byte": 5.182009597112589,
      "learning_rate": 7.5e-05,
      "loss_nats": 3.591895341873169,
      "step": 100,
      "unclipped_grad_norm": 2.292849063873291
    },
    {
      "bits_per_byte": 4.992510513259403,
      "learning_rate": 8.249999999999999e-05,
      "loss_nats": 3.4605445861816406,
      "step": 110,
      "unclipped_grad_norm": 2.057412624359131
    },
    {
      "bits_per_byte": 4.778040580396842,
      "learning_rate": 8.999999999999999e-05,
      "loss_nats": 3.311885356903076,
      "step": 120,
      "unclipped_grad_norm": 2.002248525619507
    },
    {
      "bits_per_byte": 4.588237775182233,
      "learning_rate": 9.75e-05,
      "loss_nats": 3.180324077606201,
      "step": 130,
      "unclipped_grad_norm": 1.7531825304031372
    },
    {
      "bits_per_byte": 4.430652823919056,
      "learning_rate": 0.00010499999999999999,
      "loss_nats": 3.071094512939453,
      "step": 140,
      "unclipped_grad_norm": 1.4980645179748535
    },
    {
      "bits_per_byte": 4.274817968111058,
      "learning_rate": 0.0001125,
      "loss_nats": 2.963078022003174,
      "step": 150,
      "unclipped_grad_norm": 1.3038520812988281
    },
    {
      "bits_per_byte": 4.169977000179788,
      "learning_rate": 0.00011999999999999999,
      "loss_nats": 2.8904078006744385,
      "step": 160,
      "unclipped_grad_norm": 1.0998529195785522
    },
    {
      "bits_per_byte": 4.028519206053318,
      "learning_rate": 0.00012749999999999998,
      "loss_nats": 2.7923567295074463,
      "step": 170,
      "unclipped_grad_norm": 0.9818326234817505
    },
    {
      "bits_per_byte": 3.950109907767851,
      "learning_rate": 0.00013499999999999997,
      "loss_nats": 2.7380075454711914,
      "step": 180,
      "unclipped_grad_norm": 0.8779997229576111
    },
    {
      "bits_per_byte": 3.846655463967837,
      "learning_rate": 0.0001425,
      "loss_nats": 2.6662983894348145,
      "step": 190,
      "unclipped_grad_norm": 1.2310962677001953
    },
    {
      "bits_per_byte": 3.7855644746589534,
      "learning_rate": 0.00015,
      "loss_nats": 2.623953342437744,
      "step": 200,
      "unclipped_grad_norm": 1.0928086042404175
    },
    {
      "bits_per_byte": 3.7365611139833117,
      "learning_rate": 0.0001575,
      "loss_nats": 2.589986801147461,
      "step": 210,
      "unclipped_grad_norm": 0.7771895527839661
    },
    {
      "bits_per_byte": 3.6369384436445213,
      "learning_rate": 0.00016499999999999997,
      "loss_nats": 2.5209336280822754,
      "step": 220,
      "unclipped_grad_norm": 0.7917730212211609
    },
    {
      "bits_per_byte": 3.630690657903977,
      "learning_rate": 0.0001725,
      "loss_nats": 2.5166029930114746,
      "step": 230,
      "unclipped_grad_norm": 1.0061120986938477
    },
    {
      "bits_per_byte": 3.503533565162051,
      "learning_rate": 0.00017999999999999998,
      "loss_nats": 2.428464412689209,
      "step": 240,
      "unclipped_grad_norm": 0.7126089930534363
    },
    {
      "bits_per_byte": 3.5514149110234636,
      "learning_rate": 0.0001875,
      "loss_nats": 2.461653232574463,
      "step": 250,
      "unclipped_grad_norm": 1.2347668409347534
    },
    {
      "bits_per_byte": 3.5407041754930297,
      "learning_rate": 0.000195,
      "loss_nats": 2.4542291164398193,
      "step": 260,
      "unclipped_grad_norm": 0.9977514147758484
    },
    {
      "bits_per_byte": 3.5337584841552845,
      "learning_rate": 0.00020249999999999996,
      "loss_nats": 2.4494147300720215,
      "step": 270,
      "unclipped_grad_norm": 0.7908692955970764
    },
    {
      "bits_per_byte": 3.478037825102243,
      "learning_rate": 0.00020999999999999998,
      "loss_nats": 2.410792112350464,
      "step": 280,
      "unclipped_grad_norm": 1.0802956819534302
    },
    {
      "bits_per_byte": 3.3705159916093237,
      "learning_rate": 0.00021749999999999997,
      "loss_nats": 2.336263656616211,
      "step": 290,
      "unclipped_grad_norm": 0.9241761565208435
    },
    {
      "bits_per_byte": 3.410749956917471,
      "learning_rate": 0.000225,
      "loss_nats": 2.3641517162323,
      "step": 300,
      "unclipped_grad_norm": 0.9868955612182617
    },
    {
      "bits_per_byte": 3.3596274260178975,
      "learning_rate": 0.00023249999999999996,
      "loss_nats": 2.328716278076172,
      "step": 310,
      "unclipped_grad_norm": 0.8471657037734985
    },
    {
      "bits_per_byte": 3.3434163413475693,
      "learning_rate": 0.00023999999999999998,
      "loss_nats": 2.3174796104431152,
      "step": 320,
      "unclipped_grad_norm": 0.9364105463027954
    },
    {
      "bits_per_byte": 3.3338296844231747,
      "learning_rate": 0.0002475,
      "loss_nats": 2.3108346462249756,
      "step": 330,
      "unclipped_grad_norm": 0.9398566484451294
    },
    {
      "bits_per_byte": 3.4261592584694767,
      "learning_rate": 0.00025499999999999996,
      "loss_nats": 2.3748326301574707,
      "step": 340,
      "unclipped_grad_norm": 0.949837863445282
    },
    {
      "bits_per_byte": 3.364650007353695,
      "learning_rate": 0.0002625,
      "loss_nats": 2.332197666168213,
      "step": 350,
      "unclipped_grad_norm": 0.8382212519645691
    },
    {
      "bits_per_byte": 3.2974426270495267,
      "learning_rate": 0.00026999999999999995,
      "loss_nats": 2.2856130599975586,
      "step": 360,
      "unclipped_grad_norm": 1.0729857683181763
    },
    {
      "bits_per_byte": 3.304221839181237,
      "learning_rate": 0.00027749999999999997,
      "loss_nats": 2.2903120517730713,
      "step": 370,
      "unclipped_grad_norm": 0.9106518030166626
    },
    {
      "bits_per_byte": 3.3613300542614395,
      "learning_rate": 0.000285,
      "loss_nats": 2.3298964500427246,
      "step": 380,
      "unclipped_grad_norm": 1.290928602218628
    },
    {
      "bits_per_byte": 3.2658243046190027,
      "learning_rate": 0.0002925,
      "loss_nats": 2.2636969089508057,
      "step": 390,
      "unclipped_grad_norm": 1.1257888078689575
    },
    {
      "bits_per_byte": 3.2284232375358677,
      "learning_rate": 0.0003,
      "loss_nats": 2.2377724647521973,
      "step": 400,
      "unclipped_grad_norm": 1.3445736169815063
    },
    {
      "bits_per_byte": 3.1961283353568497,
      "learning_rate": 0.0002999988466112388,
      "loss_nats": 2.2153873443603516,
      "step": 410,
      "unclipped_grad_norm": 0.7953492403030396
    },
    {
      "bits_per_byte": 3.2387222466001364,
      "learning_rate": 0.00029999538646466355,
      "loss_nats": 2.2449111938476562,
      "step": 420,
      "unclipped_grad_norm": 0.8595701456069946
    },
    {
      "bits_per_byte": 3.2223075344711862,
      "learning_rate": 0.0002999896196193985,
      "loss_nats": 2.2335333824157715,
      "step": 430,
      "unclipped_grad_norm": 1.2960833311080933
    },
    {
      "bits_per_byte": 3.2629112624786885,
      "learning_rate": 0.00029998154617398326,
      "loss_nats": 2.2616777420043945,
      "step": 440,
      "unclipped_grad_norm": 0.9954231977462769
    },
    {
      "bits_per_byte": 3.169579029577856,
      "learning_rate": 0.0002999711662663707,
      "loss_nats": 2.1969847679138184,
      "step": 450,
      "unclipped_grad_norm": 1.0229040384292603
    },
    {
      "bits_per_byte": 3.210129442963591,
      "learning_rate": 0.0002999584800739247,
      "loss_nats": 2.2250921726226807,
      "step": 460,
      "unclipped_grad_norm": 1.1146044731140137
    },
    {
      "bits_per_byte": 3.160413042217472,
      "learning_rate": 0.0002999434878134173,
      "loss_nats": 2.19063138961792,
      "step": 470,
      "unclipped_grad_norm": 0.9123480319976807
    },
    {
      "bits_per_byte": 3.1596232978847096,
      "learning_rate": 0.000299926189741025,
      "loss_nats": 2.1900839805603027,
      "step": 480,
      "unclipped_grad_norm": 0.7869184613227844
    },
    {
      "bits_per_byte": 3.0856473683731847,
      "learning_rate": 0.000299906586152324,
      "loss_nats": 2.138807773590088,
      "step": 490,
      "unclipped_grad_norm": 0.8606863021850586
    },
    {
      "bits_per_byte": 3.05385947104891,
      "learning_rate": 0.0002998846773822856,
      "loss_nats": 2.116774082183838,
      "step": 500,
      "unclipped_grad_norm": 0.9675024151802063
    },
    {
      "bits_per_byte": 3.0816491157059263,
      "learning_rate": 0.0002998604638052702,
      "loss_nats": 2.1360363960266113,
      "step": 510,
      "unclipped_grad_norm": 0.8225864768028259
    },
    {
      "bits_per_byte": 3.0308351216820006,
      "learning_rate": 0.0002998339458350211,
      "loss_nats": 2.1008148193359375,
      "step": 520,
      "unclipped_grad_norm": 1.098574161529541
    },
    {
      "bits_per_byte": 3.0891052515514326,
      "learning_rate": 0.0002998051239246572,
      "loss_nats": 2.141204595565796,
      "step": 530,
      "unclipped_grad_norm": 0.8098863363265991
    },
    {
      "bits_per_byte": 3.0354875963531414,
      "learning_rate": 0.00029977399856666537,
      "loss_nats": 2.1040396690368652,
      "step": 540,
      "unclipped_grad_norm": 1.0088073015213013
    },
    {
      "bits_per_byte": 3.0042414443790846,
      "learning_rate": 0.0002997405702928922,
      "loss_nats": 2.0823814868927,
      "step": 550,
      "unclipped_grad_norm": 1.054882287979126
    },
    {
      "bits_per_byte": 2.9880702596837563,
      "learning_rate": 0.0002997048396745345,
      "loss_nats": 2.0711724758148193,
      "step": 560,
      "unclipped_grad_norm": 0.723493754863739
    },
    {
      "bits_per_byte": 2.93916561704971,
      "learning_rate": 0.0002996668073221302,
      "loss_nats": 2.0372743606567383,
      "step": 570,
      "unclipped_grad_norm": 1.0234330892562866
    },
    {
      "bits_per_byte": 3.0140372322069284,
      "learning_rate": 0.000299626473885547,
      "loss_nats": 2.0891714096069336,
      "step": 580,
      "unclipped_grad_norm": 1.0687707662582397
    },
    {
      "bits_per_byte": 2.915940047981291,
      "learning_rate": 0.00029958384005397226,
      "loss_nats": 2.0211756229400635,
      "step": 590,
      "unclipped_grad_norm": 1.149570107460022
    },
    {
      "bits_per_byte": 2.9613039117993742,
      "learning_rate": 0.0002995389065559004,
      "loss_nats": 2.052619457244873,
      "step": 600,
      "unclipped_grad_norm": 0.7275651693344116
    },
    {
      "bits_per_byte": 2.880448160305384,
      "learning_rate": 0.000299491674159121,
      "loss_nats": 1.9965745210647583,
      "step": 610,
      "unclipped_grad_norm": 0.9942755699157715
    },
    {
      "bits_per_byte": 2.9149380770573643,
      "learning_rate": 0.00029944214367070535,
      "loss_nats": 2.0204811096191406,
      "step": 620,
      "unclipped_grad_norm": 0.8311108350753784
    },
    {
      "bits_per_byte": 2.871522948656208,
      "learning_rate": 0.00029939031593699284,
      "loss_nats": 1.990388035774231,
      "step": 630,
      "unclipped_grad_norm": 1.2078016996383667
    },
    {
      "bits_per_byte": 2.795491826466872,
      "learning_rate": 0.00029933619184357654,
      "loss_nats": 1.9376872777938843,
      "step": 640,
      "unclipped_grad_norm": 0.7877771258354187
    },
    {
      "bits_per_byte": 2.7606237198656447,
      "learning_rate": 0.00029927977231528774,
      "loss_nats": 1.9135185480117798,
      "step": 650,
      "unclipped_grad_norm": 1.1925899982452393
    },
    {
      "bits_per_byte": 2.6852617821708162,
      "learning_rate": 0.00029922105831618057,
      "loss_nats": 1.8612816333770752,
      "step": 660,
      "unclipped_grad_norm": 1.126668930053711
    },
    {
      "bits_per_byte": 2.7363994475436653,
      "learning_rate": 0.0002991600508495153,
      "loss_nats": 1.8967275619506836,
      "step": 670,
      "unclipped_grad_norm": 1.136749267578125
    },
    {
      "bits_per_byte": 2.6585009377312616,
      "learning_rate": 0.00029909675095774114,
      "loss_nats": 1.8427324295043945,
      "step": 680,
      "unclipped_grad_norm": 1.2957242727279663
    },
    {
      "bits_per_byte": 2.734127384743117,
      "learning_rate": 0.0002990311597224787,
      "loss_nats": 1.8951526880264282,
      "step": 690,
      "unclipped_grad_norm": 0.7534775137901306
    },
    {
      "bits_per_byte": 2.6542058430086137,
      "learning_rate": 0.00029896327826450103,
      "loss_nats": 1.8397552967071533,
      "step": 700,
      "unclipped_grad_norm": 0.8543418049812317
    },
    {
      "bits_per_byte": 2.6774995172067744,
      "learning_rate": 0.0002988931077437152,
      "loss_nats": 1.8559012413024902,
      "step": 710,
      "unclipped_grad_norm": 1.3297739028930664
    },
    {
      "bits_per_byte": 2.6677150802338874,
      "learning_rate": 0.00029882064935914164,
      "loss_nats": 1.8491191864013672,
      "step": 720,
      "unclipped_grad_norm": 0.7874022126197815
    },
    {
      "bits_per_byte": 2.6473420154126788,
      "learning_rate": 0.0002987459043488943,
      "loss_nats": 1.8349976539611816,
      "step": 730,
      "unclipped_grad_norm": 1.0262641906738281
    },
    {
      "bits_per_byte": 2.582519862493327,
      "learning_rate": 0.0002986688739901591,
      "loss_nats": 1.7900663614273071,
      "step": 740,
      "unclipped_grad_norm": 1.1010198593139648
    },
    {
      "bits_per_byte": 2.5681288702170795,
      "learning_rate": 0.0002985895595991723,
      "loss_nats": 1.7800912857055664,
      "step": 750,
      "unclipped_grad_norm": 0.9110153913497925
    },
    {
      "bits_per_byte": 2.5014427693276766,
      "learning_rate": 0.00029850796253119824,
      "loss_nats": 1.7338680028915405,
      "step": 760,
      "unclipped_grad_norm": 1.0652238130569458
    },
    {
      "bits_per_byte": 2.634176399523829,
      "learning_rate": 0.00029842408418050544,
      "loss_nats": 1.8258719444274902,
      "step": 770,
      "unclipped_grad_norm": 0.9231922626495361
    },
    {
      "bits_per_byte": 2.5180344516043642,
      "learning_rate": 0.0002983379259803436,
      "loss_nats": 1.745368480682373,
      "step": 780,
      "unclipped_grad_norm": 0.8394486308097839
    },
    {
      "bits_per_byte": 2.482792454720161,
      "learning_rate": 0.0002982494894029186,
      "loss_nats": 1.7209405899047852,
      "step": 790,
      "unclipped_grad_norm": 0.8228755593299866
    },
    {
      "bits_per_byte": 2.430754632152696,
      "learning_rate": 0.0002981587759593675,
      "loss_nats": 1.684870719909668,
      "step": 800,
      "unclipped_grad_norm": 0.9281390905380249
    },
    {
      "bits_per_byte": 2.489219790348206,
      "learning_rate": 0.00029806578719973283,
      "loss_nats": 1.725395679473877,
      "step": 810,
      "unclipped_grad_norm": 0.8680029511451721
    },
    {
      "bits_per_byte": 2.4286650429447127,
      "learning_rate": 0.00029797052471293596,
      "loss_nats": 1.683422327041626,
      "step": 820,
      "unclipped_grad_norm": 1.0944814682006836
    },
    {
      "bits_per_byte": 2.483867002322752,
      "learning_rate": 0.0002978729901267499,
      "loss_nats": 1.7216854095458984,
      "step": 830,
      "unclipped_grad_norm": 0.8509150743484497
    },
    {
      "bits_per_byte": 2.566762812021276,
      "learning_rate": 0.0002977731851077716,
      "loss_nats": 1.7791444063186646,
      "step": 840,
      "unclipped_grad_norm": 1.2704709768295288
    },
    {
      "bits_per_byte": 2.458091274507339,
      "learning_rate": 0.0002976711113613935,
      "loss_nats": 1.7038190364837646,
      "step": 850,
      "unclipped_grad_norm": 0.905615508556366
    },
    {
      "bits_per_byte": 2.442831941826886,
      "learning_rate": 0.0002975667706317743,
      "loss_nats": 1.693242073059082,
      "step": 860,
      "unclipped_grad_norm": 0.7913721203804016
    },
    {
      "bits_per_byte": 2.3506050414231483,
      "learning_rate": 0.00029746016470180915,
      "loss_nats": 1.6293152570724487,
      "step": 870,
      "unclipped_grad_norm": 1.0574872493743896
    },
    {
      "bits_per_byte": 2.3881724157296675,
      "learning_rate": 0.00029735129539309926,
      "loss_nats": 1.6553549766540527,
      "step": 880,
      "unclipped_grad_norm": 0.923473060131073
    },
    {
      "bits_per_byte": 2.321517099734769,
      "learning_rate": 0.0002972401645659207,
      "loss_nats": 1.6091530323028564,
      "step": 890,
      "unclipped_grad_norm": 1.0251853466033936
    },
    {
      "bits_per_byte": 2.330866592583624,
      "learning_rate": 0.00029712677411919283,
      "loss_nats": 1.6156336069107056,
      "step": 900,
      "unclipped_grad_norm": 0.9670726656913757
    },
    {
      "bits_per_byte": 2.356522276508734,
      "learning_rate": 0.0002970111259904454,
      "loss_nats": 1.633416771888733,
      "step": 910,
      "unclipped_grad_norm": 0.8682661652565002
    },
    {
      "bits_per_byte": 2.329593920967239,
      "learning_rate": 0.0002968932221557859,
      "loss_nats": 1.6147514581680298,
      "step": 920,
      "unclipped_grad_norm": 0.9293603301048279
    },
    {
      "bits_per_byte": 2.356717820782765,
      "learning_rate": 0.0002967730646298655,
      "loss_nats": 1.6335523128509521,
      "step": 930,
      "unclipped_grad_norm": 1.1330991983413696
    },
    {
      "bits_per_byte": 2.3506738344834934,
      "learning_rate": 0.0002966506554658449,
      "loss_nats": 1.629362940788269,
      "step": 940,
      "unclipped_grad_norm": 0.8093944787979126
    },
    {
      "bits_per_byte": 2.337553106066519,
      "learning_rate": 0.00029652599675535876,
      "loss_nats": 1.6202683448791504,
      "step": 950,
      "unclipped_grad_norm": 1.0507299900054932
    },
    {
      "bits_per_byte": 2.3753796582278888,
      "learning_rate": 0.0002963990906284808,
      "loss_nats": 1.6464877128601074,
      "step": 960,
      "unclipped_grad_norm": 0.8050268888473511
    },
    {
      "bits_per_byte": 2.2641018956227765,
      "learning_rate": 0.0002962699392536863,
      "loss_nats": 1.569355845451355,
      "step": 970,
      "unclipped_grad_norm": 0.8510206341743469
    },
    {
      "bits_per_byte": 2.35890939570271,
      "learning_rate": 0.0002961385448378161,
      "loss_nats": 1.6350713968276978,
      "step": 980,
      "unclipped_grad_norm": 0.8616457581520081
    },
    {
      "bits_per_byte": 2.3101203254100433,
      "learning_rate": 0.00029600490962603817,
      "loss_nats": 1.6012533903121948,
      "step": 990,
      "unclipped_grad_norm": 0.8957601189613342
    },
    {
      "bits_per_byte": 2.2887248237294555,
      "learning_rate": 0.00029586903590180956,
      "loss_nats": 1.5864231586456299,
      "step": 1000,
      "unclipped_grad_norm": 1.0717068910598755
    },
    {
      "bits_per_byte": 2.2687179099719343,
      "learning_rate": 0.00029573092598683735,
      "loss_nats": 1.572555422782898,
      "step": 1010,
      "unclipped_grad_norm": 0.9444129467010498
    },
    {
      "bits_per_byte": 2.2310683279583023,
      "learning_rate": 0.00029559058224103895,
      "loss_nats": 1.5464587211608887,
      "step": 1020,
      "unclipped_grad_norm": 0.9090887308120728
    },
    {
      "bits_per_byte": 2.2341806979909666,
      "learning_rate": 0.0002954480070625017,
      "loss_nats": 1.5486160516738892,
      "step": 1030,
      "unclipped_grad_norm": 0.8268627524375916
    },
    {
      "bits_per_byte": 2.26161898209227,
      "learning_rate": 0.00029530320288744205,
      "loss_nats": 1.5676348209381104,
      "step": 1040,
      "unclipped_grad_norm": 0.7536599636077881
    },
    {
      "bits_per_byte": 2.2636605881406626,
      "learning_rate": 0.0002951561721901638,
      "loss_nats": 1.5690499544143677,
      "step": 1050,
      "unclipped_grad_norm": 0.8793148398399353
    },
    {
      "bits_per_byte": 2.1893296864377567,
      "learning_rate": 0.00029500691748301594,
      "loss_nats": 1.51752769947052,
      "step": 1060,
      "unclipped_grad_norm": 0.9039056301116943
    },
    {
      "bits_per_byte": 2.2431745307178375,
      "learning_rate": 0.00029485544131634963,
      "loss_nats": 1.5548501014709473,
      "step": 1070,
      "unclipped_grad_norm": 0.862832248210907
    },
    {
      "bits_per_byte": 2.24520426996332,
      "learning_rate": 0.0002947017462784745,
      "loss_nats": 1.5562570095062256,
      "step": 1080,
      "unclipped_grad_norm": 0.9674477577209473
    },
    {
      "bits_per_byte": 2.1985577595351025,
      "learning_rate": 0.00029454583499561476,
      "loss_nats": 1.5239241123199463,
      "step": 1090,
      "unclipped_grad_norm": 0.7859865427017212
    },
    {
      "bits_per_byte": 2.2599985615558404,
      "learning_rate": 0.00029438771013186413,
      "loss_nats": 1.566511631011963,
      "step": 1100,
      "unclipped_grad_norm": 0.788394570350647
    },
    {
      "bits_per_byte": 2.119760812354591,
      "learning_rate": 0.0002942273743891401,
      "loss_nats": 1.469306230545044,
      "step": 1110,
      "unclipped_grad_norm": 0.8257600665092468
    },
    {
      "bits_per_byte": 2.1691607650231224,
      "learning_rate": 0.00029406483050713824,
      "loss_nats": 1.5035476684570312,
      "step": 1120,
      "unclipped_grad_norm": 0.9345623254776001
    },
    {
      "bits_per_byte": 2.1036112974732717,
      "learning_rate": 0.00029390008126328504,
      "loss_nats": 1.4581122398376465,
      "step": 1130,
      "unclipped_grad_norm": 0.6939264535903931
    },
    {
      "bits_per_byte": 2.2552164119659492,
      "learning_rate": 0.00029373312947269045,
      "loss_nats": 1.5631968975067139,
      "step": 1140,
      "unclipped_grad_norm": 0.8951012492179871
    },
    {
      "bits_per_byte": 2.1146625586524137,
      "learning_rate": 0.00029356397798809995,
      "loss_nats": 1.4657723903656006,
      "step": 1150,
      "unclipped_grad_norm": 0.9783032536506653
    },
    {
      "bits_per_byte": 2.1242843000375844,
      "learning_rate": 0.0002933926296998457,
      "loss_nats": 1.4724416732788086,
      "step": 1160,
      "unclipped_grad_norm": 0.8440186977386475
    },
    {
      "bits_per_byte": 2.174440976369912,
      "learning_rate": 0.0002932190875357972,
      "loss_nats": 1.5072076320648193,
      "step": 1170,
      "unclipped_grad_norm": 0.9151178002357483
    },
    {
      "bits_per_byte": 2.1585999983642408,
      "learning_rate": 0.00029304335446131126,
      "loss_nats": 1.496227502822876,
      "step": 1180,
      "unclipped_grad_norm": 0.7418354153633118
    },
    {
      "bits_per_byte": 2.091859378974514,
      "learning_rate": 0.00029286543347918116,
      "loss_nats": 1.4499664306640625,
      "step": 1190,
      "unclipped_grad_norm": 0.9675256609916687
    },
    {
      "bits_per_byte": 2.0676216920057677,
      "learning_rate": 0.0002926853276295856,
      "loss_nats": 1.4331661462783813,
      "step": 1200,
      "unclipped_grad_norm": 0.937224805355072
    },
    {
      "bits_per_byte": 2.116782072841647,
      "learning_rate": 0.0002925030399900368,
      "loss_nats": 1.4672415256500244,
      "step": 1210,
      "unclipped_grad_norm": 0.8679532408714294
    },
    {
      "bits_per_byte": 2.1441637746508166,
      "learning_rate": 0.00029231857367532726,
      "loss_nats": 1.4862210750579834,
      "step": 1220,
      "unclipped_grad_norm": 0.7687044143676758
    },
    {
      "bits_per_byte": 2.083470065265426,
      "learning_rate": 0.0002921319318374776,
      "loss_nats": 1.4441514015197754,
      "step": 1230,
      "unclipped_grad_norm": 0.7017022967338562
    },
    {
      "bits_per_byte": 2.025742368676816,
      "learning_rate": 0.00029194311766568174,
      "loss_nats": 1.4041376113891602,
      "step": 1240,
      "unclipped_grad_norm": 0.8267619013786316
    },
    {
      "bits_per_byte": 2.1229982137744323,
      "learning_rate": 0.000291752134386253,
      "loss_nats": 1.4715502262115479,
      "step": 1250,
      "unclipped_grad_norm": 1.0167953968048096
    },
    {
      "bits_per_byte": 2.098591123894586,
      "learning_rate": 0.00029155898526256856,
      "loss_nats": 1.4546325206756592,
      "step": 1260,
      "unclipped_grad_norm": 0.839733898639679
    },
    {
      "bits_per_byte": 2.0788139789586175,
      "learning_rate": 0.0002913636735950141,
      "loss_nats": 1.440924048423767,
      "step": 1270,
      "unclipped_grad_norm": 0.7740461230278015
    },
    {
      "bits_per_byte": 1.9768289549275753,
      "learning_rate": 0.0002911662027209272,
      "loss_nats": 1.370233416557312,
      "step": 1280,
      "unclipped_grad_norm": 0.6849040389060974
    },
    {
      "bits_per_byte": 2.098678835046526,
      "learning_rate": 0.00029096657601454013,
      "loss_nats": 1.45469331741333,
      "step": 1290,
      "unclipped_grad_norm": 0.7928483486175537
    },
    {
      "bits_per_byte": 2.041355641652745,
      "learning_rate": 0.00029076479688692266,
      "loss_nats": 1.4149599075317383,
      "step": 1300,
      "unclipped_grad_norm": 0.7687125205993652
    },
    {
      "bits_per_byte": 2.0466036922438238,
      "learning_rate": 0.00029056086878592323,
      "loss_nats": 1.4185975790023804,
      "step": 1310,
      "unclipped_grad_norm": 0.7732207179069519
    },
    {
      "bits_per_byte": 2.038361079735922,
      "learning_rate": 0.00029035479519611045,
      "loss_nats": 1.41288423538208,
      "step": 1320,
      "unclipped_grad_norm": 0.8016889691352844
    },
    {
      "bits_per_byte": 2.1092168999954937,
      "learning_rate": 0.0002901465796387134,
      "loss_nats": 1.4619977474212646,
      "step": 1330,
      "unclipped_grad_norm": 0.7918544411659241
    },
    {
      "bits_per_byte": 2.0672703314500547,
      "learning_rate": 0.00028993622567156154,
      "loss_nats": 1.432922601699829,
      "step": 1340,
      "unclipped_grad_norm": 0.9473646879196167
    },
    {
      "bits_per_byte": 2.0426272813732247,
      "learning_rate": 0.0002897237368890237,
      "loss_nats": 1.4158413410186768,
      "step": 1350,
      "unclipped_grad_norm": 0.7250363826751709
    },
    {
      "bits_per_byte": 2.10636370781768,
      "learning_rate": 0.0002895091169219469,
      "loss_nats": 1.4600200653076172,
      "step": 1360,
      "unclipped_grad_norm": 0.7928993105888367
    },
    {
      "bits_per_byte": 2.104627027009267,
      "learning_rate": 0.00028929236943759425,
      "loss_nats": 1.4588162899017334,
      "step": 1370,
      "unclipped_grad_norm": 0.794548749923706
    },
    {
      "bits_per_byte": 2.0561698832327657,
      "learning_rate": 0.00028907349813958225,
      "loss_nats": 1.4252283573150635,
      "step": 1380,
      "unclipped_grad_norm": 0.7989111542701721
    },
    {
      "bits_per_byte": 2.0527231789268243,
      "learning_rate": 0.0002888525067678175,
      "loss_nats": 1.4228392839431763,
      "step": 1390,
      "unclipped_grad_norm": 0.8791808485984802
    },
    {
      "bits_per_byte": 1.9977298345042815,
      "learning_rate": 0.00028862939909843273,
      "loss_nats": 1.384720802307129,
      "step": 1400,
      "unclipped_grad_norm": 0.7230417132377625
    },
    {
      "bits_per_byte": 2.02384763581226,
      "learning_rate": 0.0002884041789437225,
      "loss_nats": 1.4028242826461792,
      "step": 1410,
      "unclipped_grad_norm": 0.7468129396438599
    },
    {
      "bits_per_byte": 2.0636549121636167,
      "learning_rate": 0.0002881768501520778,
      "loss_nats": 1.4304165840148926,
      "step": 1420,
      "unclipped_grad_norm": 0.7919746041297913
    },
    {
      "bits_per_byte": 2.0795395737626077,
      "learning_rate": 0.0002879474166079205,
      "loss_nats": 1.4414269924163818,
      "step": 1430,
      "unclipped_grad_norm": 0.6952440738677979
    },
    {
      "bits_per_byte": 2.0428340045195617,
      "learning_rate": 0.00028771588223163694,
      "loss_nats": 1.4159846305847168,
      "step": 1440,
      "unclipped_grad_norm": 0.9689956307411194
    },
    {
      "bits_per_byte": 1.9802144334098097,
      "learning_rate": 0.00028748225097951054,
      "loss_nats": 1.3725800514221191,
      "step": 1450,
      "unclipped_grad_norm": 0.7872179746627808
    },
    {
      "bits_per_byte": 2.013908586436248,
      "learning_rate": 0.000287246526843655,
      "loss_nats": 1.39593505859375,
      "step": 1460,
      "unclipped_grad_norm": 0.7631422281265259
    },
    {
      "bits_per_byte": 1.9493597139491168,
      "learning_rate": 0.00028700871385194533,
      "loss_nats": 1.3511931896209717,
      "step": 1470,
      "unclipped_grad_norm": 0.7200304865837097
    },
    {
      "bits_per_byte": 2.0209578113298123,
      "learning_rate": 0.00028676881606794945,
      "loss_nats": 1.4008212089538574,
      "step": 1480,
      "unclipped_grad_norm": 0.7176085114479065
    },
    {
      "bits_per_byte": 2.0333618880406417,
      "learning_rate": 0.00028652683759085854,
      "loss_nats": 1.409419059753418,
      "step": 1490,
      "unclipped_grad_norm": 0.7242412567138672
    },
    {
      "bits_per_byte": 2.033254742849154,
      "learning_rate": 0.0002862827825554171,
      "loss_nats": 1.4093447923660278,
      "step": 1500,
      "unclipped_grad_norm": 0.7850398421287537
    },
    {
      "bits_per_byte": 2.0298035669942904,
      "learning_rate": 0.00028603665513185245,
      "loss_nats": 1.4069526195526123,
      "step": 1510,
      "unclipped_grad_norm": 0.7650813460350037
    },
    {
      "bits_per_byte": 1.981937699571455,
      "learning_rate": 0.000285788459525803,
      "loss_nats": 1.373774528503418,
      "step": 1520,
      "unclipped_grad_norm": 0.7211347222328186
    },
    {
      "bits_per_byte": 2.0555194448472025,
      "learning_rate": 0.00028553819997824707,
      "loss_nats": 1.4247775077819824,
      "step": 1530,
      "unclipped_grad_norm": 0.7807993292808533
    },
    {
      "bits_per_byte": 1.9955152139091212,
      "learning_rate": 0.00028528588076542964,
      "loss_nats": 1.3831857442855835,
      "step": 1540,
      "unclipped_grad_norm": 0.9850755929946899
    },
    {
      "bits_per_byte": 2.0666652964843193,
      "learning_rate": 0.00028503150619878996,
      "loss_nats": 1.4325032234191895,
      "step": 1550,
      "unclipped_grad_norm": 0.8105318546295166
    },
    {
      "bits_per_byte": 1.9693205363562067,
      "learning_rate": 0.0002847750806248876,
      "loss_nats": 1.365028977394104,
      "step": 1560,
      "unclipped_grad_norm": 0.6913962364196777
    },
    {
      "bits_per_byte": 2.059840508950131,
      "learning_rate": 0.000284516608425328,
      "loss_nats": 1.4277726411819458,
      "step": 1570,
      "unclipped_grad_norm": 0.725780189037323
    },
    {
      "bits_per_byte": 1.9988298355392002,
      "learning_rate": 0.000284256094016688,
      "loss_nats": 1.3854832649230957,
      "step": 1580,
      "unclipped_grad_norm": 0.7167323231697083
    },
    {
      "bits_per_byte": 2.0030889858878176,
      "learning_rate": 0.0002839935418504399,
      "loss_nats": 1.3884354829788208,
      "step": 1590,
      "unclipped_grad_norm": 0.7256693840026855
    },
    {
      "bits_per_byte": 2.027964040560662,
      "learning_rate": 0.000283728956412876,
      "loss_nats": 1.4056775569915771,
      "step": 1600,
      "unclipped_grad_norm": 0.709171712398529
    },
    {
      "bits_per_byte": 1.961726126459407,
      "learning_rate": 0.00028346234222503136,
      "loss_nats": 1.3597649335861206,
      "step": 1610,
      "unclipped_grad_norm": 0.6563887000083923
    },
    {
      "bits_per_byte": 1.9260072216843611,
      "learning_rate": 0.00028319370384260675,
      "loss_nats": 1.3350064754486084,
      "step": 1620,
      "unclipped_grad_norm": 0.743120014667511
    },
    {
      "bits_per_byte": 1.954758421342351,
      "learning_rate": 0.00028292304585589105,
      "loss_nats": 1.3549352884292603,
      "step": 1630,
      "unclipped_grad_norm": 0.6699017286300659
    },
    {
      "bits_per_byte": 1.9950130245686017,
      "learning_rate": 0.0002826503728896824,
      "loss_nats": 1.3828376531600952,
      "step": 1640,
      "unclipped_grad_norm": 0.7793066501617432
    },
    {
      "bits_per_byte": 2.0499475509245495,
      "learning_rate": 0.0002823756896032095,
      "loss_nats": 1.4209153652191162,
      "step": 1650,
      "unclipped_grad_norm": 0.6984328031539917
    },
    {
      "bits_per_byte": 1.996117325169792,
      "learning_rate": 0.0002820990006900518,
      "loss_nats": 1.3836030960083008,
      "step": 1660,
      "unclipped_grad_norm": 0.7241947054862976
    },
    {
      "bits_per_byte": 2.017874850330446,
      "learning_rate": 0.00028182031087805947,
      "loss_nats": 1.3986842632293701,
      "step": 1670,
      "unclipped_grad_norm": 0.7746724486351013
    },
    {
      "bits_per_byte": 2.0244201660569825,
      "learning_rate": 0.0002815396249292725,
      "loss_nats": 1.4032211303710938,
      "step": 1680,
      "unclipped_grad_norm": 0.7699829936027527
    },
    {
      "bits_per_byte": 2.06043436504356,
      "learning_rate": 0.0002812569476398391,
      "loss_nats": 1.4281842708587646,
      "step": 1690,
      "unclipped_grad_norm": 0.7417407035827637
    },
    {
      "bits_per_byte": 1.95363795437198,
      "learning_rate": 0.00028097228383993446,
      "loss_nats": 1.354158639907837,
      "step": 1700,
      "unclipped_grad_norm": 0.6731021404266357
    },
    {
      "bits_per_byte": 1.8890682719840945,
      "learning_rate": 0.0002806856383936774,
      "loss_nats": 1.309402346611023,
      "step": 1710,
      "unclipped_grad_norm": 0.6633850932121277
    },
    {
      "bits_per_byte": 1.8893066399381904,
      "learning_rate": 0.00028039701619904774,
      "loss_nats": 1.3095675706863403,
      "step": 1720,
      "unclipped_grad_norm": 0.7114297151565552
    },
    {
      "bits_per_byte": 1.8703618910323976,
      "learning_rate": 0.0002801064221878024,
      "loss_nats": 1.296436071395874,
      "step": 1730,
      "unclipped_grad_norm": 0.6913781762123108
    },
    {
      "bits_per_byte": 1.988379825707474,
      "learning_rate": 0.00027981386132539134,
      "loss_nats": 1.3782398700714111,
      "step": 1740,
      "unclipped_grad_norm": 0.7100496888160706
    },
    {
      "bits_per_byte": 2.0086424776668284,
      "learning_rate": 0.0002795193386108724,
      "loss_nats": 1.392284870147705,
      "step": 1750,
      "unclipped_grad_norm": 0.7558912038803101
    },
    {
      "bits_per_byte": 1.996569639541561,
      "learning_rate": 0.00027922285907682625,
      "loss_nats": 1.3839166164398193,
      "step": 1760,
      "unclipped_grad_norm": 0.6800652742385864
    },
    {
      "bits_per_byte": 2.0072186333003352,
      "learning_rate": 0.00027892442778927005,
      "loss_nats": 1.3912979364395142,
      "step": 1770,
      "unclipped_grad_norm": 0.7516586780548096
    },
    {
      "bits_per_byte": 1.8758278436421194,
      "learning_rate": 0.00027862404984757116,
      "loss_nats": 1.300224781036377,
      "step": 1780,
      "unclipped_grad_norm": 0.6571854948997498
    },
    {
      "bits_per_byte": 1.963898439322455,
      "learning_rate": 0.0002783217303843597,
      "loss_nats": 1.3612706661224365,
      "step": 1790,
      "unclipped_grad_norm": 0.6896912455558777
    },
    {
      "bits_per_byte": 1.880059304783948,
      "learning_rate": 0.00027801747456544134,
      "loss_nats": 1.3031578063964844,
      "step": 1800,
      "unclipped_grad_norm": 0.6719921827316284
    },
    {
      "bits_per_byte": 1.9405041552735398,
      "learning_rate": 0.00027771128758970843,
      "loss_nats": 1.3450549840927124,
      "step": 1810,
      "unclipped_grad_norm": 0.7259154915809631
    },
    {
      "bits_per_byte": 1.9116848504858082,
      "learning_rate": 0.00027740317468905165,
      "loss_nats": 1.3250789642333984,
      "step": 1820,
      "unclipped_grad_norm": 0.6911426782608032
    },
    {
      "bits_per_byte": 1.887859061965878,
      "learning_rate": 0.0002770931411282704,
      "loss_nats": 1.3085641860961914,
      "step": 1830,
      "unclipped_grad_norm": 0.6866450309753418
    },
    {
      "bits_per_byte": 1.904207044826293,
      "learning_rate": 0.0002767811922049827,
      "loss_nats": 1.3198957443237305,
      "step": 1840,
      "unclipped_grad_norm": 0.6746156215667725
    },
    {
      "bits_per_byte": 1.913379911492712,
      "learning_rate": 0.00027646733324953503,
      "loss_nats": 1.326253890991211,
      "step": 1850,
      "unclipped_grad_norm": 0.6992620825767517
    },
    {
      "bits_per_byte": 1.8924212457453158,
      "learning_rate": 0.00027615156962491097,
      "loss_nats": 1.311726450920105,
      "step": 1860,
      "unclipped_grad_norm": 0.7686212658882141
    },
    {
      "bits_per_byte": 1.9034685513234884,
      "learning_rate": 0.0002758339067266396,
      "loss_nats": 1.3193838596343994,
      "step": 1870,
      "unclipped_grad_norm": 0.6957508325576782
    },
    {
      "bits_per_byte": 1.8975736739825146,
      "learning_rate": 0.0002755143499827035,
      "loss_nats": 1.3152978420257568,
      "step": 1880,
      "unclipped_grad_norm": 0.686790406703949
    },
    {
      "bits_per_byte": 1.9108474669587572,
      "learning_rate": 0.0002751929048534456,
      "loss_nats": 1.3244985342025757,
      "step": 1890,
      "unclipped_grad_norm": 0.6941424608230591
    },
    {
      "bits_per_byte": 1.9178445811091105,
      "learning_rate": 0.0002748695768314763,
      "loss_nats": 1.3293485641479492,
      "step": 1900,
      "unclipped_grad_norm": 0.6622544527053833
    },
    {
      "bits_per_byte": 1.857984643615103,
      "learning_rate": 0.00027454437144157947,
      "loss_nats": 1.2878568172454834,
      "step": 1910,
      "unclipped_grad_norm": 0.6576761603355408
    },
    {
      "bits_per_byte": 1.8615044405476613,
      "learning_rate": 0.00027421729424061787,
      "loss_nats": 1.2902965545654297,
      "step": 1920,
      "unclipped_grad_norm": 0.7134246826171875
    },
    {
      "bits_per_byte": 1.9495869030309065,
      "learning_rate": 0.0002738883508174385,
      "loss_nats": 1.3513506650924683,
      "step": 1930,
      "unclipped_grad_norm": 0.68708336353302
    },
    {
      "bits_per_byte": 1.8269496863363057,
      "learning_rate": 0.0002735575467927768,
      "loss_nats": 1.2663450241088867,
      "step": 1940,
      "unclipped_grad_norm": 0.671497106552124
    },
    {
      "bits_per_byte": 1.904890503880822,
      "learning_rate": 0.0002732248878191609,
      "loss_nats": 1.3203694820404053,
      "step": 1950,
      "unclipped_grad_norm": 0.6914620995521545
    },
    {
      "bits_per_byte": 1.81755221032786,
      "learning_rate": 0.0002728903795808148,
      "loss_nats": 1.259831190109253,
      "step": 1960,
      "unclipped_grad_norm": 0.654005765914917
    },
    {
      "bits_per_byte": 1.9009173606805894,
      "learning_rate": 0.0002725540277935614,
      "loss_nats": 1.3176155090332031,
      "step": 1970,
      "unclipped_grad_norm": 0.6446248292922974
    },
    {
      "bits_per_byte": 1.9534350148439619,
      "learning_rate": 0.00027221583820472456,
      "loss_nats": 1.354017972946167,
      "step": 1980,
      "unclipped_grad_norm": 0.6792307496070862
    },
    {
      "bits_per_byte": 1.9578730271494766,
      "learning_rate": 0.0002718758165930314,
      "loss_nats": 1.357094168663025,
      "step": 1990,
      "unclipped_grad_norm": 0.8862876892089844
    },
    {
      "bits_per_byte": 1.9120136813142579,
      "learning_rate": 0.0002715339687685131,
      "loss_nats": 1.3253068923950195,
      "step": 2000,
      "unclipped_grad_norm": 0.6821991801261902
    },
    {
      "bits_per_byte": 1.879799782963796,
      "learning_rate": 0.00027119030057240576,
      "loss_nats": 1.3029779195785522,
      "step": 2010,
      "unclipped_grad_norm": 0.6441471576690674
    },
    {
      "bits_per_byte": 1.8960643542385427,
      "learning_rate": 0.0002708448178770506,
      "loss_nats": 1.3142516613006592,
      "step": 2020,
      "unclipped_grad_norm": 0.6215534806251526
    },
    {
      "bits_per_byte": 1.9880719767624297,
      "learning_rate": 0.00027049752658579376,
      "loss_nats": 1.3780264854431152,
      "step": 2030,
      "unclipped_grad_norm": 0.6645570993423462
    },
    {
      "bits_per_byte": 1.9132447331291338,
      "learning_rate": 0.00027014843263288514,
      "loss_nats": 1.326160192489624,
      "step": 2040,
      "unclipped_grad_norm": 0.6700881719589233
    },
    {
      "bits_per_byte": 1.943815681215903,
      "learning_rate": 0.00026979754198337724,
      "loss_nats": 1.3473503589630127,
      "step": 2050,
      "unclipped_grad_norm": 0.6627214550971985
    },
    {
      "bits_per_byte": 1.9209480080439298,
      "learning_rate": 0.00026944486063302305,
      "loss_nats": 1.331499695777893,
      "step": 2060,
      "unclipped_grad_norm": 0.6492555737495422
    },
    {
      "bits_per_byte": 1.8667327131338907,
      "learning_rate": 0.0002690903946081738,
      "loss_nats": 1.2939205169677734,
      "step": 2070,
      "unclipped_grad_norm": 0.733633279800415
    },
    {
      "bits_per_byte": 1.8747223391623733,
      "learning_rate": 0.00026873414996567596,
      "loss_nats": 1.2994585037231445,
      "step": 2080,
      "unclipped_grad_norm": 0.7179054021835327
    },
    {
      "bits_per_byte": 1.870721334772701,
      "learning_rate": 0.00026837613279276736,
      "loss_nats": 1.2966852188110352,
      "step": 2090,
      "unclipped_grad_norm": 0.6603640913963318
    },
    {
      "bits_per_byte": 1.8546041526297437,
      "learning_rate": 0.0002680163492069739,
      "loss_nats": 1.2855136394500732,
      "step": 2100,
      "unclipped_grad_norm": 0.6752005815505981
    },
    {
      "bits_per_byte": 1.9160201891487578,
      "learning_rate": 0.00026765480535600414,
      "loss_nats": 1.3280839920043945,
      "step": 2110,
      "unclipped_grad_norm": 0.6652648448944092
    },
    {
      "bits_per_byte": 1.871099696604599,
      "learning_rate": 0.0002672915074176452,
      "loss_nats": 1.2969474792480469,
      "step": 2120,
      "unclipped_grad_norm": 0.7032200694084167
    },
    {
      "bits_per_byte": 1.8333405616423677,
      "learning_rate": 0.00026692646159965624,
      "loss_nats": 1.2707748413085938,
      "step": 2130,
      "unclipped_grad_norm": 0.6664465069770813
    },
    {
      "bits_per_byte": 1.966146252569232,
      "learning_rate": 0.0002665596741396633,
      "loss_nats": 1.3628287315368652,
      "step": 2140,
      "unclipped_grad_norm": 0.7021564245223999
    },
    {
      "bits_per_byte": 1.835432558607463,
      "learning_rate": 0.000266191151305052,
      "loss_nats": 1.2722249031066895,
      "step": 2150,
      "unclipped_grad_norm": 0.6662127375602722
    },
    {
      "bits_per_byte": 1.8787831935145458,
      "learning_rate": 0.00026582089939286075,
      "loss_nats": 1.3022732734680176,
      "step": 2160,
      "unclipped_grad_norm": 0.6593465209007263
    },
    {
      "bits_per_byte": 1.9513302911627028,
      "learning_rate": 0.0002654489247296732,
      "loss_nats": 1.3525590896606445,
      "step": 2170,
      "unclipped_grad_norm": 0.721863329410553
    },
    {
      "bits_per_byte": 1.8313610413309367,
      "learning_rate": 0.00026507523367151007,
      "loss_nats": 1.2694027423858643,
      "step": 2180,
      "unclipped_grad_norm": 0.661525547504425
    },
    {
      "bits_per_byte": 1.8437549710653651,
      "learning_rate": 0.00026469983260372037,
      "loss_nats": 1.2779935598373413,
      "step": 2190,
      "unclipped_grad_norm": 0.6355888247489929
    },
    {
      "bits_per_byte": 1.8767926663134598,
      "learning_rate": 0.0002643227279408727,
      "loss_nats": 1.3008935451507568,
      "step": 2200,
      "unclipped_grad_norm": 0.650397777557373
    },
    {
      "bits_per_byte": 1.8339084483555168,
      "learning_rate": 0.0002639439261266453,
      "loss_nats": 1.2711684703826904,
      "step": 2210,
      "unclipped_grad_norm": 0.6309477686882019
    },
    {
      "bits_per_byte": 1.8366491638796665,
      "learning_rate": 0.000263563433633716,
      "loss_nats": 1.2730681896209717,
      "step": 2220,
      "unclipped_grad_norm": 0.6397845149040222
    },
    {
      "bits_per_byte": 1.9122785345965865,
      "learning_rate": 0.00026318125696365156,
      "loss_nats": 1.3254904747009277,
      "step": 2230,
      "unclipped_grad_norm": 0.7579862475395203
    },
    {
      "bits_per_byte": 1.8301518313127203,
      "learning_rate": 0.000262797402646797,
      "loss_nats": 1.2685645818710327,
      "step": 2240,
      "unclipped_grad_norm": 0.6486738324165344
    },
    {
      "bits_per_byte": 1.854603636681791,
      "learning_rate": 0.00026241187724216334,
      "loss_nats": 1.2855132818222046,
      "step": 2250,
      "unclipped_grad_norm": 0.6761392951011658
    },
    {
      "bits_per_byte": 1.8292511581701518,
      "learning_rate": 0.0002620246873373162,
      "loss_nats": 1.2679402828216553,
      "step": 2260,
      "unclipped_grad_norm": 0.6474469900131226
    },
    {
      "bits_per_byte": 1.8217264012469514,
      "learning_rate": 0.00026163583954826264,
      "loss_nats": 1.26272451877594,
      "step": 2270,
      "unclipped_grad_norm": 0.6697631478309631
    },
    {
      "bits_per_byte": 1.839495476751446,
      "learning_rate": 0.0002612453405193386,
      "loss_nats": 1.275041103363037,
      "step": 2280,
      "unclipped_grad_norm": 0.672606885433197
    },
    {
      "bits_per_byte": 1.8894086256501519,
      "learning_rate": 0.000260853196923095,
      "loss_nats": 1.309638261795044,
      "step": 2290,
      "unclipped_grad_norm": 0.7677630186080933
    },
    {
      "bits_per_byte": 1.8035906587308173,
      "learning_rate": 0.0002604594154601839,
      "loss_nats": 1.2501537799835205,
      "step": 2300,
      "unclipped_grad_norm": 0.6455734372138977
    },
    {
      "bits_per_byte": 1.834394643309506,
      "learning_rate": 0.0002600640028592441,
      "loss_nats": 1.2715054750442505,
      "step": 2310,
      "unclipped_grad_norm": 0.6606747508049011
    },
    {
      "bits_per_byte": 1.8063805612931139,
      "learning_rate": 0.0002596669658767859,
      "loss_nats": 1.2520875930786133,
      "step": 2320,
      "unclipped_grad_norm": 0.634936511516571
    },
    {
      "bits_per_byte": 1.8747587994843562,
      "learning_rate": 0.000259268311297076,
      "loss_nats": 1.2994837760925293,
      "step": 2330,
      "unclipped_grad_norm": 0.67148357629776
    },
    {
      "bits_per_byte": 1.7920827836136342,
      "learning_rate": 0.00025886804593202114,
      "loss_nats": 1.242177128791809,
      "step": 2340,
      "unclipped_grad_norm": 0.6197777390480042
    },
    {
      "bits_per_byte": 1.8459198886744261,
      "learning_rate": 0.00025846617662105193,
      "loss_nats": 1.2794941663742065,
      "step": 2350,
      "unclipped_grad_norm": 0.6368024349212646
    },
    {
      "bits_per_byte": 1.8583048753110096,
      "learning_rate": 0.00025806271023100637,
      "loss_nats": 1.288078784942627,
      "step": 2360,
      "unclipped_grad_norm": 0.662507951259613
    },
    {
      "bits_per_byte": 1.8678564477746282,
      "learning_rate": 0.00025765765365601165,
      "loss_nats": 1.2946994304656982,
      "step": 2370,
      "unclipped_grad_norm": 0.6344826221466064
    },
    {
      "bits_per_byte": 1.8307873072076584,
      "learning_rate": 0.00025725101381736707,
      "loss_nats": 1.2690050601959229,
      "step": 2380,
      "unclipped_grad_norm": 0.680084764957428
    },
    {
      "bits_per_byte": 1.888488346485385,
      "learning_rate": 0.00025684279766342567,
      "loss_nats": 1.3090003728866577,
      "step": 2390,
      "unclipped_grad_norm": 0.6502870917320251
    },
    {
      "bits_per_byte": 1.8376442554975587,
      "learning_rate": 0.00025643301216947504,
      "loss_nats": 1.2737579345703125,
      "step": 2400,
      "unclipped_grad_norm": 0.6415471434593201
    },
    {
      "bits_per_byte": 1.8272317378837208,
      "learning_rate": 0.00025602166433761866,
      "loss_nats": 1.26654052734375,
      "step": 2410,
      "unclipped_grad_norm": 0.6551245450973511
    },
    {
      "bits_per_byte": 1.8084292186301916,
      "learning_rate": 0.0002556087611966561,
      "loss_nats": 1.2535076141357422,
      "step": 2420,
      "unclipped_grad_norm": 0.6829442381858826
    },
    {
      "bits_per_byte": 1.8282901191171306,
      "learning_rate": 0.00025519430980196277,
      "loss_nats": 1.2672741413116455,
      "step": 2430,
      "unclipped_grad_norm": 0.6490575075149536
    },
    {
      "bits_per_byte": 1.8846385148458211,
      "learning_rate": 0.0002547783172353695,
      "loss_nats": 1.3063318729400635,
      "step": 2440,
      "unclipped_grad_norm": 0.6281237006187439
    },
    {
      "bits_per_byte": 1.8146233457836665,
      "learning_rate": 0.0002543607906050415,
      "loss_nats": 1.2578010559082031,
      "step": 2450,
      "unclipped_grad_norm": 0.634181559085846
    },
    {
      "bits_per_byte": 1.8319019267679002,
      "learning_rate": 0.0002539417370453568,
      "loss_nats": 1.2697776556015015,
      "step": 2460,
      "unclipped_grad_norm": 0.6390114426612854
    },
    {
      "bits_per_byte": 1.8154657168075925,
      "learning_rate": 0.00025352116371678464,
      "loss_nats": 1.2583849430084229,
      "step": 2470,
      "unclipped_grad_norm": 0.6522298455238342
    },
    {
      "bits_per_byte": 1.8417133650169726,
      "learning_rate": 0.0002530990778057627,
      "loss_nats": 1.276578426361084,
      "step": 2480,
      "unclipped_grad_norm": 0.6301186680793762
    },
    {
      "bits_per_byte": 1.7981788806561174,
      "learning_rate": 0.00025267548652457475,
      "loss_nats": 1.246402621269226,
      "step": 2490,
      "unclipped_grad_norm": 0.613478422164917
    },
    {
      "bits_per_byte": 1.8422764362158974,
      "learning_rate": 0.00025225039711122696,
      "loss_nats": 1.2769687175750732,
      "step": 2500,
      "unclipped_grad_norm": 0.6680112481117249
    },
    {
      "bits_per_byte": 1.7934540012889635,
      "learning_rate": 0.00025182381682932464,
      "loss_nats": 1.2431275844573975,
      "step": 2510,
      "unclipped_grad_norm": 0.6324169635772705
    },
    {
      "bits_per_byte": 1.8373082013977726,
      "learning_rate": 0.0002513957529679477,
      "loss_nats": 1.2735249996185303,
      "step": 2520,
      "unclipped_grad_norm": 0.6562070250511169
    },
    {
      "bits_per_byte": 1.8465858054985669,
      "learning_rate": 0.00025096621284152656,
      "loss_nats": 1.2799557447433472,
      "step": 2530,
      "unclipped_grad_norm": 0.67587810754776
    },
    {
      "bits_per_byte": 1.8180633427662245,
      "learning_rate": 0.0002505352037897168,
      "loss_nats": 1.2601854801177979,
      "step": 2540,
      "unclipped_grad_norm": 0.6210066080093384
    },
    {
      "bits_per_byte": 1.8116361791208304,
      "learning_rate": 0.0002501027331772739,
      "loss_nats": 1.2557305097579956,
      "step": 2550,
      "unclipped_grad_norm": 0.656908392906189
    },
    {
      "bits_per_byte": 1.8229865181298233,
      "learning_rate": 0.00024966880839392746,
      "loss_nats": 1.2635979652404785,
      "step": 2560,
      "unclipped_grad_norm": 0.6474040746688843
    },
    {
      "bits_per_byte": 1.8231736352539618,
      "learning_rate": 0.0002492334368542548,
      "loss_nats": 1.2637276649475098,
      "step": 2570,
      "unclipped_grad_norm": 0.6998929977416992
    },
    {
      "bits_per_byte": 1.8511411099719701,
      "learning_rate": 0.00024879662599755416,
      "loss_nats": 1.2831132411956787,
      "step": 2580,
      "unclipped_grad_norm": 0.6996825337409973
    },
    {
      "bits_per_byte": 1.805761251767357,
      "learning_rate": 0.00024835838328771803,
      "loss_nats": 1.251658320426941,
      "step": 2590,
      "unclipped_grad_norm": 0.6730060577392578
    },
    {
      "bits_per_byte": 1.7987187341971758,
      "learning_rate": 0.0002479187162131051,
      "loss_nats": 1.246776819229126,
      "step": 2600,
      "unclipped_grad_norm": 0.6812606453895569
    },
    {
      "bits_per_byte": 1.7378126262379698,
      "learning_rate": 0.00024747763228641264,
      "loss_nats": 1.2045599222183228,
      "step": 2610,
      "unclipped_grad_norm": 0.6152759194374084
    },
    {
      "bits_per_byte": 1.8067871282797534,
      "learning_rate": 0.00024703513904454795,
      "loss_nats": 1.2523694038391113,
      "step": 2620,
      "unclipped_grad_norm": 0.6381004452705383
    },
    {
      "bits_per_byte": 1.815846142431301,
      "learning_rate": 0.00024659124404849956,
      "loss_nats": 1.2586486339569092,
      "step": 2630,
      "unclipped_grad_norm": 0.6937149167060852
    },
    {
      "bits_per_byte": 1.6830253170311609,
      "learning_rate": 0.00024614595488320833,
      "loss_nats": 1.1665842533111572,
      "step": 2640,
      "unclipped_grad_norm": 0.5872177481651306
    },
    {
      "bits_per_byte": 1.7705318096166665,
      "learning_rate": 0.00024569927915743745,
      "loss_nats": 1.2272391319274902,
      "step": 2650,
      "unclipped_grad_norm": 0.6221135854721069
    },
    {
      "bits_per_byte": 1.813369076310924,
      "learning_rate": 0.0002452512245036427,
      "loss_nats": 1.2569316625595093,
      "step": 2660,
      "unclipped_grad_norm": 0.6537755727767944
    },
    {
      "bits_per_byte": 1.8137383230623265,
      "learning_rate": 0.00024480179857784177,
      "loss_nats": 1.2571876049041748,
      "step": 2670,
      "unclipped_grad_norm": 0.6585802435874939
    },
    {
      "bits_per_byte": 1.7472491423081613,
      "learning_rate": 0.00024435100905948387,
      "loss_nats": 1.2111008167266846,
      "step": 2680,
      "unclipped_grad_norm": 0.6200863122940063
    },
    {
      "bits_per_byte": 1.9002486921340347,
      "learning_rate": 0.00024389886365131793,
      "loss_nats": 1.3171520233154297,
      "step": 2690,
      "unclipped_grad_norm": 0.6396089792251587
    },
    {
      "bits_per_byte": 1.8072413344606821,
      "learning_rate": 0.00024344537007926163,
      "loss_nats": 1.252684235572815,
      "step": 2700,
      "unclipped_grad_norm": 0.6738898158073425
    },
    {
      "bits_per_byte": 1.7328827435509875,
      "learning_rate": 0.00024299053609226878,
      "loss_nats": 1.2011427879333496,
      "step": 2710,
      "unclipped_grad_norm": 0.7000491619110107
    },
    {
      "bits_per_byte": 1.7288494064229527,
      "learning_rate": 0.00024253436946219733,
      "loss_nats": 1.1983470916748047,
      "step": 2720,
      "unclipped_grad_norm": 0.6326305270195007
    },
    {
      "bits_per_byte": 1.776559801529408,
      "learning_rate": 0.0002420768779836764,
      "loss_nats": 1.2314174175262451,
      "step": 2730,
      "unclipped_grad_norm": 0.647907555103302
    },
    {
      "bits_per_byte": 1.783826928441616,
      "learning_rate": 0.00024161806947397305,
      "loss_nats": 1.2364546060562134,
      "step": 2740,
      "unclipped_grad_norm": 0.6690548658370972
    },
    {
      "bits_per_byte": 1.7821005665922551,
      "learning_rate": 0.00024115795177285886,
      "loss_nats": 1.2352579832077026,
      "step": 2750,
      "unclipped_grad_norm": 0.664342999458313
    },
    {
      "bits_per_byte": 1.8057218677403093,
      "learning_rate": 0.0002406965327424758,
      "loss_nats": 1.2516310214996338,
      "step": 2760,
      "unclipped_grad_norm": 0.6517228484153748
    },
    {
      "bits_per_byte": 1.7048344369870736,
      "learning_rate": 0.00024023382026720197,
      "loss_nats": 1.1817011833190918,
      "step": 2770,
      "unclipped_grad_norm": 0.684334397315979
    },
    {
      "bits_per_byte": 1.7173274287283988,
      "learning_rate": 0.00023976982225351698,
      "loss_nats": 1.19036066532135,
      "step": 2780,
      "unclipped_grad_norm": 0.6013396382331848
    },
    {
      "bits_per_byte": 1.804812079517245,
      "learning_rate": 0.0002393045466298666,
      "loss_nats": 1.2510004043579102,
      "step": 2790,
      "unclipped_grad_norm": 0.6293013095855713
    },
    {
      "bits_per_byte": 1.8557677872454814,
      "learning_rate": 0.00023883800134652763,
      "loss_nats": 1.2863202095031738,
      "step": 2800,
      "unclipped_grad_norm": 0.7250765562057495
    },
    {
      "bits_per_byte": 1.7793906359626097,
      "learning_rate": 0.0002383701943754715,
      "loss_nats": 1.233379602432251,
      "step": 2810,
      "unclipped_grad_norm": 0.7088303565979004
    },
    {
      "bits_per_byte": 1.7744306563117265,
      "learning_rate": 0.00023790113371022884,
      "loss_nats": 1.2299416065216064,
      "step": 2820,
      "unclipped_grad_norm": 0.6359960436820984
    },
    {
      "bits_per_byte": 1.8690950668261421,
      "learning_rate": 0.0002374308273657521,
      "loss_nats": 1.295557975769043,
      "step": 2830,
      "unclipped_grad_norm": 0.6622759103775024
    },
    {
      "bits_per_byte": 1.7494551637707785,
      "learning_rate": 0.00023695928337827911,
      "loss_nats": 1.2126299142837524,
      "step": 2840,
      "unclipped_grad_norm": 0.6139791011810303
    },
    {
      "bits_per_byte": 1.7661483158114752,
      "learning_rate": 0.00023648650980519564,
      "loss_nats": 1.22420072555542,
      "step": 2850,
      "unclipped_grad_norm": 0.6314926743507385
    },
    {
      "bits_per_byte": 1.7415515290677273,
      "learning_rate": 0.00023601251472489757,
      "loss_nats": 1.2071515321731567,
      "step": 2860,
      "unclipped_grad_norm": 0.664522111415863
    },
    {
      "bits_per_byte": 1.8281711071227333,
      "learning_rate": 0.00023553730623665305,
      "loss_nats": 1.2671916484832764,
      "step": 2870,
      "unclipped_grad_norm": 0.6572005152702332
    },
    {
      "bits_per_byte": 1.8333617155084239,
      "learning_rate": 0.00023506089246046403,
      "loss_nats": 1.2707895040512085,
      "step": 2880,
      "unclipped_grad_norm": 0.6237366795539856
    },
    {
      "bits_per_byte": 1.7312996432497951,
      "learning_rate": 0.00023458328153692736,
      "loss_nats": 1.2000454664230347,
      "step": 2890,
      "unclipped_grad_norm": 0.6335425972938538
    },
    {
      "bits_per_byte": 1.7034657990515074,
      "learning_rate": 0.0002341044816270961,
      "loss_nats": 1.1807525157928467,
      "step": 2900,
      "unclipped_grad_norm": 0.6068143844604492
    },
    {
      "bits_per_byte": 1.7597350827608012,
      "learning_rate": 0.00023362450091233953,
      "loss_nats": 1.2197554111480713,
      "step": 2910,
      "unclipped_grad_norm": 0.6452642679214478
    },
    {
      "bits_per_byte": 1.7579383800072375,
      "learning_rate": 0.00023314334759420388,
      "loss_nats": 1.2185100317001343,
      "step": 2920,
      "unclipped_grad_norm": 0.6799261569976807
    },
    {
      "bits_per_byte": 1.7601423376780443,
      "learning_rate": 0.0002326610298942717,
      "loss_nats": 1.2200376987457275,
      "step": 2930,
      "unclipped_grad_norm": 0.6305575370788574
    },
    {
      "bits_per_byte": 1.77748111259008,
      "learning_rate": 0.00023217755605402175,
      "loss_nats": 1.2320560216903687,
      "step": 2940,
      "unclipped_grad_norm": 0.6869999766349792
    },
    {
      "bits_per_byte": 1.7661192507434795,
      "learning_rate": 0.00023169293433468802,
      "loss_nats": 1.2241805791854858,
      "step": 2950,
      "unclipped_grad_norm": 0.6630687713623047
    },
    {
      "bits_per_byte": 1.7912226983766693,
      "learning_rate": 0.00023120717301711857,
      "loss_nats": 1.2415809631347656,
      "step": 2960,
      "unclipped_grad_norm": 0.6753889918327332
    },
    {
      "bits_per_byte": 1.7573228540997994,
      "learning_rate": 0.00023072028040163408,
      "loss_nats": 1.218083381652832,
      "step": 2970,
      "unclipped_grad_norm": 0.6314569115638733
    },
    {
      "bits_per_byte": 1.7556446473926801,
      "learning_rate": 0.00023023226480788593,
      "loss_nats": 1.2169201374053955,
      "step": 2980,
      "unclipped_grad_norm": 0.685643196105957
    },
    {
      "bits_per_byte": 1.778294762511312,
      "learning_rate": 0.00022974313457471416,
      "loss_nats": 1.2326200008392334,
      "step": 2990,
      "unclipped_grad_norm": 0.7002031803131104
    },
    {
      "bits_per_byte": 1.7854260231293384,
      "learning_rate": 0.0002292528980600049,
      "loss_nats": 1.2375630140304565,
      "step": 3000,
      "unclipped_grad_norm": 0.6241077184677124
    },
    {
      "bits_per_byte": 1.7589776711664014,
      "learning_rate": 0.00022876156364054758,
      "loss_nats": 1.2192304134368896,
      "step": 3010,
      "unclipped_grad_norm": 0.5821192860603333
    },
    {
      "bits_per_byte": 1.7213342805282006,
      "learning_rate": 0.0002282691397118918,
      "loss_nats": 1.1931380033493042,
      "step": 3020,
      "unclipped_grad_norm": 0.6891434192657471
    },
    {
      "bits_per_byte": 1.8083322204151049,
      "learning_rate": 0.00022777563468820376,
      "loss_nats": 1.2534403800964355,
      "step": 3030,
      "unclipped_grad_norm": 0.6679258346557617
    },
    {
      "bits_per_byte": 1.7077268412092843,
      "learning_rate": 0.00022728105700212282,
      "loss_nats": 1.1837060451507568,
      "step": 3040,
      "unclipped_grad_norm": 0.6403138637542725
    },
    {
      "bits_per_byte": 1.7462220619172086,
      "learning_rate": 0.00022678541510461698,
      "loss_nats": 1.2103888988494873,
      "step": 3050,
      "unclipped_grad_norm": 0.6389932632446289
    },
    {
      "bits_per_byte": 1.7619969985849488,
      "learning_rate": 0.00022628871746483876,
      "loss_nats": 1.2213232517242432,
      "step": 3060,
      "unclipped_grad_norm": 0.676324188709259
    },
    {
      "bits_per_byte": 1.6793940753408436,
      "learning_rate": 0.0002257909725699804,
      "loss_nats": 1.164067268371582,
      "step": 3070,
      "unclipped_grad_norm": 0.5887739062309265
    },
    {
      "bits_per_byte": 1.723725699288448,
      "learning_rate": 0.00022529218892512886,
      "loss_nats": 1.1947956085205078,
      "step": 3080,
      "unclipped_grad_norm": 0.6651143431663513
    },
    {
      "bits_per_byte": 1.792202311555984,
      "learning_rate": 0.00022479237505312034,
      "loss_nats": 1.2422599792480469,
      "step": 3090,
      "unclipped_grad_norm": 0.7115829586982727
    },
    {
      "bits_per_byte": 1.6949688242029795,
      "learning_rate": 0.00022429153949439496,
      "loss_nats": 1.1748628616333008,
      "step": 3100,
      "unclipped_grad_norm": 0.648972749710083
    },
    {
      "bits_per_byte": 1.7872602181007902,
      "learning_rate": 0.00022378969080685053,
      "loss_nats": 1.2388343811035156,
      "step": 3110,
      "unclipped_grad_norm": 0.6126834154129028
    },
    {
      "bits_per_byte": 1.7418012478767801,
      "learning_rate": 0.0002232868375656965,
      "loss_nats": 1.2073246240615845,
      "step": 3120,
      "unclipped_grad_norm": 0.6380596160888672
    },
    {
      "bits_per_byte": 1.7994572276999805,
      "learning_rate": 0.00022278298836330733,
      "loss_nats": 1.247288703918457,
      "step": 3130,
      "unclipped_grad_norm": 0.6736929416656494
    },
    {
      "bits_per_byte": 1.7652089465724625,
      "learning_rate": 0.00022227815180907572,
      "loss_nats": 1.2235496044158936,
      "step": 3140,
      "unclipped_grad_norm": 0.6578004956245422
    },
    {
      "bits_per_byte": 1.7230821402089194,
      "learning_rate": 0.00022177233652926546,
      "loss_nats": 1.1943495273590088,
      "step": 3150,
      "unclipped_grad_norm": 0.6959549784660339
    },
    {
      "bits_per_byte": 1.745404112429705,
      "learning_rate": 0.00022126555116686416,
      "loss_nats": 1.2098219394683838,
      "step": 3160,
      "unclipped_grad_norm": 0.6303315758705139
    },
    {
      "bits_per_byte": 1.6651441088383039,
      "learning_rate": 0.00022075780438143532,
      "loss_nats": 1.154189944267273,
      "step": 3170,
      "unclipped_grad_norm": 0.6514579057693481
    },
    {
      "bits_per_byte": 1.748495672561615,
      "learning_rate": 0.00022024910484897065,
      "loss_nats": 1.2119648456573486,
      "step": 3180,
      "unclipped_grad_norm": 0.6430810689926147
    },
    {
      "bits_per_byte": 1.7413002624148166,
      "learning_rate": 0.00021973946126174158,
      "loss_nats": 1.206977367401123,
      "step": 3190,
      "unclipped_grad_norm": 0.6930623650550842
    },
    {
      "bits_per_byte": 1.7004863716079601,
      "learning_rate": 0.00021922888232815086,
      "loss_nats": 1.178687334060669,
      "step": 3200,
      "unclipped_grad_norm": 0.6456758975982666
    },
    {
      "bits_per_byte": 1.6959322710131128,
      "learning_rate": 0.0002187173767725837,
      "loss_nats": 1.1755306720733643,
      "step": 3210,
      "unclipped_grad_norm": 0.6182776689529419
    },
    {
      "bits_per_byte": 1.7081593775762043,
      "learning_rate": 0.0002182049533352588,
      "loss_nats": 1.184005856513977,
      "step": 3220,
      "unclipped_grad_norm": 0.6259344816207886
    },
    {
      "bits_per_byte": 1.8040880325571125,
      "learning_rate": 0.00021769162077207884,
      "loss_nats": 1.2504985332489014,
      "step": 3230,
      "unclipped_grad_norm": 0.6837271451950073
    },
    {
      "bits_per_byte": 1.680291136847744,
      "learning_rate": 0.00021717738785448102,
      "loss_nats": 1.164689064025879,
      "step": 3240,
      "unclipped_grad_norm": 0.6078165173530579
    },
    {
      "bits_per_byte": 1.833327662943553,
      "learning_rate": 0.00021666226336928708,
      "loss_nats": 1.2707659006118774,
      "step": 3250,
      "unclipped_grad_norm": 0.6908280849456787
    },
    {
      "bits_per_byte": 1.6877314502893708,
      "learning_rate": 0.0002161462561185532,
      "loss_nats": 1.1698462963104248,
      "step": 3260,
      "unclipped_grad_norm": 0.6267281174659729
    },
    {
      "bits_per_byte": 1.7156755353668616,
      "learning_rate": 0.00021562937491941957,
      "loss_nats": 1.1892156600952148,
      "step": 3270,
      "unclipped_grad_norm": 0.6226681470870972
    },
    {
      "bits_per_byte": 1.77760270432424,
      "learning_rate": 0.00021511162860395975,
      "loss_nats": 1.232140302658081,
      "step": 3280,
      "unclipped_grad_norm": 0.62666916847229
    },
    {
      "bits_per_byte": 1.7647146684338828,
      "learning_rate": 0.0002145930260190298,
      "loss_nats": 1.2232069969177246,
      "step": 3290,
      "unclipped_grad_norm": 0.6992575526237488
    },
    {
      "bits_per_byte": 1.6858417049216903,
      "learning_rate": 0.00021407357602611698,
      "loss_nats": 1.1685364246368408,
      "step": 3300,
      "unclipped_grad_norm": 0.6361157894134521
    },
    {
      "bits_per_byte": 1.7274259060217614,
      "learning_rate": 0.0002135532875011885,
      "loss_nats": 1.1973603963851929,
      "step": 3310,
      "unclipped_grad_norm": 0.6828696727752686
    },
    {
      "bits_per_byte": 1.7324201102201664,
      "learning_rate": 0.0002130321693345397,
      "loss_nats": 1.200822114944458,
      "step": 3320,
      "unclipped_grad_norm": 0.6315762996673584
    },
    {
      "bits_per_byte": 1.7279007501207935,
      "learning_rate": 0.00021251023043064224,
      "loss_nats": 1.1976895332336426,
      "step": 3330,
      "unclipped_grad_norm": 0.6427719593048096
    },
    {
      "bits_per_byte": 1.6987109947031034,
      "learning_rate": 0.00021198747970799188,
      "loss_nats": 1.1774567365646362,
      "step": 3340,
      "unclipped_grad_norm": 0.6303005814552307
    },
    {
      "bits_per_byte": 1.7007044456092542,
      "learning_rate": 0.00021146392609895618,
      "loss_nats": 1.1788384914398193,
      "step": 3350,
      "unclipped_grad_norm": 0.6034587025642395
    },
    {
      "bits_per_byte": 1.7301495952634756,
      "learning_rate": 0.00021093957854962178,
      "loss_nats": 1.1992483139038086,
      "step": 3360,
      "unclipped_grad_norm": 0.6297745108604431
    },
    {
      "bits_per_byte": 1.6889530430584494,
      "learning_rate": 0.00021041444601964157,
      "loss_nats": 1.170693039894104,
      "step": 3370,
      "unclipped_grad_norm": 0.6365876197814941
    },
    {
      "bits_per_byte": 1.7185201284121325,
      "learning_rate": 0.00020988853748208164,
      "loss_nats": 1.1911873817443848,
      "step": 3380,
      "unclipped_grad_norm": 0.612015962600708
    },
    {
      "bits_per_byte": 1.7554757604295328,
      "learning_rate": 0.00020936186192326782,
      "loss_nats": 1.2168030738830566,
      "step": 3390,
      "unclipped_grad_norm": 0.7148704528808594
    },
    {
      "bits_per_byte": 1.805198180568432,
      "learning_rate": 0.00020883442834263225,
      "loss_nats": 1.2512680292129517,
      "step": 3400,
      "unclipped_grad_norm": 0.7259010672569275
    },
    {
      "bits_per_byte": 1.8248889902136676,
      "learning_rate": 0.0002083062457525596,
      "loss_nats": 1.2649166584014893,
      "step": 3410,
      "unclipped_grad_norm": 0.6750312447547913
    },
    {
      "bits_per_byte": 1.6908183668897072,
      "learning_rate": 0.00020777732317823306,
      "loss_nats": 1.1719859838485718,
      "step": 3420,
      "unclipped_grad_norm": 0.6068294644355774
    },
    {
      "bits_per_byte": 1.6181469257849768,
      "learning_rate": 0.00020724766965748005,
      "loss_nats": 1.1216139793395996,
      "step": 3430,
      "unclipped_grad_norm": 0.6401124596595764
    },
    {
      "bits_per_byte": 1.7073711810873,
      "learning_rate": 0.00020671729424061788,
      "loss_nats": 1.1834595203399658,
      "step": 3440,
      "unclipped_grad_norm": 0.6253624558448792
    },
    {
      "bits_per_byte": 1.7759955264519272,
      "learning_rate": 0.00020618620599029906,
      "loss_nats": 1.231026291847229,
      "step": 3450,
      "unclipped_grad_norm": 0.6825960874557495
    },
    {
      "bits_per_byte": 1.7035373438342662,
      "learning_rate": 0.00020565441398135649,
      "loss_nats": 1.1808021068572998,
      "step": 3460,
      "unclipped_grad_norm": 0.6307380795478821
    },
    {
      "bits_per_byte": 1.694909318205781,
      "learning_rate": 0.00020512192730064827,
      "loss_nats": 1.1748216152191162,
      "step": 3470,
      "unclipped_grad_norm": 0.6624554395675659
    },
    {
      "bits_per_byte": 1.640782594360936,
      "learning_rate": 0.00020458875504690273,
      "loss_nats": 1.1373038291931152,
      "step": 3480,
      "unclipped_grad_norm": 0.6500871777534485
    },
    {
      "bits_per_byte": 1.8527976468650806,
      "learning_rate": 0.00020405490633056245,
      "loss_nats": 1.2842614650726318,
      "step": 3490,
      "unclipped_grad_norm": 0.6960679888725281
    },
    {
      "bits_per_byte": 1.7002731131208901,
      "learning_rate": 0.00020352039027362917,
      "loss_nats": 1.178539514541626,
      "step": 3500,
      "unclipped_grad_norm": 0.7090469002723694
    },
    {
      "bits_per_byte": 1.7481523951904927,
      "learning_rate": 0.0002029852160095075,
      "loss_nats": 1.2117269039154053,
      "step": 3510,
      "unclipped_grad_norm": 0.6682069301605225
    },
    {
      "bits_per_byte": 1.7343439081527179,
      "learning_rate": 0.000202449392682849,
      "loss_nats": 1.202155590057373,
      "step": 3520,
      "unclipped_grad_norm": 0.6754779815673828
    },
    {
      "bits_per_byte": 1.703965236669613,
      "learning_rate": 0.0002019129294493959,
      "loss_nats": 1.1810986995697021,
      "step": 3530,
      "unclipped_grad_norm": 0.625368595123291
    },
    {
      "bits_per_byte": 1.7204380789345546,
      "learning_rate": 0.00020137583547582478,
      "loss_nats": 1.192516803741455,
      "step": 3540,
      "unclipped_grad_norm": 0.6671978235244751
    },
    {
      "bits_per_byte": 1.7171148581719324,
      "learning_rate": 0.00020083811993958973,
      "loss_nats": 1.1902133226394653,
      "step": 3550,
      "unclipped_grad_norm": 0.6819398999214172
    },
    {
      "bits_per_byte": 1.6516749435706812,
      "learning_rate": 0.00020029979202876563,
      "loss_nats": 1.1448538303375244,
      "step": 3560,
      "unclipped_grad_norm": 0.6344290375709534
    },
    {
      "bits_per_byte": 1.6805387918649863,
      "learning_rate": 0.00019976086094189118,
      "loss_nats": 1.164860725402832,
      "step": 3570,
      "unclipped_grad_norm": 0.6333562135696411
    },
    {
      "bits_per_byte": 1.6952945593437136,
      "learning_rate": 0.00019922133588781175,
      "loss_nats": 1.17508864402771,
      "step": 3580,
      "unclipped_grad_norm": 0.655963659286499
    },
    {
      "bits_per_byte": 1.7344106374212527,
      "learning_rate": 0.0001986812260855219,
      "loss_nats": 1.2022018432617188,
      "step": 3590,
      "unclipped_grad_norm": 0.7093035578727722
    },
    {
      "bits_per_byte": 1.7542852965202604,
      "learning_rate": 0.00019814054076400787,
      "loss_nats": 1.2159779071807861,
      "step": 3600,
      "unclipped_grad_norm": 0.668968915939331
    },
    {
      "bits_per_byte": 1.6751485116216442,
      "learning_rate": 0.00019759928916209004,
      "loss_nats": 1.1611244678497314,
      "step": 3610,
      "unclipped_grad_norm": 0.6502929925918579
    },
    {
      "bits_per_byte": 1.6413291552253781,
      "learning_rate": 0.00019705748052826493,
      "loss_nats": 1.1376826763153076,
      "step": 3620,
      "unclipped_grad_norm": 0.6599135398864746
    },
    {
      "bits_per_byte": 1.6914022479893867,
      "learning_rate": 0.00019651512412054723,
      "loss_nats": 1.1723906993865967,
      "step": 3630,
      "unclipped_grad_norm": 0.6377425789833069
    },
    {
      "bits_per_byte": 1.6712699588793862,
      "learning_rate": 0.00019597222920631144,
      "loss_nats": 1.1584360599517822,
      "step": 3640,
      "unclipped_grad_norm": 0.6370565891265869
    },
    {
      "bits_per_byte": 1.7179845744373456,
      "learning_rate": 0.0001954288050621338,
      "loss_nats": 1.1908161640167236,
      "step": 3650,
      "unclipped_grad_norm": 0.6391654014587402
    },
    {
      "bits_per_byte": 1.6386034021918534,
      "learning_rate": 0.00019488486097363363,
      "loss_nats": 1.1357933282852173,
      "step": 3660,
      "unclipped_grad_norm": 0.651786208152771
    },
    {
      "bits_per_byte": 1.7137386667528447,
      "learning_rate": 0.00019434040623531452,
      "loss_nats": 1.187873125076294,
      "step": 3670,
      "unclipped_grad_norm": 0.64645916223526
    },
    {
      "bits_per_byte": 1.651732729741371,
      "learning_rate": 0.0001937954501504058,
      "loss_nats": 1.1448938846588135,
      "step": 3680,
      "unclipped_grad_norm": 0.6200698018074036
    },
    {
      "bits_per_byte": 1.6382305438047828,
      "learning_rate": 0.00019325000203070333,
      "loss_nats": 1.1355348825454712,
      "step": 3690,
      "unclipped_grad_norm": 0.6166419386863708
    },
    {
      "bits_per_byte": 1.6489116983192684,
      "learning_rate": 0.0001927040711964106,
      "loss_nats": 1.142938494682312,
      "step": 3700,
      "unclipped_grad_norm": 0.6672006845474243
    },
    {
      "bits_per_byte": 1.7161945790071655,
      "learning_rate": 0.00019215766697597923,
      "loss_nats": 1.189575433731079,
      "step": 3710,
      "unclipped_grad_norm": 0.6522265672683716
    },
    {
      "bits_per_byte": 1.6845286173823526,
      "learning_rate": 0.00019161079870594978,
      "loss_nats": 1.1676262617111206,
      "step": 3720,
      "unclipped_grad_norm": 0.6373200416564941
    },
    {
      "bits_per_byte": 1.6158851819434799,
      "learning_rate": 0.0001910634757307922,
      "loss_nats": 1.1200462579727173,
      "step": 3730,
      "unclipped_grad_norm": 0.6535916328430176
    },
    {
      "bits_per_byte": 1.6679503497524322,
      "learning_rate": 0.0001905157074027459,
      "loss_nats": 1.156135082244873,
      "step": 3740,
      "unclipped_grad_norm": 0.6719768643379211
    },
    {
      "bits_per_byte": 1.6882193650698685,
      "learning_rate": 0.00018996750308166037,
      "loss_nats": 1.1701844930648804,
      "step": 3750,
      "unclipped_grad_norm": 0.6530092358589172
    },
    {
      "bits_per_byte": 1.7695160800806709,
      "learning_rate": 0.00018941887213483482,
      "loss_nats": 1.2265350818634033,
      "step": 3760,
      "unclipped_grad_norm": 0.6655569672584534
    },
    {
      "bits_per_byte": 1.7330591977507726,
      "learning_rate": 0.00018886982393685845,
      "loss_nats": 1.2012650966644287,
      "step": 3770,
      "unclipped_grad_norm": 0.6482370495796204
    },
    {
      "bits_per_byte": 1.555590988256044,
      "learning_rate": 0.00018832036786945008,
      "loss_nats": 1.0782535076141357,
      "step": 3780,
      "unclipped_grad_norm": 0.6347804665565491
    },
    {
      "bits_per_byte": 1.661505987841952,
      "learning_rate": 0.0001877705133212979,
      "loss_nats": 1.1516681909561157,
      "step": 3790,
      "unclipped_grad_norm": 0.6471394896507263
    },
    {
      "bits_per_byte": 1.671339095905033,
      "learning_rate": 0.00018722026968789906,
      "loss_nats": 1.1584839820861816,
      "step": 3800,
      "unclipped_grad_norm": 0.6512831449508667
    },
    {
      "bits_per_byte": 1.6910716973344282,
      "learning_rate": 0.00018666964637139907,
      "loss_nats": 1.17216157913208,
      "step": 3810,
      "unclipped_grad_norm": 0.66470867395401
    },
    {
      "bits_per_byte": 1.662058912064476,
      "learning_rate": 0.00018611865278043115,
      "loss_nats": 1.1520514488220215,
      "step": 3820,
      "unclipped_grad_norm": 0.6363425254821777
    },
    {
      "bits_per_byte": 1.7124487968713735,
      "learning_rate": 0.00018556729832995555,
      "loss_nats": 1.186979055404663,
      "step": 3830,
      "unclipped_grad_norm": 0.6397905945777893
    },
    {
      "bits_per_byte": 1.6528970522877124,
      "learning_rate": 0.00018501559244109864,
      "loss_nats": 1.1457009315490723,
      "step": 3840,
      "unclipped_grad_norm": 0.6409087777137756
    },
    {
      "bits_per_byte": 1.5958772362900533,
      "learning_rate": 0.00018446354454099178,
      "loss_nats": 1.106177806854248,
      "step": 3850,
      "unclipped_grad_norm": 0.6576514840126038
    },
    {
      "bits_per_byte": 1.6597844415068153,
      "learning_rate": 0.00018391116406261048,
      "loss_nats": 1.1504749059677124,
      "step": 3860,
      "unclipped_grad_norm": 0.6520916819572449
    },
    {
      "bits_per_byte": 1.7387071080051073,
      "learning_rate": 0.00018335846044461305,
      "loss_nats": 1.2051799297332764,
      "step": 3870,
      "unclipped_grad_norm": 0.6458169221878052
    },
    {
      "bits_per_byte": 1.631997548572213,
      "learning_rate": 0.0001828054431311793,
      "loss_nats": 1.1312144994735718,
      "step": 3880,
      "unclipped_grad_norm": 0.6073523163795471
    },
    {
      "bits_per_byte": 1.6031751480967658,
      "learning_rate": 0.00018225212157184934,
      "loss_nats": 1.111236333847046,
      "step": 3890,
      "unclipped_grad_norm": 0.632803201675415
    },
    {
      "bits_per_byte": 1.6778582702686387,
      "learning_rate": 0.00018169850522136192,
      "loss_nats": 1.1630027294158936,
      "step": 3900,
      "unclipped_grad_norm": 0.7183825969696045
    },
    {
      "bits_per_byte": 1.69528785202033,
      "learning_rate": 0.000181144603539493,
      "loss_nats": 1.1750839948654175,
      "step": 3910,
      "unclipped_grad_norm": 0.6837812662124634
    },
    {
      "bits_per_byte": 1.6365686754494955,
      "learning_rate": 0.00018059042599089412,
      "loss_nats": 1.134382963180542,
      "step": 3920,
      "unclipped_grad_norm": 0.6382866501808167
    },
    {
      "bits_per_byte": 1.6653448125918606,
      "learning_rate": 0.00018003598204493048,
      "loss_nats": 1.1543290615081787,
      "step": 3930,
      "unclipped_grad_norm": 0.6285881400108337
    },
    {
      "bits_per_byte": 1.6987961261152804,
      "learning_rate": 0.00017948128117551936,
      "loss_nats": 1.1775157451629639,
      "step": 3940,
      "unclipped_grad_norm": 0.666064977645874
    },
    {
      "bits_per_byte": 1.7229675997634448,
      "learning_rate": 0.00017892633286096816,
      "loss_nats": 1.194270133972168,
      "step": 3950,
      "unclipped_grad_norm": 0.6532164812088013
    },
    {
      "bits_per_byte": 1.7031080751377128,
      "learning_rate": 0.00017837114658381247,
      "loss_nats": 1.180504560470581,
      "step": 3960,
      "unclipped_grad_norm": 0.651078999042511
    },
    {
      "bits_per_byte": 1.631284336519085,
      "learning_rate": 0.00017781573183065386,
      "loss_nats": 1.1307201385498047,
      "step": 3970,
      "unclipped_grad_norm": 0.6460844278335571
    },
    {
      "bits_per_byte": 1.6744433827531067,
      "learning_rate": 0.0001772600980919981,
      "loss_nats": 1.1606357097625732,
      "step": 3980,
      "unclipped_grad_norm": 0.6715673208236694
    },
    {
      "bits_per_byte": 1.6984889651008395,
      "learning_rate": 0.0001767042548620927,
      "loss_nats": 1.1773028373718262,
      "step": 3990,
      "unclipped_grad_norm": 0.7048564553260803
    },
    {
      "bits_per_byte": 1.5840455178412955,
      "learning_rate": 0.00017614821163876486,
      "loss_nats": 1.0979766845703125,
      "step": 4000,
      "unclipped_grad_norm": 0.6564667224884033
    },
    {
      "bits_per_byte": 1.67799809216379,
      "learning_rate": 0.00017559197792325906,
      "loss_nats": 1.1630996465682983,
      "step": 4010,
      "unclipped_grad_norm": 0.6846939921379089
    },
    {
      "bits_per_byte": 1.645402220345762,
      "learning_rate": 0.0001750355632200748,
      "loss_nats": 1.1405059099197388,
      "step": 4020,
      "unclipped_grad_norm": 0.6446395516395569
    },
    {
      "bits_per_byte": 1.678588508604202,
      "learning_rate": 0.00017447897703680418,
      "loss_nats": 1.1635088920593262,
      "step": 4030,
      "unclipped_grad_norm": 0.744084894657135
    },
    {
      "bits_per_byte": 1.6238756678852169,
      "learning_rate": 0.0001739222288839694,
      "loss_nats": 1.1255848407745361,
      "step": 4040,
      "unclipped_grad_norm": 0.6811469793319702
    },
    {
      "bits_per_byte": 1.6502275375810198,
      "learning_rate": 0.00017336532827486026,
      "loss_nats": 1.143850564956665,
      "step": 4050,
      "unclipped_grad_norm": 0.6219719052314758
    },
    {
      "bits_per_byte": 1.655425885185999,
      "learning_rate": 0.00017280828472537163,
      "loss_nats": 1.147453784942627,
      "step": 4060,
      "unclipped_grad_norm": 0.6728122234344482
    },
    {
      "bits_per_byte": 1.6906052803852882,
      "learning_rate": 0.0001722511077538408,
      "loss_nats": 1.1718382835388184,
      "step": 4070,
      "unclipped_grad_norm": 0.6714834570884705
    },
    {
      "bits_per_byte": 1.7592499197027172,
      "learning_rate": 0.00017169380688088497,
      "loss_nats": 1.2194191217422485,
      "step": 4080,
      "unclipped_grad_norm": 0.7062271237373352
    },
    {
      "bits_per_byte": 1.656125338627058,
      "learning_rate": 0.0001711363916292384,
      "loss_nats": 1.14793860912323,
      "step": 4090,
      "unclipped_grad_norm": 0.7272074222564697
    },
    {
      "bits_per_byte": 1.6359631245358075,
      "learning_rate": 0.00017057887152358974,
      "loss_nats": 1.1339632272720337,
      "step": 4100,
      "unclipped_grad_norm": 0.6811342239379883
    },
    {
      "bits_per_byte": 1.6284930580955814,
      "learning_rate": 0.0001700212560904194,
      "loss_nats": 1.1287853717803955,
      "step": 4110,
      "unclipped_grad_norm": 0.6341036558151245
    },
    {
      "bits_per_byte": 1.6773249520683131,
      "learning_rate": 0.00016946355485783656,
      "loss_nats": 1.1626330614089966,
      "step": 4120,
      "unclipped_grad_norm": 0.6589056849479675
    },
    {
      "bits_per_byte": 1.6258371300183072,
      "learning_rate": 0.0001689057773554166,
      "loss_nats": 1.1269444227218628,
      "step": 4130,
      "unclipped_grad_norm": 0.6739165186882019
    },
    {
      "bits_per_byte": 1.688376729195408,
      "learning_rate": 0.00016834793311403806,
      "loss_nats": 1.1702935695648193,
      "step": 4140,
      "unclipped_grad_norm": 0.6829613447189331
    },
    {
      "bits_per_byte": 1.6211284170203344,
      "learning_rate": 0.00016779003166571981,
      "loss_nats": 1.123680591583252,
      "step": 4150,
      "unclipped_grad_norm": 0.6710932850837708
    },
    {
      "bits_per_byte": 1.6548134549662765,
      "learning_rate": 0.00016723208254345838,
      "loss_nats": 1.1470292806625366,
      "step": 4160,
      "unclipped_grad_norm": 0.7125453948974609
    },
    {
      "bits_per_byte": 1.6949535177470527,
      "learning_rate": 0.00016667409528106481,
      "loss_nats": 1.1748522520065308,
      "step": 4170,
      "unclipped_grad_norm": 0.6789602041244507
    },
    {
      "bits_per_byte": 1.7346840898361244,
      "learning_rate": 0.00016611607941300184,
      "loss_nats": 1.2023913860321045,
      "step": 4180,
      "unclipped_grad_norm": 0.7571533918380737
    },
    {
      "bits_per_byte": 1.6432809863300202,
      "learning_rate": 0.0001655580444742211,
      "loss_nats": 1.1390355825424194,
      "step": 4190,
      "unclipped_grad_norm": 0.6785401701927185
    },
    {
      "bits_per_byte": 1.6844624040617706,
      "learning_rate": 0.00016499999999999997,
      "loss_nats": 1.1675803661346436,
      "step": 4200,
      "unclipped_grad_norm": 0.7054091095924377
    },
    {
      "bits_per_byte": 1.672555185229284,
      "learning_rate": 0.0001644419555257789,
      "loss_nats": 1.1593269109725952,
      "step": 4210,
      "unclipped_grad_norm": 0.6699897050857544
    },
    {
      "bits_per_byte": 1.6983758005165717,
      "learning_rate": 0.0001638839205869981,
      "loss_nats": 1.1772243976593018,
      "step": 4220,
      "unclipped_grad_norm": 0.6806776523590088
    },
    {
      "bits_per_byte": 1.6692897506373519,
      "learning_rate": 0.0001633259047189352,
      "loss_nats": 1.1570634841918945,
      "step": 4230,
      "unclipped_grad_norm": 0.6995632648468018
    },
    {
      "bits_per_byte": 1.611002250520183,
      "learning_rate": 0.0001627679174565416,
      "loss_nats": 1.1166616678237915,
      "step": 4240,
      "unclipped_grad_norm": 0.6468433737754822
    },
    {
      "bits_per_byte": 1.646616933808806,
      "learning_rate": 0.00016220996833428016,
      "loss_nats": 1.141347885131836,
      "step": 4250,
      "unclipped_grad_norm": 0.6655133962631226
    },
    {
      "bits_per_byte": 1.61702336312689,
      "learning_rate": 0.00016165206688596191,
      "loss_nats": 1.1208351850509644,
      "step": 4260,
      "unclipped_grad_norm": 0.6660183072090149
    },
    {
      "bits_per_byte": 1.6213850151354217,
      "learning_rate": 0.00016109422264458335,
      "loss_nats": 1.1238584518432617,
      "step": 4270,
      "unclipped_grad_norm": 0.6792552471160889
    },
    {
      "bits_per_byte": 1.6175876382043708,
      "learning_rate": 0.00016053644514216336,
      "loss_nats": 1.1212263107299805,
      "step": 4280,
      "unclipped_grad_norm": 0.6596150398254395
    },
    {
      "bits_per_byte": 1.673857609844268,
      "learning_rate": 0.00015997874390958058,
      "loss_nats": 1.1602296829223633,
      "step": 4290,
      "unclipped_grad_norm": 0.6736516952514648
    },
    {
      "bits_per_byte": 1.5981075073064421,
      "learning_rate": 0.00015942112847641018,
      "loss_nats": 1.1077237129211426,
      "step": 4300,
      "unclipped_grad_norm": 0.65042644739151
    },
    {
      "bits_per_byte": 1.6804373221009774,
      "learning_rate": 0.00015886360837076156,
      "loss_nats": 1.164790391921997,
      "step": 4310,
      "unclipped_grad_norm": 0.7301046252250671
    },
    {
      "bits_per_byte": 1.6671354959526443,
      "learning_rate": 0.00015830619311911495,
      "loss_nats": 1.1555702686309814,
      "step": 4320,
      "unclipped_grad_norm": 0.6921650767326355
    },
    {
      "bits_per_byte": 1.6208601240849883,
      "learning_rate": 0.00015774889224615915,
      "loss_nats": 1.1234946250915527,
      "step": 4330,
      "unclipped_grad_norm": 0.6875792145729065
    },
    {
      "bits_per_byte": 1.651699193124453,
      "learning_rate": 0.00015719171527462834,
      "loss_nats": 1.144870638847351,
      "step": 4340,
      "unclipped_grad_norm": 0.6566780209541321
    },
    {
      "bits_per_byte": 1.638596522885819,
      "learning_rate": 0.0001566346717251397,
      "loss_nats": 1.1357885599136353,
      "step": 4350,
      "unclipped_grad_norm": 0.6942516565322876
    },
    {
      "bits_per_byte": 1.6271663839268258,
      "learning_rate": 0.00015607777111603056,
      "loss_nats": 1.1278657913208008,
      "step": 4360,
      "unclipped_grad_norm": 0.6855849623680115
    },
    {
      "bits_per_byte": 1.6394584999319433,
      "learning_rate": 0.00015552102296319576,
      "loss_nats": 1.1363860368728638,
      "step": 4370,
      "unclipped_grad_norm": 0.7280991673469543
    },
    {
      "bits_per_byte": 1.69825386481711,
      "learning_rate": 0.00015496443677992516,
      "loss_nats": 1.1771398782730103,
      "step": 4380,
      "unclipped_grad_norm": 0.6875903010368347
    },
    {
      "bits_per_byte": 1.6491371675745496,
      "learning_rate": 0.0001544080220767409,
      "loss_nats": 1.143094778060913,
      "step": 4390,
      "unclipped_grad_norm": 0.7626568078994751
    },
    {
      "bits_per_byte": 1.5979716410122606,
      "learning_rate": 0.00015385178836123511,
      "loss_nats": 1.1076295375823975,
      "step": 4400,
      "unclipped_grad_norm": 0.6716449856758118
    },
    {
      "bits_per_byte": 1.6724951632841327,
      "learning_rate": 0.00015329574513790726,
      "loss_nats": 1.159285306930542,
      "step": 4410,
      "unclipped_grad_norm": 0.7023071646690369
    },
    {
      "bits_per_byte": 1.6178187828871304,
      "learning_rate": 0.00015273990190800186,
      "loss_nats": 1.1213865280151367,
      "step": 4420,
      "unclipped_grad_norm": 0.6506969332695007
    },
    {
      "bits_per_byte": 1.6647528483075908,
      "learning_rate": 0.0001521842681693461,
      "loss_nats": 1.153918743133545,
      "step": 4430,
      "unclipped_grad_norm": 0.6848753094673157
    },
    {
      "bits_per_byte": 1.604941065955825,
      "learning_rate": 0.00015162885341618752,
      "loss_nats": 1.1124603748321533,
      "step": 4440,
      "unclipped_grad_norm": 0.6678289771080017
    },
    {
      "bits_per_byte": 1.6073562183218915,
      "learning_rate": 0.00015107366713903178,
      "loss_nats": 1.114134430885315,
      "step": 4450,
      "unclipped_grad_norm": 0.6393097639083862
    },
    {
      "bits_per_byte": 1.581368607880616,
      "learning_rate": 0.0001505187188244806,
      "loss_nats": 1.0961211919784546,
      "step": 4460,
      "unclipped_grad_norm": 0.6358886957168579
    },
    {
      "bits_per_byte": 1.6384745871863573,
      "learning_rate": 0.00014996401795506952,
      "loss_nats": 1.1357040405273438,
      "step": 4470,
      "unclipped_grad_norm": 0.7018935680389404
    },
    {
      "bits_per_byte": 1.6524677835911588,
      "learning_rate": 0.00014940957400910585,
      "loss_nats": 1.1454033851623535,
      "step": 4480,
      "unclipped_grad_norm": 0.7296347618103027
    },
    {
      "bits_per_byte": 1.6287001252072204,
      "learning_rate": 0.00014885539646050699,
      "loss_nats": 1.1289288997650146,
      "step": 4490,
      "unclipped_grad_norm": 0.6741621494293213
    },
    {
      "bits_per_byte": 1.644477125666771,
      "learning_rate": 0.00014830149477863805,
      "loss_nats": 1.1398646831512451,
      "step": 4500,
      "unclipped_grad_norm": 0.656035304069519
    },
    {
      "bits_per_byte": 1.6005383100937371,
      "learning_rate": 0.00014774787842815066,
      "loss_nats": 1.1094086170196533,
      "step": 4510,
      "unclipped_grad_norm": 0.6776733994483948
    },
    {
      "bits_per_byte": 1.6433757487706455,
      "learning_rate": 0.00014719455686882066,
      "loss_nats": 1.139101266860962,
      "step": 4520,
      "unclipped_grad_norm": 0.6692283153533936
    },
    {
      "bits_per_byte": 1.6564529655769518,
      "learning_rate": 0.00014664153955538695,
      "loss_nats": 1.1481657028198242,
      "step": 4530,
      "unclipped_grad_norm": 0.6900064945220947
    },
    {
      "bits_per_byte": 1.7115691056122104,
      "learning_rate": 0.00014608883593738946,
      "loss_nats": 1.1863692998886108,
      "step": 4540,
      "unclipped_grad_norm": 0.7291412353515625
    },
    {
      "bits_per_byte": 1.568905197155239,
      "learning_rate": 0.00014553645545900817,
      "loss_nats": 1.087482213973999,
      "step": 4550,
      "unclipped_grad_norm": 0.6766284108161926
    },
    {
      "bits_per_byte": 1.6280190739098037,
      "learning_rate": 0.0001449844075589013,
      "loss_nats": 1.1284568309783936,
      "step": 4560,
      "unclipped_grad_norm": 0.7198469042778015
    },
    {
      "bits_per_byte": 1.6109731854521872,
      "learning_rate": 0.0001444327016700444,
      "loss_nats": 1.1166415214538574,
      "step": 4570,
      "unclipped_grad_norm": 0.6274889707565308
    },
    {
      "bits_per_byte": 1.5632356170868957,
      "learning_rate": 0.0001438813472195688,
      "loss_nats": 1.083552360534668,
      "step": 4580,
      "unclipped_grad_norm": 0.7059792280197144
    },
    {
      "bits_per_byte": 1.6166451732776426,
      "learning_rate": 0.00014333035362860093,
      "loss_nats": 1.1205730438232422,
      "step": 4590,
      "unclipped_grad_norm": 0.7005748748779297
    },
    {
      "bits_per_byte": 1.6071720249028174,
      "learning_rate": 0.00014277973031210088,
      "loss_nats": 1.114006757736206,
      "step": 4600,
      "unclipped_grad_norm": 0.6841927170753479
    },
    {
      "bits_per_byte": 1.66910400937442,
      "learning_rate": 0.00014222948667870207,
      "loss_nats": 1.1569347381591797,
      "step": 4610,
      "unclipped_grad_norm": 0.7007519006729126
    },
    {
      "bits_per_byte": 1.6744798430750896,
      "learning_rate": 0.00014167963213054991,
      "loss_nats": 1.160660982131958,
      "step": 4620,
      "unclipped_grad_norm": 0.7218911051750183
    },
    {
      "bits_per_byte": 1.648005693714523,
      "learning_rate": 0.00014113017606314152,
      "loss_nats": 1.1423105001449585,
      "step": 4630,
      "unclipped_grad_norm": 0.7027076482772827
    },
    {
      "bits_per_byte": 1.692054922149411,
      "learning_rate": 0.00014058112786516518,
      "loss_nats": 1.172843098640442,
      "step": 4640,
      "unclipped_grad_norm": 0.6847496628761292
    },
    {
      "bits_per_byte": 1.6069941948418252,
      "learning_rate": 0.0001400324969183396,
      "loss_nats": 1.1138834953308105,
      "step": 4650,
      "unclipped_grad_norm": 0.695062518119812
    },
    {
      "bits_per_byte": 1.5888770264519845,
      "learning_rate": 0.00013948429259725406,
      "loss_nats": 1.1013256311416626,
      "step": 4660,
      "unclipped_grad_norm": 0.7018181681632996
    },
    {
      "bits_per_byte": 1.721095740591454,
      "learning_rate": 0.00013893652426920776,
      "loss_nats": 1.1929726600646973,
      "step": 4670,
      "unclipped_grad_norm": 0.7332665920257568
    },
    {
      "bits_per_byte": 1.7135371030860336,
      "learning_rate": 0.00013838920129405016,
      "loss_nats": 1.1877334117889404,
      "step": 4680,
      "unclipped_grad_norm": 0.7111047506332397
    },
    {
      "bits_per_byte": 1.6797499074454787,
      "learning_rate": 0.0001378423330240207,
      "loss_nats": 1.1643139123916626,
      "step": 4690,
      "unclipped_grad_norm": 0.7494509816169739
    },
    {
      "bits_per_byte": 1.6073170062774949,
      "learning_rate": 0.00013729592880358936,
      "loss_nats": 1.1141072511672974,
      "step": 4700,
      "unclipped_grad_norm": 0.6868770718574524
    },
    {
      "bits_per_byte": 1.6766973873753146,
      "learning_rate": 0.00013674999796929659,
      "loss_nats": 1.1621980667114258,
      "step": 4710,
      "unclipped_grad_norm": 0.7650524377822876
    },
    {
      "bits_per_byte": 1.6150646527162134,
      "learning_rate": 0.00013620454984959417,
      "loss_nats": 1.1194775104522705,
      "step": 4720,
      "unclipped_grad_norm": 0.7059918642044067
    },
    {
      "bits_per_byte": 1.5217877982289587,
      "learning_rate": 0.00013565959376468542,
      "loss_nats": 1.0548229217529297,
      "step": 4730,
      "unclipped_grad_norm": 0.6677312850952148
    },
    {
      "bits_per_byte": 1.5772625220912664,
      "learning_rate": 0.00013511513902636634,
      "loss_nats": 1.0932750701904297,
      "step": 4740,
      "unclipped_grad_norm": 0.6796184182167053
    },
    {
      "bits_per_byte": 1.6299009080755444,
      "learning_rate": 0.0001345711949378662,
      "loss_nats": 1.1297612190246582,
      "step": 4750,
      "unclipped_grad_norm": 0.7213169932365417
    },
    {
      "bits_per_byte": 1.6096504668844014,
      "learning_rate": 0.00013402777079368853,
      "loss_nats": 1.1157246828079224,
      "step": 4760,
      "unclipped_grad_norm": 0.7036519050598145
    },
    {
      "bits_per_byte": 1.5962356481344513,
      "learning_rate": 0.0001334848758794528,
      "loss_nats": 1.1064262390136719,
      "step": 4770,
      "unclipped_grad_norm": 0.7376033663749695
    },
    {
      "bits_per_byte": 1.626534347684905,
      "learning_rate": 0.00013294251947173504,
      "loss_nats": 1.1274276971817017,
      "step": 4780,
      "unclipped_grad_norm": 0.7321033477783203
    },
    {
      "bits_per_byte": 1.5732460392630163,
      "learning_rate": 0.00013240071083790996,
      "loss_nats": 1.0904910564422607,
      "step": 4790,
      "unclipped_grad_norm": 0.6640080809593201
    },
    {
      "bits_per_byte": 1.5751923669228307,
      "learning_rate": 0.0001318594592359921,
      "loss_nats": 1.091840147972107,
      "step": 4800,
      "unclipped_grad_norm": 0.7126999497413635
    },
    {
      "bits_per_byte": 1.637988564215019,
      "learning_rate": 0.0001313187739144781,
      "loss_nats": 1.1353671550750732,
      "step": 4810,
      "unclipped_grad_norm": 0.7521406412124634
    },
    {
      "bits_per_byte": 1.57020504203046,
      "learning_rate": 0.0001307786641121882,
      "loss_nats": 1.0883831977844238,
      "step": 4820,
      "unclipped_grad_norm": 0.7331681847572327
    },
    {
      "bits_per_byte": 1.6098945102659756,
      "learning_rate": 0.00013023913905810876,
      "loss_nats": 1.115893840789795,
      "step": 4830,
      "unclipped_grad_norm": 0.7234310507774353
    },
    {
      "bits_per_byte": 1.6131886659606018,
      "learning_rate": 0.00012970020797123432,
      "loss_nats": 1.1181771755218506,
      "step": 4840,
      "unclipped_grad_norm": 0.7273361682891846
    },
    {
      "bits_per_byte": 1.6277132887565695,
      "learning_rate": 0.00012916188006041027,
      "loss_nats": 1.1282448768615723,
      "step": 4850,
      "unclipped_grad_norm": 0.6802273988723755
    },
    {
      "bits_per_byte": 1.5401256203598566,
      "learning_rate": 0.00012862416452417516,
      "loss_nats": 1.0675337314605713,
      "step": 4860,
      "unclipped_grad_norm": 0.6796969771385193
    },
    {
      "bits_per_byte": 1.619305056955887,
      "learning_rate": 0.00012808707055060405,
      "loss_nats": 1.1224167346954346,
      "step": 4870,
      "unclipped_grad_norm": 0.7333647608757019
    },
    {
      "bits_per_byte": 1.4835698135542255,
      "learning_rate": 0.000127550607317151,
      "loss_nats": 1.028332233428955,
      "step": 4880,
      "unclipped_grad_norm": 0.7043541073799133
    },
    {
      "bits_per_byte": 1.5411697270332447,
      "learning_rate": 0.00012701478399049248,
      "loss_nats": 1.068257451057434,
      "step": 4890,
      "unclipped_grad_norm": 0.741724967956543
    },
    {
      "bits_per_byte": 1.588994318619873,
      "learning_rate": 0.0001264796097263708,
      "loss_nats": 1.1014069318771362,
      "step": 4900,
      "unclipped_grad_norm": 0.6961487531661987
    },
    {
      "bits_per_byte": 1.693890149016768,
      "learning_rate": 0.0001259450936694375,
      "loss_nats": 1.1741151809692383,
      "step": 4910,
      "unclipped_grad_norm": 0.7309879064559937
    },
    {
      "bits_per_byte": 1.6231887691776707,
      "learning_rate": 0.00012541124495309727,
      "loss_nats": 1.1251087188720703,
      "step": 4920,
      "unclipped_grad_norm": 0.7229281067848206
    },
    {
      "bits_per_byte": 1.5846615596966862,
      "learning_rate": 0.00012487807269935167,
      "loss_nats": 1.0984036922454834,
      "step": 4930,
      "unclipped_grad_norm": 0.6784905791282654
    },
    {
      "bits_per_byte": 1.5777433855830787,
      "learning_rate": 0.00012434558601864349,
      "loss_nats": 1.0936083793640137,
      "step": 4940,
      "unclipped_grad_norm": 0.6933380365371704
    },
    {
      "bits_per_byte": 1.5189858568811019,
      "learning_rate": 0.0001238137940097009,
      "loss_nats": 1.0528807640075684,
      "step": 4950,
      "unclipped_grad_norm": 0.6876996159553528
    },
    {
      "bits_per_byte": 1.5293708572708018,
      "learning_rate": 0.0001232827057593821,
      "loss_nats": 1.0600790977478027,
      "step": 4960,
      "unclipped_grad_norm": 0.7548285126686096
    },
    {
      "bits_per_byte": 1.613208615948102,
      "learning_rate": 0.00012275233034251992,
      "loss_nats": 1.1181910037994385,
      "step": 4970,
      "unclipped_grad_norm": 0.7192531824111938
    },
    {
      "bits_per_byte": 1.6464662770066503,
      "learning_rate": 0.0001222226768217669,
      "loss_nats": 1.1412434577941895,
      "step": 4980,
      "unclipped_grad_norm": 0.7251392602920532
    },
    {
      "bits_per_byte": 1.5932895853251714,
      "learning_rate": 0.00012169375424744034,
      "loss_nats": 1.104384183883667,
      "step": 4990,
      "unclipped_grad_norm": 0.770077109336853
    },
    {
      "bits_per_byte": 1.5394951319617936,
      "learning_rate": 0.00012116557165736772,
      "loss_nats": 1.0670967102050781,
      "step": 5000,
      "unclipped_grad_norm": 0.689707338809967
    },
    {
      "bits_per_byte": 1.5463551759394092,
      "learning_rate": 0.00012063813807673214,
      "loss_nats": 1.0718517303466797,
      "step": 5010,
      "unclipped_grad_norm": 0.6723923087120056
    },
    {
      "bits_per_byte": 1.5950135394174203,
      "learning_rate": 0.00012011146251791832,
      "loss_nats": 1.105579137802124,
      "step": 5020,
      "unclipped_grad_norm": 0.6902953386306763
    },
    {
      "bits_per_byte": 1.575558174021216,
      "learning_rate": 0.0001195855539803584,
      "loss_nats": 1.0920937061309814,
      "step": 5030,
      "unclipped_grad_norm": 0.7085601687431335
    },
    {
      "bits_per_byte": 1.5476398863413545,
      "learning_rate": 0.00011906042145037817,
      "loss_nats": 1.072742223739624,
      "step": 5040,
      "unclipped_grad_norm": 0.7076513767242432
    },
    {
      "bits_per_byte": 1.517178147237883,
      "learning_rate": 0.0001185360739010438,
      "loss_nats": 1.0516277551651,
      "step": 5050,
      "unclipped_grad_norm": 0.6860785484313965
    },
    {
      "bits_per_byte": 1.5405944450661087,
      "learning_rate": 0.00011801252029200808,
      "loss_nats": 1.0678586959838867,
      "step": 5060,
      "unclipped_grad_norm": 0.6969667673110962
    },
    {
      "bits_per_byte": 1.603204729112714,
      "learning_rate": 0.00011748976956935777,
      "loss_nats": 1.1112568378448486,
      "step": 5070,
      "unclipped_grad_norm": 0.6925143003463745
    },
    {
      "bits_per_byte": 1.568753508457178,
      "learning_rate": 0.00011696783066546024,
      "loss_nats": 1.0873770713806152,
      "step": 5080,
      "unclipped_grad_norm": 0.7227545380592346
    },
    {
      "bits_per_byte": 1.5571398640097143,
      "learning_rate": 0.00011644671249881147,
      "loss_nats": 1.07932710647583,
      "step": 5090,
      "unclipped_grad_norm": 0.7346438765525818
    },
    {
      "bits_per_byte": 1.5756090808858714,
      "learning_rate": 0.00011592642397388293,
      "loss_nats": 1.0921289920806885,
      "step": 5100,
      "unclipped_grad_norm": 0.687656581401825
    },
    {
      "bits_per_byte": 1.4874313400140482,
      "learning_rate": 0.00011540697398097015,
      "loss_nats": 1.0310088396072388,
      "step": 5110,
      "unclipped_grad_norm": 0.7022747993469238
    },
    {
      "bits_per_byte": 1.5848280389027214,
      "learning_rate": 0.00011488837139604016,
      "loss_nats": 1.0985190868377686,
      "step": 5120,
      "unclipped_grad_norm": 0.6745842695236206
    },
    {
      "bits_per_byte": 1.5742394110543998,
      "learning_rate": 0.00011437062508058039,
      "loss_nats": 1.091179609298706,
      "step": 5130,
      "unclipped_grad_norm": 0.6994429230690002
    },
    {
      "bits_per_byte": 1.5480249554966363,
      "learning_rate": 0.00011385374388144673,
      "loss_nats": 1.0730091333389282,
      "step": 5140,
      "unclipped_grad_norm": 0.7030988335609436
    },
    {
      "bits_per_byte": 1.639304919424723,
      "learning_rate": 0.00011333773663071286,
      "loss_nats": 1.136279582977295,
      "step": 5150,
      "unclipped_grad_norm": 0.7419465184211731
    },
    {
      "bits_per_byte": 1.5116879450743894,
      "learning_rate": 0.00011282261214551897,
      "loss_nats": 1.0478222370147705,
      "step": 5160,
      "unclipped_grad_norm": 0.7094154953956604
    },
    {
      "bits_per_byte": 1.6830882626813766,
      "learning_rate": 0.00011230837922792112,
      "loss_nats": 1.1666278839111328,
      "step": 5170,
      "unclipped_grad_norm": 0.7866556644439697
    },
    {
      "bits_per_byte": 1.5735232752962072,
      "learning_rate": 0.0001117950466647412,
      "loss_nats": 1.0906832218170166,
      "step": 5180,
      "unclipped_grad_norm": 0.7095000743865967
    },
    {
      "bits_per_byte": 1.51639820591622,
      "learning_rate": 0.00011128262322741627,
      "loss_nats": 1.0510871410369873,
      "step": 5190,
      "unclipped_grad_norm": 0.701707124710083
    },
    {
      "bits_per_byte": 1.549430569702138,
      "learning_rate": 0.00011077111767184913,
      "loss_nats": 1.0739834308624268,
      "step": 5200,
      "unclipped_grad_norm": 0.7002435922622681
    },
    {
      "bits_per_byte": 1.5581800150821326,
      "learning_rate": 0.00011026053873825837,
      "loss_nats": 1.0800480842590332,
      "step": 5210,
      "unclipped_grad_norm": 0.6994377374649048
    },
    {
      "bits_per_byte": 1.6434304392536199,
      "learning_rate": 0.0001097508951510293,
      "loss_nats": 1.139139175415039,
      "step": 5220,
      "unclipped_grad_norm": 0.764930009841919
    },
    {
      "bits_per_byte": 1.5617875231666307,
      "learning_rate": 0.0001092421956185646,
      "loss_nats": 1.0825486183166504,
      "step": 5230,
      "unclipped_grad_norm": 0.7441672086715698
    },
    {
      "bits_per_byte": 1.5466518460121477,
      "learning_rate": 0.00010873444883313581,
      "loss_nats": 1.0720573663711548,
      "step": 5240,
      "unclipped_grad_norm": 0.688214123249054
    },
    {
      "bits_per_byte": 1.5248055778036484,
      "learning_rate": 0.00010822766347073447,
      "loss_nats": 1.0569146871566772,
      "step": 5250,
      "unclipped_grad_norm": 0.6841322779655457
    },
    {
      "bits_per_byte": 1.5447307998020101,
      "learning_rate": 0.00010772184819092424,
      "loss_nats": 1.0707257986068726,
      "step": 5260,
      "unclipped_grad_norm": 0.694692850112915
    },
    {
      "bits_per_byte": 1.539800573149726,
      "learning_rate": 0.0001072170116366926,
      "loss_nats": 1.0673084259033203,
      "step": 5270,
      "unclipped_grad_norm": 0.6789148449897766
    },
    {
      "bits_per_byte": 1.5548948024853513,
      "learning_rate": 0.00010671316243430345,
      "loss_nats": 1.0777709484100342,
      "step": 5280,
      "unclipped_grad_norm": 0.7349528074264526
    },
    {
      "bits_per_byte": 1.5569278094012005,
      "learning_rate": 0.00010621030919314944,
      "loss_nats": 1.079180121421814,
      "step": 5290,
      "unclipped_grad_norm": 0.7407549023628235
    },
    {
      "bits_per_byte": 1.5483455311578447,
      "learning_rate": 0.000105708460505605,
      "loss_nats": 1.0732313394546509,
      "step": 5300,
      "unclipped_grad_norm": 0.7291215658187866
    },
    {
      "bits_per_byte": 1.5444855525418797,
      "learning_rate": 0.00010520762494687964,
      "loss_nats": 1.0705558061599731,
      "step": 5310,
      "unclipped_grad_norm": 0.7253612279891968
    },
    {
      "bits_per_byte": 1.5505493168460005,
      "learning_rate": 0.00010470781107487111,
      "loss_nats": 1.0747588872909546,
      "step": 5320,
      "unclipped_grad_norm": 0.7087648510932922
    },
    {
      "bits_per_byte": 1.5517671259967603,
      "learning_rate": 0.00010420902743001955,
      "loss_nats": 1.0756030082702637,
      "step": 5330,
      "unclipped_grad_norm": 0.7454737424850464
    },
    {
      "bits_per_byte": 1.5001372462771416,
      "learning_rate": 0.00010371128253516117,
      "loss_nats": 1.039815902709961,
      "step": 5340,
      "unclipped_grad_norm": 0.7009119391441345
    },
    {
      "bits_per_byte": 1.5857225206698589,
      "learning_rate": 0.000103214584895383,
      "loss_nats": 1.0991390943527222,
      "step": 5350,
      "unclipped_grad_norm": 0.7474352121353149
    },
    {
      "bits_per_byte": 1.4946740453898335,
      "learning_rate": 0.00010271894299787715,
      "loss_nats": 1.0360291004180908,
      "step": 5360,
      "unclipped_grad_norm": 0.7168655395507812
    },
    {
      "bits_per_byte": 1.4766953230339372,
      "learning_rate": 0.00010222436531179622,
      "loss_nats": 1.0235671997070312,
      "step": 5370,
      "unclipped_grad_norm": 0.7242471575737
    },
    {
      "bits_per_byte": 1.5813711876203789,
      "learning_rate": 0.0001017308602881082,
      "loss_nats": 1.0961229801177979,
      "step": 5380,
      "unclipped_grad_norm": 0.7494375109672546
    },
    {
      "bits_per_byte": 1.5460944502407012,
      "learning_rate": 0.00010123843635945242,
      "loss_nats": 1.0716710090637207,
      "step": 5390,
      "unclipped_grad_norm": 0.7461474537849426
    },
    {
      "bits_per_byte": 1.5453937929210861,
      "learning_rate": 0.00010074710193999505,
      "loss_nats": 1.0711853504180908,
      "step": 5400,
      "unclipped_grad_norm": 0.7291334271430969
    },
    {
      "bits_per_byte": 1.5672311180317404,
      "learning_rate": 0.0001002568654252858,
      "loss_nats": 1.0863218307495117,
      "step": 5410,
      "unclipped_grad_norm": 0.720815896987915
    },
    {
      "bits_per_byte": 1.541436472124733,
      "learning_rate": 9.976773519211404e-05,
      "loss_nats": 1.0684423446655273,
      "step": 5420,
      "unclipped_grad_norm": 0.743694543838501
    },
    {
      "bits_per_byte": 1.5782952779096975,
      "learning_rate": 9.92797195983659e-05,
      "loss_nats": 1.0939909219741821,
      "step": 5430,
      "unclipped_grad_norm": 0.7555011510848999
    },
    {
      "bits_per_byte": 1.508865365808429,
      "learning_rate": 9.879282698288142e-05,
      "loss_nats": 1.045865774154663,
      "step": 5440,
      "unclipped_grad_norm": 0.7259441018104553
    },
    {
      "bits_per_byte": 1.5083157092562716,
      "learning_rate": 9.830706566531197e-05,
      "loss_nats": 1.0454847812652588,
      "step": 5450,
      "unclipped_grad_norm": 0.7505731582641602
    },
    {
      "bits_per_byte": 1.5045496331676778,
      "learning_rate": 9.782244394597826e-05,
      "loss_nats": 1.0428743362426758,
      "step": 5460,
      "unclipped_grad_norm": 0.7536665201187134
    },
    {
      "bits_per_byte": 1.5449808625763646,
      "learning_rate": 9.733897010572829e-05,
      "loss_nats": 1.0708991289138794,
      "step": 5470,
      "unclipped_grad_norm": 0.7403784394264221
    },
    {
      "bits_per_byte": 1.550094938682421,
      "learning_rate": 9.685665240579612e-05,
      "loss_nats": 1.0744439363479614,
      "step": 5480,
      "unclipped_grad_norm": 0.725517213344574
    },
    {
      "bits_per_byte": 1.5697909078071826,
      "learning_rate": 9.637549908766041e-05,
      "loss_nats": 1.0880961418151855,
      "step": 5490,
      "unclipped_grad_norm": 0.776546061038971
    },
    {
      "bits_per_byte": 1.546138821764624,
      "learning_rate": 9.589551837290389e-05,
      "loss_nats": 1.0717017650604248,
      "step": 5500,
      "unclipped_grad_norm": 0.7528209686279297
    },
    {
      "bits_per_byte": 1.6319261757721049,
      "learning_rate": 9.541671846307257e-05,
      "loss_nats": 1.1311650276184082,
      "step": 5510,
      "unclipped_grad_norm": 0.8170714974403381
    },
    {
      "bits_per_byte": 1.5554193495704827,
      "learning_rate": 9.493910753953597e-05,
      "loss_nats": 1.078134536743164,
      "step": 5520,
      "unclipped_grad_norm": 0.7434027194976807
    },
    {
      "bits_per_byte": 1.5251368163892103,
      "learning_rate": 9.446269376334689e-05,
      "loss_nats": 1.057144284248352,
      "step": 5530,
      "unclipped_grad_norm": 0.7136013507843018
    },
    {
      "bits_per_byte": 1.586380354309409,
      "learning_rate": 9.39874852751024e-05,
      "loss_nats": 1.099595069885254,
      "step": 5540,
      "unclipped_grad_norm": 0.7905043959617615
    },
    {
      "bits_per_byte": 1.5182255215816374,
      "learning_rate": 9.351349019480433e-05,
      "loss_nats": 1.0523537397384644,
      "step": 5550,
      "unclipped_grad_norm": 0.7156078815460205
    },
    {
      "bits_per_byte": 1.5143791295950908,
      "learning_rate": 9.304071662172086e-05,
      "loss_nats": 1.0496876239776611,
      "step": 5560,
      "unclipped_grad_norm": 0.7718254923820496
    },
    {
      "bits_per_byte": 1.4761380992451418,
      "learning_rate": 9.256917263424787e-05,
      "loss_nats": 1.0231809616088867,
      "step": 5570,
      "unclipped_grad_norm": 0.7351405620574951
    },
    {
      "bits_per_byte": 1.5698151573609542,
      "learning_rate": 9.209886628977114e-05,
      "loss_nats": 1.0881129503250122,
      "step": 5580,
      "unclipped_grad_norm": 0.7265479564666748
    },
    {
      "bits_per_byte": 1.5069491351125157,
      "learning_rate": 9.162980562452843e-05,
      "loss_nats": 1.0445375442504883,
      "step": 5590,
      "unclipped_grad_norm": 0.7370763421058655
    },
    {
      "bits_per_byte": 1.5604274843636077,
      "learning_rate": 9.116199865347238e-05,
      "loss_nats": 1.0816059112548828,
      "step": 5600,
      "unclipped_grad_norm": 0.7772327661514282
    },
    {
      "bits_per_byte": 1.5487797873512732,
      "learning_rate": 9.069545337013333e-05,
      "loss_nats": 1.0735323429107666,
      "step": 5610,
      "unclipped_grad_norm": 0.6945621371269226
    },
    {
      "bits_per_byte": 1.5643851491252627,
      "learning_rate": 9.0230177746483e-05,
      "loss_nats": 1.0843491554260254,
      "step": 5620,
      "unclipped_grad_norm": 0.772422194480896
    },
    {
      "bits_per_byte": 1.5390314667350675,
      "learning_rate": 8.976617973279797e-05,
      "loss_nats": 1.0667753219604492,
      "step": 5630,
      "unclipped_grad_norm": 0.7249912023544312
    },
    {
      "bits_per_byte": 1.5236104703628028,
      "learning_rate": 8.930346725752419e-05,
      "loss_nats": 1.0560863018035889,
      "step": 5640,
      "unclipped_grad_norm": 0.7168114185333252
    },
    {
      "bits_per_byte": 1.5290047062071148,
      "learning_rate": 8.88420482271411e-05,
      "loss_nats": 1.0598253011703491,
      "step": 5650,
      "unclipped_grad_norm": 0.7599905729293823
    },
    {
      "bits_per_byte": 1.6052637054088437,
      "learning_rate": 8.838193052602691e-05,
      "loss_nats": 1.1126840114593506,
      "step": 5660,
      "unclipped_grad_norm": 0.8106476068496704
    },
    {
      "bits_per_byte": 1.5110031101586539,
      "learning_rate": 8.792312201632352e-05,
      "loss_nats": 1.0473475456237793,
      "step": 5670,
      "unclipped_grad_norm": 0.7452697157859802
    },
    {
      "bits_per_byte": 1.532641279359609,
      "learning_rate": 8.746563053780261e-05,
      "loss_nats": 1.0623459815979004,
      "step": 5680,
      "unclipped_grad_norm": 0.7387374043464661
    },
    {
      "bits_per_byte": 1.6103865526300942,
      "learning_rate": 8.700946390773114e-05,
      "loss_nats": 1.1162348985671997,
      "step": 5690,
      "unclipped_grad_norm": 0.7787995338439941
    },
    {
      "bits_per_byte": 1.5250291552497701,
      "learning_rate": 8.655462992073833e-05,
      "loss_nats": 1.0570696592330933,
      "step": 5700,
      "unclipped_grad_norm": 0.7521193027496338
    },
    {
      "bits_per_byte": 1.5451155249919901,
      "learning_rate": 8.610113634868199e-05,
      "loss_nats": 1.0709924697875977,
      "step": 5710,
      "unclipped_grad_norm": 0.7714623808860779
    },
    {
      "bits_per_byte": 1.5521905472831843,
      "learning_rate": 8.564899094051613e-05,
      "loss_nats": 1.0758965015411377,
      "step": 5720,
      "unclipped_grad_norm": 0.7254540324211121
    },
    {
      "bits_per_byte": 1.554917504195265,
      "learning_rate": 8.519820142215816e-05,
      "loss_nats": 1.0777866840362549,
      "step": 5730,
      "unclipped_grad_norm": 0.7474860548973083
    },
    {
      "bits_per_byte": 1.5468128217733552,
      "learning_rate": 8.474877549635728e-05,
      "loss_nats": 1.0721689462661743,
      "step": 5740,
      "unclipped_grad_norm": 0.7887082099914551
    },
    {
      "bits_per_byte": 1.541234736475271,
      "learning_rate": 8.430072084256247e-05,
      "loss_nats": 1.0683025121688843,
      "step": 5750,
      "unclipped_grad_norm": 0.7614914774894714
    },
    {
      "bits_per_byte": 1.5622339901282707,
      "learning_rate": 8.385404511679161e-05,
      "loss_nats": 1.0828580856323242,
      "step": 5760,
      "unclipped_grad_norm": 0.7596094608306885
    },
    {
      "bits_per_byte": 1.5113494832174914,
      "learning_rate": 8.340875595150034e-05,
      "loss_nats": 1.0475876331329346,
      "step": 5770,
      "unclipped_grad_norm": 0.7298161387443542
    },
    {
      "bits_per_byte": 1.4968841944360716,
      "learning_rate": 8.296486095545203e-05,
      "loss_nats": 1.037561058998108,
      "step": 5780,
      "unclipped_grad_norm": 0.7402821779251099
    },
    {
      "bits_per_byte": 1.524952966935438,
      "learning_rate": 8.252236771358729e-05,
      "loss_nats": 1.0570168495178223,
      "step": 5790,
      "unclipped_grad_norm": 0.7289884686470032
    },
    {
      "bits_per_byte": 1.4886130328081264,
      "learning_rate": 8.208128378689484e-05,
      "loss_nats": 1.0318279266357422,
      "step": 5800,
      "unclipped_grad_norm": 0.7239735722541809
    },
    {
      "bits_per_byte": 1.5780909625204724,
      "learning_rate": 8.16416167122819e-05,
      "loss_nats": 1.0938493013381958,
      "step": 5810,
      "unclipped_grad_norm": 0.7893304824829102
    },
    {
      "bits_per_byte": 1.5071924905634866,
      "learning_rate": 8.120337400244578e-05,
      "loss_nats": 1.0447062253952026,
      "step": 5820,
      "unclipped_grad_norm": 0.7217836976051331
    },
    {
      "bits_per_byte": 1.516913465938205,
      "learning_rate": 8.076656314574521e-05,
      "loss_nats": 1.0514442920684814,
      "step": 5830,
      "unclipped_grad_norm": 0.7600290775299072
    },
    {
      "bits_per_byte": 1.5193552756151552,
      "learning_rate": 8.03311916060725e-05,
      "loss_nats": 1.0531368255615234,
      "step": 5840,
      "unclipped_grad_norm": 0.7521220445632935
    },
    {
      "bits_per_byte": 1.4707125625583728,
      "learning_rate": 7.989726682272606e-05,
      "loss_nats": 1.0194202661514282,
      "step": 5850,
      "unclipped_grad_norm": 0.7758342027664185
    },
    {
      "bits_per_byte": 1.5340661556220072,
      "learning_rate": 7.946479621028318e-05,
      "loss_nats": 1.0633336305618286,
      "step": 5860,
      "unclipped_grad_norm": 0.7845423817634583
    },
    {
      "bits_per_byte": 1.5391652692374387,
      "learning_rate": 7.903378715847343e-05,
      "loss_nats": 1.0668680667877197,
      "step": 5870,
      "unclipped_grad_norm": 0.7568844556808472
    },
    {
      "bits_per_byte": 1.550266749350633,
      "learning_rate": 7.860424703205224e-05,
      "loss_nats": 1.0745630264282227,
      "step": 5880,
      "unclipped_grad_norm": 0.7594192028045654
    },
    {
      "bits_per_byte": 1.5444004211297027,
      "learning_rate": 7.817618317067534e-05,
      "loss_nats": 1.0704967975616455,
      "step": 5890,
      "unclipped_grad_norm": 0.722252607345581
    },
    {
      "bits_per_byte": 1.5452520792167752,
      "learning_rate": 7.774960288877295e-05,
      "loss_nats": 1.071087121963501,
      "step": 5900,
      "unclipped_grad_norm": 0.7907101511955261
    },
    {
      "bits_per_byte": 1.554658154357764,
      "learning_rate": 7.732451347542522e-05,
      "loss_nats": 1.0776069164276123,
      "step": 5910,
      "unclipped_grad_norm": 0.7964528799057007
    },
    {
      "bits_per_byte": 1.4817756905404247,
      "learning_rate": 7.690092219423722e-05,
      "loss_nats": 1.0270886421203613,
      "step": 5920,
      "unclipped_grad_norm": 0.7341460585594177
    },
    {
      "bits_per_byte": 1.5817024262059407,
      "learning_rate": 7.647883628321536e-05,
      "loss_nats": 1.0963525772094727,
      "step": 5930,
      "unclipped_grad_norm": 0.8211336731910706
    },
    {
      "bits_per_byte": 1.4638614616786019,
      "learning_rate": 7.605826295464316e-05,
      "loss_nats": 1.0146714448928833,
      "step": 5940,
      "unclipped_grad_norm": 0.7324587106704712
    },
    {
      "bits_per_byte": 1.5159049596735457,
      "learning_rate": 7.563920939495851e-05,
      "loss_nats": 1.0507452487945557,
      "step": 5950,
      "unclipped_grad_norm": 0.7514658570289612
    },
    {
      "bits_per_byte": 1.553970911684916,
      "learning_rate": 7.522168276463047e-05,
      "loss_nats": 1.0771305561065674,
      "step": 5960,
      "unclipped_grad_norm": 0.8067389130592346
    },
    {
      "bits_per_byte": 1.5228774803048255,
      "learning_rate": 7.480569019803719e-05,
      "loss_nats": 1.0555782318115234,
      "step": 5970,
      "unclipped_grad_norm": 0.8027852177619934
    },
    {
      "bits_per_byte": 1.564259601790133,
      "learning_rate": 7.439123880334388e-05,
      "loss_nats": 1.0842621326446533,
      "step": 5980,
      "unclipped_grad_norm": 0.7799409031867981
    },
    {
      "bits_per_byte": 1.5324249251848234,
      "learning_rate": 7.39783356623813e-05,
      "loss_nats": 1.0621960163116455,
      "step": 5990,
      "unclipped_grad_norm": 0.7881770133972168
    },
    {
      "bits_per_byte": 1.480203081180935,
      "learning_rate": 7.356698783052497e-05,
      "loss_nats": 1.025998592376709,
      "step": 6000,
      "unclipped_grad_norm": 0.7544389367103577
    },
    {
      "bits_per_byte": 1.5204526969103107,
      "learning_rate": 7.315720233657431e-05,
      "loss_nats": 1.053897500038147,
      "step": 6010,
      "unclipped_grad_norm": 0.7601525783538818
    },
    {
      "bits_per_byte": 1.4842316027947455,
      "learning_rate": 7.274898618263286e-05,
      "loss_nats": 1.0287909507751465,
      "step": 6020,
      "unclipped_grad_norm": 0.7522627711296082
    },
    {
      "bits_per_byte": 1.524829999340071,
      "learning_rate": 7.234234634398831e-05,
      "loss_nats": 1.0569316148757935,
      "step": 6030,
      "unclipped_grad_norm": 0.7871755361557007
    },
    {
      "bits_per_byte": 1.5740971814021363,
      "learning_rate": 7.193728976899362e-05,
      "loss_nats": 1.0910810232162476,
      "step": 6040,
      "unclipped_grad_norm": 0.8342390656471252
    },
    {
      "bits_per_byte": 1.475741851217554,
      "learning_rate": 7.153382337894797e-05,
      "loss_nats": 1.0229063034057617,
      "step": 6050,
      "unclipped_grad_norm": 0.7723444104194641
    },
    {
      "bits_per_byte": 1.4753325325085003,
      "learning_rate": 7.113195406797887e-05,
      "loss_nats": 1.0226225852966309,
      "step": 6060,
      "unclipped_grad_norm": 0.769443154335022
    },
    {
      "bits_per_byte": 1.5574803896584226,
      "learning_rate": 7.073168870292397e-05,
      "loss_nats": 1.0795631408691406,
      "step": 6070,
      "unclipped_grad_norm": 0.8035310506820679
    },
    {
      "bits_per_byte": 1.4788967009649812,
      "learning_rate": 7.033303412321404e-05,
      "loss_nats": 1.0250930786132812,
      "step": 6080,
      "unclipped_grad_norm": 0.7756995558738708
    },
    {
      "bits_per_byte": 1.4841172343319218,
      "learning_rate": 6.993599714075586e-05,
      "loss_nats": 1.0287116765975952,
      "step": 6090,
      "unclipped_grad_norm": 0.7475231289863586
    },
    {
      "bits_per_byte": 1.4853364193438883,
      "learning_rate": 6.954058453981609e-05,
      "loss_nats": 1.0295567512512207,
      "step": 6100,
      "unclipped_grad_norm": 0.7613393664360046
    },
    {
      "bits_per_byte": 1.5560739155396666,
      "learning_rate": 6.914680307690504e-05,
      "loss_nats": 1.0785882472991943,
      "step": 6110,
      "unclipped_grad_norm": 0.8045461177825928
    },
    {
      "bits_per_byte": 1.548306147130797,
      "learning_rate": 6.875465948066139e-05,
      "loss_nats": 1.0732040405273438,
      "step": 6120,
      "unclipped_grad_norm": 0.8037658333778381
    },
    {
      "bits_per_byte": 1.482889794152714,
      "learning_rate": 6.836416045173733e-05,
      "loss_nats": 1.0278608798980713,
      "step": 6130,
      "unclipped_grad_norm": 0.7671685814857483
    },
    {
      "bits_per_byte": 1.4915240111566304,
      "learning_rate": 6.797531266268377e-05,
      "loss_nats": 1.0338456630706787,
      "step": 6140,
      "unclipped_grad_norm": 0.7935377359390259
    },
    {
      "bits_per_byte": 1.5420002312542613,
      "learning_rate": 6.758812275783663e-05,
      "loss_nats": 1.0688331127166748,
      "step": 6150,
      "unclipped_grad_norm": 0.7919352650642395
    },
    {
      "bits_per_byte": 1.4144471484587233,
      "learning_rate": 6.720259735320298e-05,
      "loss_nats": 0.9804200530052185,
      "step": 6160,
      "unclipped_grad_norm": 0.7595162987709045
    },
    {
      "bits_per_byte": 1.5291687776560379,
      "learning_rate": 6.681874303634842e-05,
      "loss_nats": 1.0599390268325806,
      "step": 6170,
      "unclipped_grad_norm": 0.7883366942405701
    },
    {
      "bits_per_byte": 1.4831051164315943,
      "learning_rate": 6.643656636628402e-05,
      "loss_nats": 1.0280101299285889,
      "step": 6180,
      "unclipped_grad_norm": 0.7718276977539062
    },
    {
      "bits_per_byte": 1.4948415564917739,
      "learning_rate": 6.605607387335466e-05,
      "loss_nats": 1.0361452102661133,
      "step": 6190,
      "unclipped_grad_norm": 0.7770188450813293
    },
    {
      "bits_per_byte": 1.5074203675758797,
      "learning_rate": 6.567727205912724e-05,
      "loss_nats": 1.0448641777038574,
      "step": 6200,
      "unclipped_grad_norm": 0.7592265009880066
    },
    {
      "bits_per_byte": 1.5232448352470684,
      "learning_rate": 6.530016739627959e-05,
      "loss_nats": 1.055832862854004,
      "step": 6210,
      "unclipped_grad_norm": 0.8102302551269531
    },
    {
      "bits_per_byte": 1.5164056011702072,
      "learning_rate": 6.492476632848993e-05,
      "loss_nats": 1.051092267036438,
      "step": 6220,
      "unclipped_grad_norm": 0.787604033946991
    },
    {
      "bits_per_byte": 1.4586058438508853,
      "learning_rate": 6.455107527032678e-05,
      "loss_nats": 1.011028528213501,
      "step": 6230,
      "unclipped_grad_norm": 0.7935925126075745
    },
    {
      "bits_per_byte": 1.4934343944424144,
      "learning_rate": 6.417910060713926e-05,
      "loss_nats": 1.0351698398590088,
      "step": 6240,
      "unclipped_grad_norm": 0.8212590217590332
    },
    {
      "bits_per_byte": 1.5503186881111937,
      "learning_rate": 6.380884869494799e-05,
      "loss_nats": 1.074599027633667,
      "step": 6250,
      "unclipped_grad_norm": 0.8379690647125244
    },
    {
      "bits_per_byte": 1.403218659157867,
      "learning_rate": 6.34403258603367e-05,
      "loss_nats": 0.9726370573043823,
      "step": 6260,
      "unclipped_grad_norm": 0.7395487427711487
    },
    {
      "bits_per_byte": 1.5229197880369378,
      "learning_rate": 6.30735384003437e-05,
      "loss_nats": 1.055607557296753,
      "step": 6270,
      "unclipped_grad_norm": 0.8249105215072632
    },
    {
      "bits_per_byte": 1.509112332895068,
      "learning_rate": 6.270849258235484e-05,
      "loss_nats": 1.046036958694458,
      "step": 6280,
      "unclipped_grad_norm": 0.7966778874397278
    },
    {
      "bits_per_byte": 1.4545608119025921,
      "learning_rate": 6.234519464399581e-05,
      "loss_nats": 1.0082247257232666,
      "step": 6290,
      "unclipped_grad_norm": 0.7541455030441284
    },
    {
      "bits_per_byte": 1.5209793077872527,
      "learning_rate": 6.198365079302614e-05,
      "loss_nats": 1.0542625188827515,
      "step": 6300,
      "unclipped_grad_norm": 0.8022943139076233
    },
    {
      "bits_per_byte": 1.4739374092447013,
      "learning_rate": 6.16238672072326e-05,
      "loss_nats": 1.021655559539795,
      "step": 6310,
      "unclipped_grad_norm": 0.7839627861976624
    },
    {
      "bits_per_byte": 1.4204840834693098,
      "learning_rate": 6.126585003432406e-05,
      "loss_nats": 0.98460453748703,
      "step": 6320,
      "unclipped_grad_norm": 0.7670372128486633
    },
    {
      "bits_per_byte": 1.5195884840897251,
      "learning_rate": 6.090960539182614e-05,
      "loss_nats": 1.0532984733581543,
      "step": 6330,
      "unclipped_grad_norm": 0.7843371033668518
    },
    {
      "bits_per_byte": 1.4664117924082465,
      "learning_rate": 6.0555139366976935e-05,
      "loss_nats": 1.0164391994476318,
      "step": 6340,
      "unclipped_grad_norm": 0.7673349380493164
    },
    {
      "bits_per_byte": 1.577301734135663,
      "learning_rate": 6.020245801662274e-05,
      "loss_nats": 1.0933022499084473,
      "step": 6350,
      "unclipped_grad_norm": 0.8766481280326843
    },
    {
      "bits_per_byte": 1.4611757806027283,
      "learning_rate": 5.985156736711483e-05,
      "loss_nats": 1.0128098726272583,
      "step": 6360,
      "unclipped_grad_norm": 0.8388718962669373
    },
    {
      "bits_per_byte": 1.4358727471032122,
      "learning_rate": 5.950247341420618e-05,
      "loss_nats": 0.9952711462974548,
      "step": 6370,
      "unclipped_grad_norm": 0.775425136089325
    },
    {
      "bits_per_byte": 1.4795478272811478,
      "learning_rate": 5.915518212294935e-05,
      "loss_nats": 1.0255444049835205,
      "step": 6380,
      "unclipped_grad_norm": 0.7803093194961548
    },
    {
      "bits_per_byte": 1.5296866174177859,
      "learning_rate": 5.880969942759425e-05,
      "loss_nats": 1.060297966003418,
      "step": 6390,
      "unclipped_grad_norm": 0.8152122497558594
    },
    {
      "bits_per_byte": 1.5162291469704219,
      "learning_rate": 5.846603123148688e-05,
      "loss_nats": 1.0509699583053589,
      "step": 6400,
      "unclipped_grad_norm": 0.792400598526001
    },
    {
      "bits_per_byte": 1.5084371290077807,
      "learning_rate": 5.812418340696858e-05,
      "loss_nats": 1.0455689430236816,
      "step": 6410,
      "unclipped_grad_norm": 0.8240766525268555
    },
    {
      "bits_per_byte": 1.4750114408993396,
      "learning_rate": 5.778416179527541e-05,
      "loss_nats": 1.0224000215530396,
      "step": 6420,
      "unclipped_grad_norm": 0.853355348110199
    },
    {
      "bits_per_byte": 1.5300785658791023,
      "learning_rate": 5.7445972206438635e-05,
      "loss_nats": 1.0605696439743042,
      "step": 6430,
      "unclipped_grad_norm": 0.8168259859085083
    },
    {
      "bits_per_byte": 1.4651398087224652,
      "learning_rate": 5.710962041918516e-05,
      "loss_nats": 1.0155575275421143,
      "step": 6440,
      "unclipped_grad_norm": 0.8302746415138245
    },
    {
      "bits_per_byte": 1.53347470728569,
      "learning_rate": 5.677511218083908e-05,
      "loss_nats": 1.0629236698150635,
      "step": 6450,
      "unclipped_grad_norm": 0.8165877461433411
    },
    {
      "bits_per_byte": 1.585776179256928,
      "learning_rate": 5.644245320722316e-05,
      "loss_nats": 1.099176287651062,
      "step": 6460,
      "unclipped_grad_norm": 0.8728512525558472
    },
    {
      "bits_per_byte": 1.5696127337808887,
      "learning_rate": 5.61116491825615e-05,
      "loss_nats": 1.087972640991211,
      "step": 6470,
      "unclipped_grad_norm": 0.8505071401596069
    },
    {
      "bits_per_byte": 1.4561444281517368,
      "learning_rate": 5.5782705759382104e-05,
      "loss_nats": 1.0093224048614502,
      "step": 6480,
      "unclipped_grad_norm": 0.7639529705047607
    },
    {
      "bits_per_byte": 1.5541174409034513,
      "learning_rate": 5.5455628558420546e-05,
      "loss_nats": 1.0772321224212646,
      "step": 6490,
      "unclipped_grad_norm": 0.8572817444801331
    },
    {
      "bits_per_byte": 1.5173276001614828,
      "learning_rate": 5.513042316852368e-05,
      "loss_nats": 1.0517313480377197,
      "step": 6500,
      "unclipped_grad_norm": 0.825986921787262
    },
    {
      "bits_per_byte": 1.4589312350263177,
      "learning_rate": 5.480709514655443e-05,
      "loss_nats": 1.011254072189331,
      "step": 6510,
      "unclipped_grad_norm": 0.7764277458190918
    },
    {
      "bits_per_byte": 1.4310413244838487,
      "learning_rate": 5.448565001729653e-05,
      "loss_nats": 0.9919222593307495,
      "step": 6520,
      "unclipped_grad_norm": 0.7579286098480225
    },
    {
      "bits_per_byte": 1.4675702675444584,
      "learning_rate": 5.41660932733604e-05,
      "loss_nats": 1.017242193222046,
      "step": 6530,
      "unclipped_grad_norm": 0.8082018494606018
    },
    {
      "bits_per_byte": 1.5128018767040279,
      "learning_rate": 5.384843037508902e-05,
      "loss_nats": 1.048594355583191,
      "step": 6540,
      "unclipped_grad_norm": 0.8025459051132202
    },
    {
      "bits_per_byte": 1.4392037930764485,
      "learning_rate": 5.353266675046498e-05,
      "loss_nats": 0.9975800514221191,
      "step": 6550,
      "unclipped_grad_norm": 0.790289044380188
    },
    {
      "bits_per_byte": 1.527554032547087,
      "learning_rate": 5.321880779501729e-05,
      "loss_nats": 1.0588197708129883,
      "step": 6560,
      "unclipped_grad_norm": 0.818975567817688
    },
    {
      "bits_per_byte": 1.47142164702788,
      "learning_rate": 5.290685887172962e-05,
      "loss_nats": 1.019911766052246,
      "step": 6570,
      "unclipped_grad_norm": 0.8489425778388977
    },
    {
      "bits_per_byte": 1.514791200026558,
      "learning_rate": 5.2596825310948295e-05,
      "loss_nats": 1.0499732494354248,
      "step": 6580,
      "unclipped_grad_norm": 0.8219936490058899
    },
    {
      "bits_per_byte": 1.5010776474120593,
      "learning_rate": 5.228871241029154e-05,
      "loss_nats": 1.0404677391052246,
      "step": 6590,
      "unclipped_grad_norm": 0.7910152077674866
    },
    {
      "bits_per_byte": 1.4169470882716653,
      "learning_rate": 5.198252543455865e-05,
      "loss_nats": 0.9821528792381287,
      "step": 6600,
      "unclipped_grad_norm": 0.7658081650733948
    },
    {
      "bits_per_byte": 1.448498423459678,
      "learning_rate": 5.167826961564027e-05,
      "loss_nats": 1.0040225982666016,
      "step": 6610,
      "unclipped_grad_norm": 0.7563595771789551
    },
    {
      "bits_per_byte": 1.4411885588587308,
      "learning_rate": 5.137595015242884e-05,
      "loss_nats": 0.9989557862281799,
      "step": 6620,
      "unclipped_grad_norm": 0.8031777143478394
    },
    {
      "bits_per_byte": 1.471761484745985,
      "learning_rate": 5.1075572210729925e-05,
      "loss_nats": 1.0201473236083984,
      "step": 6630,
      "unclipped_grad_norm": 0.802291989326477
    },
    {
      "bits_per_byte": 1.511551906797557,
      "learning_rate": 5.07771409231737e-05,
      "loss_nats": 1.0477279424667358,
      "step": 6640,
      "unclipped_grad_norm": 0.8319684863090515
    },
    {
      "bits_per_byte": 1.5027432993856658,
      "learning_rate": 5.048066138912756e-05,
      "loss_nats": 1.041622281074524,
      "step": 6650,
      "unclipped_grad_norm": 0.7944393754005432
    },
    {
      "bits_per_byte": 1.5208112807373597,
      "learning_rate": 5.018613867460863e-05,
      "loss_nats": 1.0541460514068604,
      "step": 6660,
      "unclipped_grad_norm": 0.7983800768852234
    },
    {
      "bits_per_byte": 1.499563684136514,
      "learning_rate": 4.989357781219755e-05,
      "loss_nats": 1.039418339729309,
      "step": 6670,
      "unclipped_grad_norm": 0.8299691677093506
    },
    {
      "bits_per_byte": 1.4686320884308854,
      "learning_rate": 4.960298380095222e-05,
      "loss_nats": 1.0179781913757324,
      "step": 6680,
      "unclipped_grad_norm": 0.8146572113037109
    },
    {
      "bits_per_byte": 1.4999098852127009,
      "learning_rate": 4.931436160632256e-05,
      "loss_nats": 1.0396583080291748,
      "step": 6690,
      "unclipped_grad_norm": 0.8203337788581848
    },
    {
      "bits_per_byte": 1.5587066259590745,
      "learning_rate": 4.9027716160065485e-05,
      "loss_nats": 1.0804131031036377,
      "step": 6700,
      "unclipped_grad_norm": 0.8495987057685852
    },
    {
      "bits_per_byte": 1.5200210204566451,
      "learning_rate": 4.874305236016084e-05,
      "loss_nats": 1.0535982847213745,
      "step": 6710,
      "unclipped_grad_norm": 0.7851044535636902
    },
    {
      "bits_per_byte": 1.5404384568017762,
      "learning_rate": 4.846037507072752e-05,
      "loss_nats": 1.0677505731582642,
      "step": 6720,
      "unclipped_grad_norm": 0.8347281217575073
    },
    {
      "bits_per_byte": 1.4888245714686876,
      "learning_rate": 4.817968912194049e-05,
      "loss_nats": 1.0319745540618896,
      "step": 6730,
      "unclipped_grad_norm": 0.8212841153144836
    },
    {
      "bits_per_byte": 1.4304995791336308,
      "learning_rate": 4.790099930994813e-05,
      "loss_nats": 0.9915467500686646,
      "step": 6740,
      "unclipped_grad_norm": 0.7957568168640137
    },
    {
      "bits_per_byte": 1.5212219753076202,
      "learning_rate": 4.762431039679047e-05,
      "loss_nats": 1.0544307231903076,
      "step": 6750,
      "unclipped_grad_norm": 0.7967842817306519
    },
    {
      "bits_per_byte": 1.4749558905031108,
      "learning_rate": 4.734962711031755e-05,
      "loss_nats": 1.0223615169525146,
      "step": 6760,
      "unclipped_grad_norm": 0.8139022588729858
    },
    {
      "bits_per_byte": 1.3835263876600992,
      "learning_rate": 4.707695414410892e-05,
      "loss_nats": 0.9589874148368835,
      "step": 6770,
      "unclipped_grad_norm": 0.785581648349762
    },
    {
      "bits_per_byte": 1.4903192726873364,
      "learning_rate": 4.680629615739322e-05,
      "loss_nats": 1.0330106019973755,
      "step": 6780,
      "unclipped_grad_norm": 0.8513718843460083
    },
    {
      "bits_per_byte": 1.4442869123053497,
      "learning_rate": 4.653765777496863e-05,
      "loss_nats": 1.001103401184082,
      "step": 6790,
      "unclipped_grad_norm": 0.8081458210945129
    },
    {
      "bits_per_byte": 1.5181372944817448,
      "learning_rate": 4.6271043587123986e-05,
      "loss_nats": 1.0522925853729248,
      "step": 6800,
      "unclipped_grad_norm": 0.8491948843002319
    },
    {
      "bits_per_byte": 1.5056836867674672,
      "learning_rate": 4.600645814956006e-05,
      "loss_nats": 1.0436604022979736,
      "step": 6810,
      "unclipped_grad_norm": 0.8421627879142761
    },
    {
      "bits_per_byte": 1.5049580919634769,
      "learning_rate": 4.574390598331204e-05,
      "loss_nats": 1.0431574583053589,
      "step": 6820,
      "unclipped_grad_norm": 0.8799064755439758
    },
    {
      "bits_per_byte": 1.4726698971078425,
      "learning_rate": 4.548339157467199e-05,
      "loss_nats": 1.0207769870758057,
      "step": 6830,
      "unclipped_grad_norm": 0.8110744953155518
    },
    {
      "bits_per_byte": 1.4402100635666468,
      "learning_rate": 4.52249193751124e-05,
      "loss_nats": 0.9982775449752808,
      "step": 6840,
      "unclipped_grad_norm": 0.8030053973197937
    },
    {
      "bits_per_byte": 1.4826992373755579,
      "learning_rate": 4.4968493801209985e-05,
      "loss_nats": 1.027728796005249,
      "step": 6850,
      "unclipped_grad_norm": 0.8305994868278503
    },
    {
      "bits_per_byte": 1.4689192994578262,
      "learning_rate": 4.471411923457034e-05,
      "loss_nats": 1.0181772708892822,
      "step": 6860,
      "unclipped_grad_norm": 0.8366835117340088
    },
    {
      "bits_per_byte": 1.419515391188325,
      "learning_rate": 4.446180002175291e-05,
      "loss_nats": 0.9839330911636353,
      "step": 6870,
      "unclipped_grad_norm": 0.8165588974952698
    },
    {
      "bits_per_byte": 1.4317985640955977,
      "learning_rate": 4.421154047419693e-05,
      "loss_nats": 0.9924471378326416,
      "step": 6880,
      "unclipped_grad_norm": 0.7981967926025391
    },
    {
      "bits_per_byte": 1.426708565560662,
      "learning_rate": 4.396334486814753e-05,
      "loss_nats": 0.9889190196990967,
      "step": 6890,
      "unclipped_grad_norm": 0.8219130635261536
    },
    {
      "bits_per_byte": 1.544462506866664,
      "learning_rate": 4.371721744458285e-05,
      "loss_nats": 1.0705398321151733,
      "step": 6900,
      "unclipped_grad_norm": 0.8717687129974365
    },
    {
      "bits_per_byte": 1.4952656657088017,
      "learning_rate": 4.347316240914145e-05,
      "loss_nats": 1.0364391803741455,
      "step": 6910,
      "unclipped_grad_norm": 0.8866252899169922
    },
    {
      "bits_per_byte": 1.4499126367977229,
      "learning_rate": 4.3231183932050546e-05,
      "loss_nats": 1.0050028562545776,
      "step": 6920,
      "unclipped_grad_norm": 0.8052312135696411
    },
    {
      "bits_per_byte": 1.437854245215128,
      "learning_rate": 4.299128614805464e-05,
      "loss_nats": 0.9966446161270142,
      "step": 6930,
      "unclipped_grad_norm": 0.8604745268821716
    },
    {
      "bits_per_byte": 1.4202836376897292,
      "learning_rate": 4.2753473156344965e-05,
      "loss_nats": 0.9844655990600586,
      "step": 6940,
      "unclipped_grad_norm": 0.7888393998146057
    },
    {
      "bits_per_byte": 1.4300773617257627,
      "learning_rate": 4.251774902048942e-05,
      "loss_nats": 0.9912540912628174,
      "step": 6950,
      "unclipped_grad_norm": 0.8240157961845398
    },
    {
      "bits_per_byte": 1.4910837355704216,
      "learning_rate": 4.228411776836306e-05,
      "loss_nats": 1.0335404872894287,
      "step": 6960,
      "unclipped_grad_norm": 0.833236575126648
    },
    {
      "bits_per_byte": 1.496710663941351,
      "learning_rate": 4.2052583392079427e-05,
      "loss_nats": 1.0374407768249512,
      "step": 6970,
      "unclipped_grad_norm": 0.828235387802124
    },
    {
      "bits_per_byte": 1.4445393828368163,
      "learning_rate": 4.182314984792214e-05,
      "loss_nats": 1.0012784004211426,
      "step": 6980,
      "unclipped_grad_norm": 0.8125858902931213
    },
    {
      "bits_per_byte": 1.4867339503647994,
      "learning_rate": 4.159582105627748e-05,
      "loss_nats": 1.0305254459381104,
      "step": 6990,
      "unclipped_grad_norm": 0.8594081401824951
    },
    {
      "bits_per_byte": 1.533073471761227,
      "learning_rate": 4.137060090156724e-05,
      "loss_nats": 1.0626455545425415,
      "step": 7000,
      "unclipped_grad_norm": 0.8734340071678162
    },
    {
      "bits_per_byte": 1.532040715942796,
      "learning_rate": 4.114749323218249e-05,
      "loss_nats": 1.061929702758789,
      "step": 7010,
      "unclipped_grad_norm": 0.8863697648048401
    },
    {
      "bits_per_byte": 1.459764662952399,
      "learning_rate": 4.0926501860417706e-05,
      "loss_nats": 1.0118317604064941,
      "step": 7020,
      "unclipped_grad_norm": 0.8206458687782288
    },
    {
      "bits_per_byte": 1.5177566968753855,
      "learning_rate": 4.070763056240571e-05,
      "loss_nats": 1.052028775215149,
      "step": 7030,
      "unclipped_grad_norm": 0.8729474544525146
    },
    {
      "bits_per_byte": 1.50918250181662,
      "learning_rate": 4.049088307805306e-05,
      "loss_nats": 1.0460855960845947,
      "step": 7040,
      "unclipped_grad_norm": 0.8456775546073914
    },
    {
      "bits_per_byte": 1.4815705152379453,
      "learning_rate": 4.027626311097629e-05,
      "loss_nats": 1.0269464254379272,
      "step": 7050,
      "unclipped_grad_norm": 0.8360646367073059
    },
    {
      "bits_per_byte": 1.4998292253494463,
      "learning_rate": 4.006377432843846e-05,
      "loss_nats": 1.0396023988723755,
      "step": 7060,
      "unclipped_grad_norm": 0.8408570289611816
    },
    {
      "bits_per_byte": 1.453465282416596,
      "learning_rate": 3.985342036128656e-05,
      "loss_nats": 1.0074653625488281,
      "step": 7070,
      "unclipped_grad_norm": 0.8465698957443237
    },
    {
      "bits_per_byte": 1.431578598285144,
      "learning_rate": 3.964520480388956e-05,
      "loss_nats": 0.9922946691513062,
      "step": 7080,
      "unclipped_grad_norm": 0.8291451930999756
    },
    {
      "bits_per_byte": 1.4576069686146742,
      "learning_rate": 3.943913121407678e-05,
      "loss_nats": 1.01033616065979,
      "step": 7090,
      "unclipped_grad_norm": 0.8671156764030457
    },
    {
      "bits_per_byte": 1.4654204844086733,
      "learning_rate": 3.923520311307735e-05,
      "loss_nats": 1.0157520771026611,
      "step": 7100,
      "unclipped_grad_norm": 0.831190288066864
    },
    {
      "bits_per_byte": 1.478559958934592,
      "learning_rate": 3.9033423985459816e-05,
      "loss_nats": 1.0248596668243408,
      "step": 7110,
      "unclipped_grad_norm": 0.8307176828384399
    },
    {
      "bits_per_byte": 1.5234764958777807,
      "learning_rate": 3.88337972790728e-05,
      "loss_nats": 1.0559934377670288,
      "step": 7120,
      "unclipped_grad_norm": 0.8790873289108276
    },
    {
      "bits_per_byte": 1.486035012871693,
      "learning_rate": 3.8636326404985845e-05,
      "loss_nats": 1.030040979385376,
      "step": 7130,
      "unclipped_grad_norm": 0.8940773606300354
    },
    {
      "bits_per_byte": 1.4639995637472447,
      "learning_rate": 3.844101473743143e-05,
      "loss_nats": 1.0147671699523926,
      "step": 7140,
      "unclipped_grad_norm": 0.8168482184410095
    },
    {
      "bits_per_byte": 1.430851627619947,
      "learning_rate": 3.824786561374701e-05,
      "loss_nats": 0.991790771484375,
      "step": 7150,
      "unclipped_grad_norm": 0.7878822088241577
    },
    {
      "bits_per_byte": 1.5069802639723218,
      "learning_rate": 3.805688233431824e-05,
      "loss_nats": 1.044559121131897,
      "step": 7160,
      "unclipped_grad_norm": 0.8418039679527283
    },
    {
      "bits_per_byte": 1.4531985373251077,
      "learning_rate": 3.786806816252236e-05,
      "loss_nats": 1.0072804689407349,
      "step": 7170,
      "unclipped_grad_norm": 0.815416157245636
    },
    {
      "bits_per_byte": 1.4725130489302556,
      "learning_rate": 3.768142632467269e-05,
      "loss_nats": 1.0206682682037354,
      "step": 7180,
      "unclipped_grad_norm": 0.8564274907112122
    },
    {
      "bits_per_byte": 1.437259873173746,
      "learning_rate": 3.7496960009963236e-05,
      "loss_nats": 0.9962326288223267,
      "step": 7190,
      "unclipped_grad_norm": 0.8627841472625732
    },
    {
      "bits_per_byte": 1.4854186270510006,
      "learning_rate": 3.7314672370414325e-05,
      "loss_nats": 1.029613733291626,
      "step": 7200,
      "unclipped_grad_norm": 0.8707078695297241
    },
    {
      "bits_per_byte": 1.5339277095880628,
      "learning_rate": 3.713456652081882e-05,
      "loss_nats": 1.0632376670837402,
      "step": 7210,
      "unclipped_grad_norm": 0.8451290726661682
    },
    {
      "bits_per_byte": 1.5149174352922914,
      "learning_rate": 3.695664553868873e-05,
      "loss_nats": 1.050060749053955,
      "step": 7220,
      "unclipped_grad_norm": 0.8569709062576294
    },
    {
      "bits_per_byte": 1.4518221601702528,
      "learning_rate": 3.678091246420277e-05,
      "loss_nats": 1.00632643699646,
      "step": 7230,
      "unclipped_grad_norm": 0.8588427901268005
    },
    {
      "bits_per_byte": 1.4720953030713098,
      "learning_rate": 3.660737030015427e-05,
      "loss_nats": 1.0203787088394165,
      "step": 7240,
      "unclipped_grad_norm": 0.8185886144638062
    },
    {
      "bits_per_byte": 1.4930615360553439,
      "learning_rate": 3.6436022011900056e-05,
      "loss_nats": 1.0349113941192627,
      "step": 7250,
      "unclipped_grad_norm": 0.9033716917037964
    },
    {
      "bits_per_byte": 1.392136183127593,
      "learning_rate": 3.626687052730954e-05,
      "loss_nats": 0.9649552702903748,
      "step": 7260,
      "unclipped_grad_norm": 0.8070151209831238
    },
    {
      "bits_per_byte": 1.419755822934231,
      "learning_rate": 3.6099918736714955e-05,
      "loss_nats": 0.9840997457504272,
      "step": 7270,
      "unclipped_grad_norm": 0.8482468724250793
    },
    {
      "bits_per_byte": 1.499443640246212,
      "learning_rate": 3.5935169492861716e-05,
      "loss_nats": 1.0393351316452026,
      "step": 7280,
      "unclipped_grad_norm": 0.9195974469184875
    },
    {
      "bits_per_byte": 1.424472533125469,
      "learning_rate": 3.5772625610859886e-05,
      "loss_nats": 0.9873691201210022,
      "step": 7290,
      "unclipped_grad_norm": 0.84366774559021
    },
    {
      "bits_per_byte": 1.449468749575846,
      "learning_rate": 3.5612289868135887e-05,
      "loss_nats": 1.004695177078247,
      "step": 7300,
      "unclipped_grad_norm": 0.8698141574859619
    },
    {
      "bits_per_byte": 1.4526158601039845,
      "learning_rate": 3.545416500438522e-05,
      "loss_nats": 1.0068765878677368,
      "step": 7310,
      "unclipped_grad_norm": 0.8154390454292297
    },
    {
      "bits_per_byte": 1.40692677709312,
      "learning_rate": 3.52982537215255e-05,
      "loss_nats": 0.9752073287963867,
      "step": 7320,
      "unclipped_grad_norm": 0.835978627204895
    },
    {
      "bits_per_byte": 1.4261922736427717,
      "learning_rate": 3.5144558683650396e-05,
      "loss_nats": 0.9885611534118652,
      "step": 7330,
      "unclipped_grad_norm": 0.8695172071456909
    },
    {
      "bits_per_byte": 1.392467249730504,
      "learning_rate": 3.499308251698404e-05,
      "loss_nats": 0.96518474817276,
      "step": 7340,
      "unclipped_grad_norm": 0.8504277467727661
    },
    {
      "bits_per_byte": 1.4759263886019296,
      "learning_rate": 3.484382780983616e-05,
      "loss_nats": 1.0230342149734497,
      "step": 7350,
      "unclipped_grad_norm": 0.8514381051063538
    },
    {
      "bits_per_byte": 1.4321242992363319,
      "learning_rate": 3.469679711255795e-05,
      "loss_nats": 0.9926729202270508,
      "step": 7360,
      "unclipped_grad_norm": 0.8621233105659485
    },
    {
      "bits_per_byte": 1.4814723131443026,
      "learning_rate": 3.4551992937498284e-05,
      "loss_nats": 1.0268783569335938,
      "step": 7370,
      "unclipped_grad_norm": 0.8794771432876587
    },
    {
      "bits_per_byte": 1.401024074541532,
      "learning_rate": 3.440941775896104e-05,
      "loss_nats": 0.9711158871650696,
      "step": 7380,
      "unclipped_grad_norm": 0.8035550117492676
    },
    {
      "bits_per_byte": 1.474961909895891,
      "learning_rate": 3.4269074013162596e-05,
      "loss_nats": 1.022365689277649,
      "step": 7390,
      "unclipped_grad_norm": 0.8444681763648987
    },
    {
      "bits_per_byte": 1.4183169300857874,
      "learning_rate": 3.41309640981904e-05,
      "loss_nats": 0.9831023812294006,
      "step": 7400,
      "unclipped_grad_norm": 0.880998969078064
    },
    {
      "bits_per_byte": 1.4093773578852642,
      "learning_rate": 3.3995090373961795e-05,
      "loss_nats": 0.9769059419631958,
      "step": 7410,
      "unclipped_grad_norm": 0.8176257014274597
    },
    {
      "bits_per_byte": 1.476101638923159,
      "learning_rate": 3.3861455162183876e-05,
      "loss_nats": 1.023155689239502,
      "step": 7420,
      "unclipped_grad_norm": 0.8866280913352966
    },
    {
      "bits_per_byte": 1.489828262219123,
      "learning_rate": 3.3730060746313664e-05,
      "loss_nats": 1.032670259475708,
      "step": 7430,
      "unclipped_grad_norm": 0.8741231560707092
    },
    {
      "bits_per_byte": 1.5070485410847143,
      "learning_rate": 3.3600909371519225e-05,
      "loss_nats": 1.0446064472198486,
      "step": 7440,
      "unclipped_grad_norm": 0.8906005620956421
    },
    {
      "bits_per_byte": 1.4470038942236803,
      "learning_rate": 3.3474003244641194e-05,
      "loss_nats": 1.0029866695404053,
      "step": 7450,
      "unclipped_grad_norm": 0.833862841129303
    },
    {
      "bits_per_byte": 1.4545604679372903,
      "learning_rate": 3.334934453415514e-05,
      "loss_nats": 1.0082244873046875,
      "step": 7460,
      "unclipped_grad_norm": 0.851222813129425
    },
    {
      "bits_per_byte": 1.4397255884391662,
      "learning_rate": 3.3226935370134486e-05,
      "loss_nats": 0.9979417324066162,
      "step": 7470,
      "unclipped_grad_norm": 0.8808716535568237
    },
    {
      "bits_per_byte": 1.469680494670545,
      "learning_rate": 3.31067778442141e-05,
      "loss_nats": 1.018704891204834,
      "step": 7480,
      "unclipped_grad_norm": 0.9037331938743591
    },
    {
      "bits_per_byte": 1.5297206699826567,
      "learning_rate": 3.298887400955457e-05,
      "loss_nats": 1.060321569442749,
      "step": 7490,
      "unclipped_grad_norm": 0.8992816209793091
    },
    {
      "bits_per_byte": 1.48344512613235,
      "learning_rate": 3.287322588080714e-05,
      "loss_nats": 1.0282458066940308,
      "step": 7500,
      "unclipped_grad_norm": 0.8851720690727234
    },
    {
      "bits_per_byte": 1.4271278592634655,
      "learning_rate": 3.275983543407924e-05,
      "loss_nats": 0.9892096519470215,
      "step": 7510,
      "unclipped_grad_norm": 0.8391884565353394
    },
    {
      "bits_per_byte": 1.4636686691269847,
      "learning_rate": 3.2648704606900735e-05,
      "loss_nats": 1.0145378112792969,
      "step": 7520,
      "unclipped_grad_norm": 0.8814444541931152
    },
    {
      "bits_per_byte": 1.5040419403823309,
      "learning_rate": 3.253983529819086e-05,
      "loss_nats": 1.0425224304199219,
      "step": 7530,
      "unclipped_grad_norm": 0.8750982880592346
    },
    {
      "bits_per_byte": 1.448819515068839,
      "learning_rate": 3.243322936822571e-05,
      "loss_nats": 1.0042451620101929,
      "step": 7540,
      "unclipped_grad_norm": 0.8588963150978088
    },
    {
      "bits_per_byte": 1.4846842611318165,
      "learning_rate": 3.232888863860648e-05,
      "loss_nats": 1.0291047096252441,
      "step": 7550,
      "unclipped_grad_norm": 0.8553798198699951
    },
    {
      "bits_per_byte": 1.4456509067093426,
      "learning_rate": 3.222681489222838e-05,
      "loss_nats": 1.0020488500595093,
      "step": 7560,
      "unclipped_grad_norm": 0.8477172255516052
    },
    {
      "bits_per_byte": 1.4068580700241002,
      "learning_rate": 3.212700987325009e-05,
      "loss_nats": 0.9751597046852112,
      "step": 7570,
      "unclipped_grad_norm": 0.8558363914489746
    },
    {
      "bits_per_byte": 1.5845748804406512,
      "learning_rate": 3.202947528706403e-05,
      "loss_nats": 1.0983436107635498,
      "step": 7580,
      "unclipped_grad_norm": 0.9179992079734802
    },
    {
      "bits_per_byte": 1.435446746077025,
      "learning_rate": 3.1934212800267116e-05,
      "loss_nats": 0.9949758648872375,
      "step": 7590,
      "unclipped_grad_norm": 0.8465545177459717
    },
    {
      "bits_per_byte": 1.4142041369730542,
      "learning_rate": 3.1841224040632484e-05,
      "loss_nats": 0.9802516102790833,
      "step": 7600,
      "unclipped_grad_norm": 0.8474127054214478
    },
    {
      "bits_per_byte": 1.441303443269507,
      "learning_rate": 3.175051059708143e-05,
      "loss_nats": 0.9990354180335999,
      "step": 7610,
      "unclipped_grad_norm": 0.8852339386940002
    },
    {
      "bits_per_byte": 1.4014744111128163,
      "learning_rate": 3.1662074019656414e-05,
      "loss_nats": 0.9714280366897583,
      "step": 7620,
      "unclipped_grad_norm": 0.8255653977394104
    },
    {
      "bits_per_byte": 1.3991135192730972,
      "learning_rate": 3.157591581949455e-05,
      "loss_nats": 0.96979159116745,
      "step": 7630,
      "unclipped_grad_norm": 0.8985438942909241
    },
    {
      "bits_per_byte": 1.4200022740729177,
      "learning_rate": 3.149203746880175e-05,
      "loss_nats": 0.9842705726623535,
      "step": 7640,
      "unclipped_grad_norm": 0.8505365252494812
    },
    {
      "bits_per_byte": 1.5509151239443857,
      "learning_rate": 3.141044040082763e-05,
      "loss_nats": 1.075012445449829,
      "step": 7650,
      "unclipped_grad_norm": 0.8840228915214539
    },
    {
      "bits_per_byte": 1.4633312391659918,
      "learning_rate": 3.133112600984091e-05,
      "loss_nats": 1.0143039226531982,
      "step": 7660,
      "unclipped_grad_norm": 0.9004685878753662
    },
    {
      "bits_per_byte": 1.5004719245157205,
      "learning_rate": 3.125409565110571e-05,
      "loss_nats": 1.0400478839874268,
      "step": 7670,
      "unclipped_grad_norm": 0.9166978597640991
    },
    {
      "bits_per_byte": 1.4376227565670667,
      "learning_rate": 3.117935064085835e-05,
      "loss_nats": 0.9964841604232788,
      "step": 7680,
      "unclipped_grad_norm": 0.8822170495986938
    },
    {
      "bits_per_byte": 1.4453855374790612,
      "learning_rate": 3.110689225628479e-05,
      "loss_nats": 1.0018649101257324,
      "step": 7690,
      "unclipped_grad_norm": 0.8656983375549316
    },
    {
      "bits_per_byte": 1.396036663657836,
      "learning_rate": 3.1036721735498926e-05,
      "loss_nats": 0.9676588773727417,
      "step": 7700,
      "unclipped_grad_norm": 0.840412974357605
    },
    {
      "bits_per_byte": 1.4832948132954957,
      "learning_rate": 3.096884027752132e-05,
      "loss_nats": 1.0281416177749634,
      "step": 7710,
      "unclipped_grad_norm": 0.8699836730957031
    },
    {
      "bits_per_byte": 1.4846416094344024,
      "learning_rate": 3.090324904225885e-05,
      "loss_nats": 1.0290751457214355,
      "step": 7720,
      "unclipped_grad_norm": 0.8882943391799927
    },
    {
      "bits_per_byte": 1.4868813394965887,
      "learning_rate": 3.083994915048472e-05,
      "loss_nats": 1.0306276082992554,
      "step": 7730,
      "unclipped_grad_norm": 0.8993004560470581
    },
    {
      "bits_per_byte": 1.4329027787054622,
      "learning_rate": 3.077894168381942e-05,
      "loss_nats": 0.9932125210762024,
      "step": 7740,
      "unclipped_grad_norm": 0.8985061049461365
    },
    {
      "bits_per_byte": 1.457792709877606,
      "learning_rate": 3.0720227684712254e-05,
      "loss_nats": 1.0104649066925049,
      "step": 7750,
      "unclipped_grad_norm": 0.8967045545578003
    },
    {
      "bits_per_byte": 1.475737723633933,
      "learning_rate": 3.0663808156423456e-05,
      "loss_nats": 1.0229034423828125,
      "step": 7760,
      "unclipped_grad_norm": 0.8843175768852234
    },
    {
      "bits_per_byte": 1.4006856986759595,
      "learning_rate": 3.060968406300711e-05,
      "loss_nats": 0.9708813428878784,
      "step": 7770,
      "unclipped_grad_norm": 0.8820574879646301
    },
    {
      "bits_per_byte": 1.4787585988963383,
      "learning_rate": 3.055785632929463e-05,
      "loss_nats": 1.024997353553772,
      "step": 7780,
      "unclipped_grad_norm": 0.8838117718696594
    },
    {
      "bits_per_byte": 1.4172842602586817,
      "learning_rate": 3.0508325840878984e-05,
      "loss_nats": 0.982386589050293,
      "step": 7790,
      "unclipped_grad_norm": 0.8568381071090698
    },
    {
      "bits_per_byte": 1.4471380406913532,
      "learning_rate": 3.046109344409957e-05,
      "loss_nats": 1.0030796527862549,
      "step": 7800,
      "unclipped_grad_norm": 0.8466761708259583
    },
    {
      "bits_per_byte": 1.4200249757828314,
      "learning_rate": 3.0416159946027727e-05,
      "loss_nats": 0.9842863082885742,
      "step": 7810,
      "unclipped_grad_norm": 0.9039835929870605
    },
    {
      "bits_per_byte": 1.410308987904988,
      "learning_rate": 3.0373526114452946e-05,
      "loss_nats": 0.9775516986846924,
      "step": 7820,
      "unclipped_grad_norm": 0.8540316224098206
    },
    {
      "bits_per_byte": 1.4409317887609925,
      "learning_rate": 3.0333192677869803e-05,
      "loss_nats": 0.9987778067588806,
      "step": 7830,
      "unclipped_grad_norm": 0.9217174649238586
    },
    {
      "bits_per_byte": 1.4393295123942291,
      "learning_rate": 3.029516032546544e-05,
      "loss_nats": 0.9976671934127808,
      "step": 7840,
      "unclipped_grad_norm": 0.894000232219696
    },
    {
      "bits_per_byte": 1.3879004224194929,
      "learning_rate": 3.02594297071078e-05,
      "loss_nats": 0.9620192646980286,
      "step": 7850,
      "unclipped_grad_norm": 0.856924295425415
    },
    {
      "bits_per_byte": 1.4481823193473922,
      "learning_rate": 3.0226001433334603e-05,
      "loss_nats": 1.0038034915924072,
      "step": 7860,
      "unclipped_grad_norm": 0.8965994119644165
    },
    {
      "bits_per_byte": 1.4733294505739012,
      "learning_rate": 3.019487607534278e-05,
      "loss_nats": 1.021234154701233,
      "step": 7870,
      "unclipped_grad_norm": 0.9309923648834229
    },
    {
      "bits_per_byte": 1.3911279348369099,
      "learning_rate": 3.016605416497886e-05,
      "loss_nats": 0.9642564058303833,
      "step": 7880,
      "unclipped_grad_norm": 0.8823378086090088
    },
    {
      "bits_per_byte": 1.46931073197119,
      "learning_rate": 3.0139536194729734e-05,
      "loss_nats": 1.0184485912322998,
      "step": 7890,
      "unclipped_grad_norm": 0.9113782048225403
    },
    {
      "bits_per_byte": 1.449501254296859,
      "learning_rate": 3.0115322617714362e-05,
      "loss_nats": 1.0047177076339722,
      "step": 7900,
      "unclipped_grad_norm": 0.8579812049865723
    },
    {
      "bits_per_byte": 1.457684704772864,
      "learning_rate": 3.0093413847675965e-05,
      "loss_nats": 1.010390043258667,
      "step": 7910,
      "unclipped_grad_norm": 0.8884916305541992
    },
    {
      "bits_per_byte": 1.4556867823177908,
      "learning_rate": 3.0073810258974985e-05,
      "loss_nats": 1.0090051889419556,
      "step": 7920,
      "unclipped_grad_norm": 0.8975357413291931
    },
    {
      "bits_per_byte": 1.4187827450956494,
      "learning_rate": 3.0056512186582658e-05,
      "loss_nats": 0.9834252595901489,
      "step": 7930,
      "unclipped_grad_norm": 0.844153106212616
    },
    {
      "bits_per_byte": 1.4233246349222852,
      "learning_rate": 3.0041519926075316e-05,
      "loss_nats": 0.9865734577178955,
      "step": 7940,
      "unclipped_grad_norm": 0.9031311273574829
    },
    {
      "bits_per_byte": 1.4939792354803478,
      "learning_rate": 3.0028833733629313e-05,
      "loss_nats": 1.0355474948883057,
      "step": 7950,
      "unclipped_grad_norm": 0.910707950592041
    },
    {
      "bits_per_byte": 1.4252896226997183,
      "learning_rate": 3.0018453826016686e-05,
      "loss_nats": 0.987935483455658,
      "step": 7960,
      "unclipped_grad_norm": 0.8988999128341675
    },
    {
      "bits_per_byte": 1.446669559950403,
      "learning_rate": 3.001038038060143e-05,
      "loss_nats": 1.0027549266815186,
      "step": 7970,
      "unclipped_grad_norm": 0.9068711400032043
    },
    {
      "bits_per_byte": 1.484100552014788,
      "learning_rate": 3.0004613535336433e-05,
      "loss_nats": 1.0287001132965088,
      "step": 7980,
      "unclipped_grad_norm": 0.9153758883476257
    },
    {
      "bits_per_byte": 1.5191428770413398,
      "learning_rate": 3.000115338876117e-05,
      "loss_nats": 1.0529896020889282,
      "step": 7990,
      "unclipped_grad_norm": 0.9229548573493958
    },
    {
      "bits_per_byte": 1.419257675186007,
      "learning_rate": 3e-05,
      "loss_nats": 0.9837544560432434,
      "step": 8000,
      "unclipped_grad_norm": 0.8808227777481079
    }
  ]
}