{
  "architecture": "baseline",
  "batch_size": 32,
  "canonical_sha256_without_self": "5f1f6749be2f106aaf597f0478348ce6e226aa739e557d119b2388ec4460aa16",
  "depth": 32,
  "diagnostics": [
    {
      "activation_grad_rms_by_block": [
        0.0002897916710935533,
        0.0002662435290403664,
        0.00024211003619711846,
        0.00022891115804668516,
        0.00021424151782412082,
        0.00020374931045807898,
        0.00019545880786608905,
        0.00018862746946979314,
        0.00018024678865913302,
        0.00017406532424502075,
        0.0001698832056717947,
        0.00016633344057481736,
        0.00016165875422302634,
        0.0001582534605404362,
        0.00015452370280399919,
        0.0001515774056315422,
        0.00014964381989557296,
        0.00014728632231708616,
        0.00014459768135566264,
        0.00014173447561915964,
        0.00013920740457251668,
        0.00013723202573601156,
        0.00013503432273864746,
        0.0001331205276073888,
        0.00013141198724042624,
        0.00013009003305342048,
        0.0001286487968172878,
        0.00012732617324218154,
        0.00012632299331016839,
        0.00012548104859888554,
        0.00012477955897338688,
        0.00012385686568450183
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00022864168749947567,
        "first_to_last_ratio": 1.796936959436678,
        "imbalance_abs_log_ratio": 0.5860835261825511,
        "last_quartile_mean": 0.00012723968211503234,
        "mean": 0.00016535780059712124,
        "normalized": [
          1.7525128542293782,
          1.6101056501655084,
          1.4641585417974736,
          1.3843384298779209,
          1.2956238958819981,
          1.2321723542664615,
          1.1820356049746095,
          1.1407231396924917,
          1.0900410383317047,
          1.052658680851196,
          1.0273673516358577,
          1.0059001751001344,
          0.9776300461137163,
          0.9570365593214796,
          0.9344808787127115,
          0.9166631697094613,
          0.9049698251621409,
          0.8907128770776013,
          0.874453341986335,
          0.8571381277892198,
          0.8418556854882369,
          0.8299095974937676,
          0.816619005883163,
          0.8050453448623477,
          0.7947129604160569,
          0.786718452855893,
          0.7780025880407573,
          0.7700040323613145,
          0.7639373096038118,
          0.7588456555769529,
          0.7546034025779078,
          0.7490234221623898
        ],
        "population_cv": 0.25804396357350073
      },
      "activation_output_rms_by_block": [
        0.028274519369006157,
        0.02845057100057602,
        0.02867918275296688,
        0.02882254309952259,
        0.0290658138692379,
        0.029274335131049156,
        0.029572302475571632,
        0.02980491891503334,
        0.03008633479475975,
        0.03044024296104908,
        0.030671589076519012,
        0.030981140211224556,
        0.0313686840236187,
        0.031716227531433105,
        0.03204340860247612,
        0.03237249702215195,
        0.032637808471918106,
        0.03308072313666344,
        0.03343126177787781,
        0.033881161361932755,
        0.03435666114091873,
        0.03482793644070625,
        0.03533180430531502,
        0.03546451777219772,
        0.036012887954711914,
        0.03628316894173622,
        0.03689942881464958,
        0.03742557764053345,
        0.03777860850095749,
        0.03856590390205383,
        0.038892220705747604,
        0.039198413491249084
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.02899302332662046,
        "first_to_last_ratio": 0.7704348189669382,
        "imbalance_abs_log_ratio": 0.2608002235603661,
        "last_quartile_mean": 0.0376320262439549,
        "mean": 0.032990387349855155,
        "normalized": [
          0.8570532703712039,
          0.8623897227657417,
          0.8693193701798957,
          0.8736648889225345,
          0.8810388784163674,
          0.887359545694382,
          0.8963914901017818,
          0.9034425270294439,
          0.9119727657545023,
          0.9227003805150147,
          0.9297129115597873,
          0.9390959821925394,
          0.9508431559460433,
          0.961377846070436,
          0.9712953128638184,
          0.9812705949417742,
          0.9893126784418129,
          1.002738245715344,
          1.0133637239037931,
          1.0270010170729782,
          1.0414142997647942,
          1.0556995306349188,
          1.070972702764284,
          1.0749954947817073,
          1.0916176149374623,
          1.099810334354729,
          1.1184903172957619,
          1.1344388668021435,
          1.1451398887901616,
          1.1690042767025335,
          1.1788955459450936,
          1.1881768187672155
        ],
        "population_cv": 0.10167083190861884
      },
      "bits_per_byte": 8.0691508062416,
      "branch_output_rms_by_sublayer": [
        0.002122601494193077,
        0.002755566267296672,
        0.002245423151180148,
        0.0027348215226083994,
        0.0024954702239483595,
        0.002720654010772705,
        0.0023770928382873535,
        0.002733660163357854,
        0.0023630354553461075,
        0.0027867795433849096,
        0.0024109049700200558,
        0.0027537678834050894,
        0.002701795194298029,
        0.0027542903553694487,
        0.0029824089724570513,
        0.002726059639826417,
        0.0029706284403800964,
        0.0027311695739626884,
        0.003079906338825822,
        0.0027289879508316517,
        0.0028494365978986025,
        0.0027595392893999815,
        0.0032860173378139734,
        0.002710360335186124,
        0.0035252906382083893,
        0.0027479089330881834,
        0.003967344760894775,
        0.002725270576775074,
        0.0035556324291974306,
        0.002750981133431196,
        0.004272372927516699,
        0.0027044012676924467,
        0.004134721122682095,
        0.0027556782588362694,
        0.0041320133022964,
        0.0027166390791535378,
        0.0040069944225251675,
        0.0026826481334865093,
        0.004499548580497503,
        0.002729416359215975,
        0.0049178507179021835,
        0.002681323792785406,
        0.004439458716660738,
        0.002806412521749735,
        0.004581923596560955,
        0.0027696131728589535,
        0.004871132783591747,
        0.0027697959449142218,
        0.0049627418629825115,
        0.0028166540432721376,
        0.0051520452834665775,
        0.0027135310228914022,
        0.005508258938789368,
        0.002744151744991541,
        0.005101518705487251,
        0.002700615907087922,
        0.005637416150420904,
        0.0027939171995967627,
        0.005635067820549011,
        0.0027390308678150177,
        0.00545669998973608,
        0.002716719638556242,
        0.005571703426539898,
        0.002707480685785413
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.006421208939522194,
        0.005746349410576503,
        0.005404164999391892,
        0.005101115831054325,
        0.00485388847333966,
        0.0045855144167447825,
        0.004662942770173713,
        0.005059104716899598,
        0.00461737020280362,
        0.004446534400320367,
        0.004114212128857249,
        0.004678028852169936,
        0.004404060169276539,
        0.004656150674369456,
        0.004513829588134647,
        0.004839518675960965,
        0.004348161380019098,
        0.0047201153472655015,
        0.004169246752947831,
        0.004575240937826365,
        0.0047925402340120975,
        0.004508705150890575,
        0.004155367099556682,
        0.004478773084339979,
        0.004234175381969994,
        0.004428191743895327,
        0.004153362780335295,
        0.00410615213012631,
        0.004175626130102292,
        0.004125604583408443,
        0.0038633837013440408,
        0.003947405979968418
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.005229286194712833,
        "first_to_last_ratio": 1.266404707857162,
        "imbalance_abs_log_ratio": 0.236181947089728,
        "last_quartile_mean": 0.004129237803893765,
        "mean": 0.004590188958362615,
        "normalized": [
          1.3988986069568536,
          1.2518764396632391,
          1.1773295279154767,
          1.1113084618795224,
          1.057448509716931,
          0.9989816232707989,
          1.0158498511653973,
          1.1021560904769923,
          1.0059215959707901,
          0.9687039990411438,
          0.8963056131625672,
          1.0191364439686714,
          0.9594507348663767,
          1.0143701526462583,
          0.983364656461723,
          1.0543179637831923,
          0.9472728507390573,
          1.0283052375580706,
          0.9082952337620236,
          0.9967434847079623,
          1.044083430439357,
          0.9822482672911342,
          0.9052714686148694,
          0.9757273883421174,
          0.9224403222564466,
          0.9647079421050512,
          0.9048348157364001,
          0.894549694440255,
          0.9096850190654888,
          0.8987875272307099,
          0.8416611464993293,
          0.859965900265795
        ],
        "population_cv": 0.11345956482154346
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.028097067028284073,
        0.028159458190202713,
        0.028274519369006157,
        0.02833542972803116,
        0.02845057100057602,
        0.02851351536810398,
        0.02867918275296688,
        0.028714356943964958,
        0.02882254309952259,
        0.028916271403431892,
        0.0290658138692379,
        0.02915445901453495,
        0.029274335131049156,
        0.02944180741906166,
        0.029572302475571632,
        0.02970401756465435,
        0.02980491891503334,
        0.02993646264076233,
        0.03008633479475975,
        0.030266879126429558,
        0.03044024296104908,
        0.03057776764035225,
        0.030671589076519012,
        0.030867453664541245,
        0.030981140211224556,
        0.03126408904790878,
        0.0313686840236187,
        0.031607914716005325,
        0.031716227531433105,
        0.03191319853067398,
        0.03204340860247612,
        0.032251372933387756,
        0.03237249702215195,
        0.03250224143266678,
        0.032637808471918106,
        0.03294040635228157,
        0.03308072313666344,
        0.0333157554268837,
        0.03343126177787781,
        0.03377276286482811,
        0.033881161361932755,
        0.034277379512786865,
        0.03435666114091873,
        0.03472030907869339,
        0.03482793644070625,
        0.0352315790951252,
        0.03533180430531502,
        0.03532300516963005,
        0.03546451777219772,
        0.03583687171339989,
        0.036012887954711914,
        0.03619367629289627,
        0.03628316894173622,
        0.03677845373749733,
        0.03689942881464958,
        0.03733044117689133,
        0.03742557764053345,
        0.0377187617123127,
        0.03777860850095749,
        0.03849533945322037,
        0.03856590390205383,
        0.03877026215195656,
        0.038892220705747604,
        0.03908585384488106
      ],
      "loss_nats": 5.593109130859375,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 0,
      "stream_state_rms_by_sublayer": [
        0.028159458190202713,
        0.028274519369006157,
        0.02833542972803116,
        0.02845057100057602,
        0.02851351536810398,
        0.02867918275296688,
        0.028714356943964958,
        0.02882254309952259,
        0.028916271403431892,
        0.0290658138692379,
        0.02915445901453495,
        0.029274335131049156,
        0.02944180741906166,
        0.029572302475571632,
        0.02970401756465435,
        0.02980491891503334,
        0.02993646264076233,
        0.03008633479475975,
        0.030266879126429558,
        0.03044024296104908,
        0.03057776764035225,
        0.030671589076519012,
        0.030867453664541245,
        0.030981140211224556,
        0.03126408904790878,
        0.0313686840236187,
        0.031607914716005325,
        0.031716227531433105,
        0.03191319853067398,
        0.03204340860247612,
        0.032251372933387756,
        0.03237249702215195,
        0.03250224143266678,
        0.032637808471918106,
        0.03294040635228157,
        0.03308072313666344,
        0.0333157554268837,
        0.03343126177787781,
        0.03377276286482811,
        0.033881161361932755,
        0.034277379512786865,
        0.03435666114091873,
        0.03472030907869339,
        0.03482793644070625,
        0.0352315790951252,
        0.03533180430531502,
        0.03532300516963005,
        0.03546451777219772,
        0.03583687171339989,
        0.036012887954711914,
        0.03619367629289627,
        0.03628316894173622,
        0.03677845373749733,
        0.03689942881464958,
        0.03733044117689133,
        0.03742557764053345,
        0.0377187617123127,
        0.03777860850095749,
        0.03849533945322037,
        0.03856590390205383,
        0.03877026215195656,
        0.038892220705747604,
        0.03908585384488106,
        0.039198413491249084
      ]
    },
    {
      "activation_grad_rms_by_block": [
        7.023746729828417e-05,
        6.667003617621958e-05,
        6.32333685643971e-05,
        6.012299854774028e-05,
        5.672227416653186e-05,
        5.335300011211075e-05,
        5.0173275667475536e-05,
        4.719031130662188e-05,
        4.408728636917658e-05,
        4.096491466043517e-05,
        3.824968007393181e-05,
        3.570610351744108e-05,
        3.324189310660586e-05,
        3.1057134037837386e-05,
        2.914190736191813e-05,
        2.7255719032837078e-05,
        2.56736693700077e-05,
        2.4158533051377162e-05,
        2.2910870029591024e-05,
        2.187793143093586e-05,
        2.0998755644541234e-05,
        2.0234831026755273e-05,
        1.963501563295722e-05,
        1.908456215460319e-05,
        1.8524928236729465e-05,
        1.8092870959662832e-05,
        1.7702994227875024e-05,
        1.74019587575458e-05,
        1.7127178580267355e-05,
        1.6885946024558507e-05,
        1.666023126745131e-05,
        1.649606565479189e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 5.8462841479922645e-05,
        "first_to_last_ratio": 3.3673800283354014,
        "imbalance_abs_log_ratio": 1.2141350023382471,
        "last_quartile_mean": 1.7361521713610273e-05,
        "mean": 3.315230350153797e-05,
        "normalized": [
          2.1186300763392136,
          2.011022738529366,
          1.9073597272498315,
          1.8135390967614395,
          1.710960270495245,
          1.6093301061157226,
          1.5134174813870156,
          1.4234398917237432,
          1.3298408168570044,
          1.235658169530656,
          1.1537563316575383,
          1.0770323551057208,
          1.0027023644092703,
          0.9368016927208878,
          0.8790311466763148,
          0.8221365079977823,
          0.7744158522443748,
          0.7287135583280486,
          0.691079279861448,
          0.6599219094963015,
          0.6334026123876152,
          0.610359730382432,
          0.5922670088986827,
          0.5756632311754095,
          0.5587825363589011,
          0.5457500399277982,
          0.5339898697251538,
          0.5249094910324558,
          0.516621072181992,
          0.5093445776331997,
          0.5025361591142052,
          0.49758429769523016
        ],
        "population_cv": 0.5033867184228629
      },
      "activation_output_rms_by_block": [
        0.029364673420786858,
        0.03058232180774212,
        0.03253435716032982,
        0.035004038363695145,
        0.03789948299527168,
        0.041081421077251434,
        0.04464675113558769,
        0.04816310852766037,
        0.05221380665898323,
        0.056122854351997375,
        0.06031070277094841,
        0.06517986208200455,
        0.07041798532009125,
        0.07559357583522797,
        0.08127844333648682,
        0.08752159774303436,
        0.09348469227552414,
        0.10084687173366547,
        0.10794128477573395,
        0.11579020321369171,
        0.12420836836099625,
        0.13254153728485107,
        0.14211823046207428,
        0.151936337351799,
        0.16282601654529572,
        0.17539408802986145,
        0.19021794199943542,
        0.2045798897743225,
        0.2205442190170288,
        0.23628318309783936,
        0.2528626322746277,
        0.27021101117134094
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.03740951931104064,
        "first_to_last_ratio": 0.17471705179812935,
        "imbalance_abs_log_ratio": 1.74458746043398,
        "last_quartile_mean": 0.214114872738719,
        "mean": 0.11030317156109959,
        "normalized": [
          0.2662178521722837,
          0.2772569580268309,
          0.29495395916434053,
          0.31734389744455865,
          0.34359377394944846,
          0.37244097785978386,
          0.40476398369793726,
          0.43664300714129206,
          0.47336632229165543,
          0.5088054455524842,
          0.5467721545752731,
          0.590915575314168,
          0.6384039944045038,
          0.685325496677626,
          0.736864064615447,
          0.793464018344695,
          0.8475249709727586,
          0.9142699190458359,
          0.9785873175545334,
          1.0497450034748341,
          1.126063435920283,
          1.2016112991949022,
          1.2884328569224464,
          1.3774430526472923,
          1.476168039783901,
          1.5901092012817277,
          1.7245011118657556,
          1.8547054167069055,
          1.9994367876798904,
          2.142125015570893,
          2.292433016167273,
          2.4497120739784397
        ],
        "population_cv": 0.6367597776091559
      },
      "bits_per_byte": 5.184434896455056,
      "branch_output_rms_by_sublayer": [
        0.0026808311231434345,
        0.00477010989561677,
        0.0025234855711460114,
        0.004875619895756245,
        0.0027965111657977104,
        0.005397462286055088,
        0.002715951530262828,
        0.00578050222247839,
        0.0028346737381070852,
        0.005865047685801983,
        0.0025751085486263037,
        0.0059114922769367695,
        0.0034423002507537603,
        0.006014656741172075,
        0.0034009802620857954,
        0.005813058000057936,
        0.0035280236043035984,
        0.006192575208842754,
        0.003798100398853421,
        0.006026907358318567,
        0.0032950215972959995,
        0.006300882901996374,
        0.004497972317039967,
        0.006683246698230505,
        0.004819076508283615,
        0.006906723603606224,
        0.004967817571014166,
        0.006679704412817955,
        0.005188894923776388,
        0.006651155184954405,
        0.005964669398963451,
        0.006957803387194872,
        0.005442580208182335,
        0.006974066607654095,
        0.00629971781745553,
        0.007549752946943045,
        0.007111458573490381,
        0.007369632367044687,
        0.008486796170473099,
        0.0074090128764510155,
        0.008528640493750572,
        0.007484533358365297,
        0.007537186611443758,
        0.007801020983606577,
        0.00965037476271391,
        0.007585691753774881,
        0.009028015658259392,
        0.007420822512358427,
        0.010029759258031845,
        0.00862929131835699,
        0.01243194192647934,
        0.0086217625066638,
        0.0149146169424057,
        0.00852242112159729,
        0.014407102949917316,
        0.008064002729952335,
        0.01547334622591734,
        0.008129187859594822,
        0.015585782937705517,
        0.008083954453468323,
        0.014871877618134022,
        0.009126133285462856,
        0.016274450346827507,
        0.008773879148066044
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00026099601887828554,
        0.00022299818257146475,
        0.00021593606328411676,
        0.0001980902028289896,
        0.00019039206376599677,
        0.0001764249768268048,
        0.00019262760932023515,
        0.00017741019012756782,
        0.0001866267032514166,
        0.00018194420611625288,
        0.00016335812718032435,
        0.00018628060025801724,
        0.00018197064718745634,
        0.00018231380553041538,
        0.00016812738233283717,
        0.0001758928024359223,
        0.00015172497688937343,
        0.0001676233054310719,
        0.0001547068922126687,
        0.0001733627635453279,
        0.0001619137110647745,
        0.00014190991701657472,
        0.0001580141865100647,
        0.00014951069781870242,
        0.00014861801255322757,
        0.00015877260520542335,
        0.00016243531213177502,
        0.00014972906733594748,
        0.0001506252547372875,
        0.00014728724932094392,
        0.00014196626632039671,
        0.00013988030321371655
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00020435941345043265,
        "first_to_last_ratio": 1.3631752910956885,
        "imbalance_abs_log_ratio": 0.30981675126526836,
        "last_quartile_mean": 0.00014991425885233975,
        "mean": 0.00017248344072510563,
        "normalized": [
          1.513165656836857,
          1.2928671971871588,
          1.251923444803397,
          1.1484592491675452,
          1.1038280716433115,
          1.0228516783125923,
          1.1167890002103689,
          1.028563608087539,
          1.0819977991327974,
          1.0548502821567973,
          0.9470945529239257,
          1.0799912123442663,
          1.0550035784447904,
          1.0569930931571387,
          0.9747450632132802,
          1.0197663131978583,
          0.8796495260735443,
          0.9718225977311089,
          0.8969376513031868,
          1.0050980129833083,
          0.9387203222761739,
          0.8227451656808188,
          0.9161122125450802,
          0.8668118933050746,
          0.8616364094341472,
          0.9205092647616311,
          0.941744386875116,
          0.8680779250837024,
          0.8732737131406461,
          0.853921099334295,
          0.823071859672921,
          0.8109781589796199
        ],
        "population_cv": 0.15033028761203962
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.028610490262508392,
        0.028607122600078583,
        0.029364673420786858,
        0.029366537928581238,
        0.03058232180774212,
        0.03066444955766201,
        0.03253435716032982,
        0.03263945132493973,
        0.035004038363695145,
        0.035203028470277786,
        0.03789948299527168,
        0.03799305483698845,
        0.041081421077251434,
        0.04133254662156105,
        0.04464675113558769,
        0.04496326297521591,
        0.04816310852766037,
        0.048516932874917984,
        0.05221380665898323,
        0.05251964181661606,
        0.056122854351997375,
        0.056522201746702194,
        0.06031070277094841,
        0.060862768441438675,
        0.06517986208200455,
        0.06596556305885315,
        0.07041798532009125,
        0.0712207704782486,
        0.07559357583522797,
        0.07684765011072159,
        0.08127844333648682,
        0.08279602974653244,
        0.08752159774303436,
        0.08871174603700638,
        0.09348469227552414,
        0.09565582871437073,
        0.10084687173366547,
        0.10279740393161774,
        0.10794128477573395,
        0.11048463732004166,
        0.11579020321369171,
        0.11891037970781326,
        0.12420836836099625,
        0.1270110011100769,
        0.13254153728485107,
        0.13657133281230927,
        0.14211823046207428,
        0.14646445214748383,
        0.151936337351799,
        0.15653157234191895,
        0.16282601654529572,
        0.16900379955768585,
        0.17539408802986145,
        0.18384796380996704,
        0.19021794199943542,
        0.1985616832971573,
        0.2045798897743225,
        0.21447749435901642,
        0.2205442190170288,
        0.23017354309558868,
        0.23628318309783936,
        0.24575340747833252,
        0.2528626322746277,
        0.26334208250045776
      ],
      "loss_nats": 3.593576431274414,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 100,
      "stream_state_rms_by_sublayer": [
        0.028607122600078583,
        0.029364673420786858,
        0.029366537928581238,
        0.03058232180774212,
        0.03066444955766201,
        0.03253435716032982,
        0.03263945132493973,
        0.035004038363695145,
        0.035203028470277786,
        0.03789948299527168,
        0.03799305483698845,
        0.041081421077251434,
        0.04133254662156105,
        0.04464675113558769,
        0.04496326297521591,
        0.04816310852766037,
        0.048516932874917984,
        0.05221380665898323,
        0.05251964181661606,
        0.056122854351997375,
        0.056522201746702194,
        0.06031070277094841,
        0.060862768441438675,
        0.06517986208200455,
        0.06596556305885315,
        0.07041798532009125,
        0.0712207704782486,
        0.07559357583522797,
        0.07684765011072159,
        0.08127844333648682,
        0.08279602974653244,
        0.08752159774303436,
        0.08871174603700638,
        0.09348469227552414,
        0.09565582871437073,
        0.10084687173366547,
        0.10279740393161774,
        0.10794128477573395,
        0.11048463732004166,
        0.11579020321369171,
        0.11891037970781326,
        0.12420836836099625,
        0.1270110011100769,
        0.13254153728485107,
        0.13657133281230927,
        0.14211823046207428,
        0.14646445214748383,
        0.151936337351799,
        0.15653157234191895,
        0.16282601654529572,
        0.16900379955768585,
        0.17539408802986145,
        0.18384796380996704,
        0.19021794199943542,
        0.1985616832971573,
        0.2045798897743225,
        0.21447749435901642,
        0.2205442190170288,
        0.23017354309558868,
        0.23628318309783936,
        0.24575340747833252,
        0.2528626322746277,
        0.26334208250045776,
        0.27021101117134094
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.0001506932603660971,
        0.00015340163372457027,
        0.00015237691695801914,
        0.000148982901009731,
        0.0001392244448652491,
        0.00012851000064983964,
        0.00011962141434196383,
        0.0001119004882639274,
        0.00010405662760604173,
        9.720238449517637e-05,
        9.135912114288658e-05,
        8.468296437058598e-05,
        7.889774860814214e-05,
        7.427640230162069e-05,
        6.881110311951488e-05,
        6.43495368422009e-05,
        5.969387711957097e-05,
        5.503925422090106e-05,
        5.034419518779032e-05,
        4.6255379857029766e-05,
        4.248250479577109e-05,
        3.87999571103137e-05,
        3.563905556802638e-05,
        3.2581570849288255e-05,
        3.028466016985476e-05,
        2.807797682180535e-05,
        2.6328352760174312e-05,
        2.5215304049197584e-05,
        2.4513139578630216e-05,
        2.4017937903408892e-05,
        2.3832226361264475e-05,
        2.3948901798576117e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00013808888252242468,
        "first_to_last_ratio": 5.356993010635397,
        "imbalance_abs_log_ratio": 1.678402812181787,
        "last_quartile_mean": 2.5777312430363963e-05,
        "mean": 7.298128883803656e-05,
        "normalized": [
          2.0648204870774824,
          2.1019310040551114,
          2.087890189171378,
          2.041384899907163,
          1.907673145841839,
          1.7608623072556988,
          1.6390696334156716,
          1.533276406124618,
          1.425798711735127,
          1.3318808977310936,
          1.2518156721736573,
          1.1603380225103768,
          1.0810681732852876,
          1.0177458288857888,
          0.9428595221471582,
          0.881726506433839,
          0.8179339947263247,
          0.7541556897281809,
          0.6898233230645799,
          0.6337977938383883,
          0.5821013231220159,
          0.5316425309563984,
          0.488331408439747,
          0.44643731794864266,
          0.4149647211227507,
          0.3847284320247795,
          0.3607548342781861,
          0.3455036825282238,
          0.3358825250816124,
          0.32909720129375364,
          0.3265525553289974,
          0.3281512587661287
        ],
        "population_cv": 0.6061250608697893
      },
      "activation_output_rms_by_block": [
        0.048818644136190414,
        0.05689374357461929,
        0.06500593572854996,
        0.07087047398090363,
        0.07611831277608871,
        0.0806790366768837,
        0.08478696644306183,
        0.08861920237541199,
        0.0918315127491951,
        0.09433750063180923,
        0.09700070321559906,
        0.09972338378429413,
        0.10201811045408249,
        0.10424874722957611,
        0.10649008303880692,
        0.10806933790445328,
        0.11109450459480286,
        0.11401215940713882,
        0.11832662671804428,
        0.12293888628482819,
        0.12948566675186157,
        0.1387748122215271,
        0.14864373207092285,
        0.16069285571575165,
        0.1741981953382492,
        0.18964411318302155,
        0.20650440454483032,
        0.22468066215515137,
        0.24238750338554382,
        0.2638821303844452,
        0.2911032736301422,
        0.32429978251457214
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.07147403946146369,
        "first_to_last_ratio": 0.2983212272448851,
        "imbalance_abs_log_ratio": 1.2095844292661684,
        "last_quartile_mean": 0.23958750814199448,
        "mean": 0.13550565636251122,
        "normalized": [
          0.3602701573253034,
          0.41986249948426085,
          0.4797285771941724,
          0.523007495652488,
          0.5617353165867361,
          0.5953923905659501,
          0.6257079499045831,
          0.6539889533344176,
          0.67769505136762,
          0.6961886548811884,
          0.7158424660598531,
          0.7359352108336301,
          0.7528697560872202,
          0.7693313329348029,
          0.7858718661449788,
          0.7975263970925391,
          0.8198514185828341,
          0.8413830275994386,
          0.8732227856340641,
          0.907260180755379,
          0.9555738869339544,
          1.024125604397429,
          1.0969559209636535,
          1.185875630799193,
          1.285541873412471,
          1.3995291287005507,
          1.523954129209041,
          1.6580906523494112,
          1.7887629925727762,
          1.9473883044298541,
          2.148273964677672,
          2.393256423532534
        ],
        "population_cv": 0.5081939166956976
      },
      "bits_per_byte": 3.1154419867739596,
      "branch_output_rms_by_sublayer": [
        0.010533316060900688,
        0.01737932860851288,
        0.0117434561252594,
        0.012238939292728901,
        0.01211429014801979,
        0.011732303537428379,
        0.014587659388780594,
        0.009741203859448433,
        0.012628368102014065,
        0.009234351105988026,
        0.014413262717425823,
        0.00937508512288332,
        0.015033986419439316,
        0.00873780157417059,
        0.016592979431152344,
        0.008810308761894703,
        0.018142949789762497,
        0.00861456897109747,
        0.01592753641307354,
        0.008246068842709064,
        0.013729470781981945,
        0.008380995132029057,
        0.017536180093884468,
        0.008834624662995338,
        0.021067099645733833,
        0.009005429223179817,
        0.022672412917017937,
        0.009119905531406403,
        0.019366111606359482,
        0.0090357456356287,
        0.0166512168943882,
        0.009071468375623226,
        0.014951958321034908,
        0.01051315013319254,
        0.017114169895648956,
        0.010944797657430172,
        0.01693827658891678,
        0.012414379045367241,
        0.01647246815264225,
        0.013282178901135921,
        0.01672390289604664,
        0.014548475854098797,
        0.017381425946950912,
        0.017058325931429863,
        0.02400100603699684,
        0.018193069845438004,
        0.017901180312037468,
        0.02045683190226555,
        0.01391419768333435,
        0.021929442882537842,
        0.014953838661313057,
        0.02380906604230404,
        0.010647366754710674,
        0.02548205293715,
        0.005664720665663481,
        0.026819046586751938,
        0.008504491299390793,
        0.024410424754023552,
        0.008407330140471458,
        0.02826099656522274,
        0.01054995134472847,
        0.033158741891384125,
        0.02235519140958786,
        0.03220592811703682
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0005931392614171548,
        0.0004012275990867258,
        0.00038098222807989436,
        0.0003649111519840816,
        0.00034656151495199636,
        0.00031005860528648927,
        0.00027821812913176474,
        0.0002561842642921938,
        0.0002305368063696604,
        0.00020943363497294808,
        0.0001942700943631853,
        0.0001900904574854747,
        0.00018678033884870243,
        0.00017357948080148642,
        0.00015815775362626997,
        0.00014490835987546657,
        0.0001375100278294081,
        0.0001295570650502436,
        0.00012744106699623042,
        0.00011906627060222178,
        0.00011368520556157287,
        0.0001091753996736031,
        0.00010941555658600093,
        9.636520203422524e-05,
        9.391483800802352e-05,
        9.101856383525677e-05,
        7.998410113295477e-05,
        7.102545922046693e-05,
        6.763159533414917e-05,
        6.887797791066819e-05,
        7.001610738319118e-05,
        7.664349539138475e-05
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00036641034427878756,
        "first_to_last_ratio": 4.73465560322638,
        "imbalance_abs_log_ratio": 1.5549089897216843,
        "last_quartile_mean": 7.738901727701191e-05,
        "mean": 0.00018688648791009675,
        "normalized": [
          3.173794253667441,
          2.146905341170195,
          2.038575567128047,
          1.9525817840807467,
          1.8543957823141959,
          1.6590744935805388,
          1.4887011481836172,
          1.3708014268823612,
          1.2335659412710565,
          1.1206462131906392,
          1.0395085088047697,
          1.017143933792149,
          0.9994320131830752,
          0.9287963123636216,
          0.8462770925544549,
          0.7753816848715992,
          0.7357943817509076,
          0.6932393374130293,
          0.681916966932018,
          0.6371047577259816,
          0.6083115308810451,
          0.5841802737826761,
          0.5854653153877886,
          0.5156349349375247,
          0.5025234250921448,
          0.4870259206703161,
          0.4279822582541749,
          0.38004598414110224,
          0.3618859559642637,
          0.3685551517443171,
          0.3746451023086966,
          0.4101072059755049
        ],
        "population_cv": 0.6522632271008562
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.04385505989193916,
        0.04404326155781746,
        0.048818644136190414,
        0.05170967057347298,
        0.05689374357461929,
        0.059983257204294205,
        0.06500593572854996,
        0.06660757958889008,
        0.07087047398090363,
        0.07236418128013611,
        0.07611831277608871,
        0.07718922197818756,
        0.0806790366768837,
        0.08160495012998581,
        0.08478696644306183,
        0.08585704118013382,
        0.08861920237541199,
        0.08891960233449936,
        0.0918315127491951,
        0.0918683186173439,
        0.09433750063180923,
        0.09460321068763733,
        0.09700070321559906,
        0.09746499359607697,
        0.09972338378429413,
        0.0994088351726532,
        0.10201811045408249,
        0.10191889107227325,
        0.10424874722957611,
        0.10386203229427338,
        0.10649008303880692,
        0.10544375330209732,
        0.10806933790445328,
        0.10778804868459702,
        0.11109450459480286,
        0.11027765274047852,
        0.11401215940713882,
        0.11356210708618164,
        0.11832662671804428,
        0.11750734597444534,
        0.12293888628482819,
        0.1225726455450058,
        0.12948566675186157,
        0.12988287210464478,
        0.1387748122215271,
        0.13870327174663544,
        0.14864373207092285,
        0.14834089577198029,
        0.16069285571575165,
        0.16013014316558838,
        0.1741981953382492,
        0.1741078794002533,
        0.18964411318302155,
        0.18942901492118835,
        0.20650440454483032,
        0.20662079751491547,
        0.22468066215515137,
        0.2257053554058075,
        0.24238750338554382,
        0.24448354542255402,
        0.2638821303844452,
        0.2678608000278473,
        0.2911032736301422,
        0.30181384086608887
      ],
      "loss_nats": 2.1594598293304443,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 500,
      "stream_state_rms_by_sublayer": [
        0.04404326155781746,
        0.048818644136190414,
        0.05170967057347298,
        0.05689374357461929,
        0.059983257204294205,
        0.06500593572854996,
        0.06660757958889008,
        0.07087047398090363,
        0.07236418128013611,
        0.07611831277608871,
        0.07718922197818756,
        0.0806790366768837,
        0.08160495012998581,
        0.08478696644306183,
        0.08585704118013382,
        0.08861920237541199,
        0.08891960233449936,
        0.0918315127491951,
        0.0918683186173439,
        0.09433750063180923,
        0.09460321068763733,
        0.09700070321559906,
        0.09746499359607697,
        0.09972338378429413,
        0.0994088351726532,
        0.10201811045408249,
        0.10191889107227325,
        0.10424874722957611,
        0.10386203229427338,
        0.10649008303880692,
        0.10544375330209732,
        0.10806933790445328,
        0.10778804868459702,
        0.11109450459480286,
        0.11027765274047852,
        0.11401215940713882,
        0.11356210708618164,
        0.11832662671804428,
        0.11750734597444534,
        0.12293888628482819,
        0.1225726455450058,
        0.12948566675186157,
        0.12988287210464478,
        0.1387748122215271,
        0.13870327174663544,
        0.14864373207092285,
        0.14834089577198029,
        0.16069285571575165,
        0.16013014316558838,
        0.1741981953382492,
        0.1741078794002533,
        0.18964411318302155,
        0.18942901492118835,
        0.20650440454483032,
        0.20662079751491547,
        0.22468066215515137,
        0.2257053554058075,
        0.24238750338554382,
        0.24448354542255402,
        0.2638821303844452,
        0.2678608000278473,
        0.2911032736301422,
        0.30181384086608887,
        0.32429978251457214
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.00010946426482405514,
        0.00010930908320005983,
        0.00012598594184964895,
        0.00012082153261872008,
        0.00011562183499336243,
        0.00011443938274169341,
        0.00011117216490674764,
        0.00010893856233451515,
        0.00010224300058325753,
        9.949541708920151e-05,
        9.518271690467373e-05,
        9.17384895728901e-05,
        8.801270450931042e-05,
        8.554522355552763e-05,
        8.286484808195382e-05,
        8.027214062167332e-05,
        7.707033364567906e-05,
        7.422274211421609e-05,
        7.329088111873716e-05,
        7.053956505842507e-05,
        6.641655636485666e-05,
        6.314766505965963e-05,
        5.928793325438164e-05,
        5.485894507728517e-05,
        5.013645932194777e-05,
        4.561002060654573e-05,
        4.094795440323651e-05,
        3.615552486735396e-05,
        3.135877341264859e-05,
        2.6529152819421142e-05,
        2.2670756152365357e-05,
        2.0441382730496116e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00011446909593360033,
        "first_to_last_ratio": 3.3439937416939496,
        "imbalance_abs_log_ratio": 1.2071658237176983,
        "last_quartile_mean": 3.4231253039251897e-05,
        "mean": 7.668099857482957e-05,
        "normalized": [
          1.427527899460436,
          1.4255041696332367,
          1.642987756956567,
          1.5756384875558933,
          1.5078290207779732,
          1.4924086131979242,
          1.4498006934307157,
          1.4206721920582042,
          1.3333551007878848,
          1.297523753451233,
          1.2412816561300926,
          1.196365348364192,
          1.1477772348442066,
          1.115598716050207,
          1.0806438312235815,
          1.046832228499728,
          1.0050773343864263,
          0.9679417781940518,
          0.955789341308797,
          0.91990931742482,
          0.866140994500009,
          0.8235112509396527,
          0.7731763325503017,
          0.7154177188205337,
          0.6538315913168741,
          0.5948021211804766,
          0.5340039274955088,
          0.47150566032432917,
          0.4089510308351939,
          0.3459677535828191,
          0.29565024678495777,
          0.26657689793317285
        ],
        "population_cv": 0.3996524250074872
      },
      "activation_output_rms_by_block": [
        0.09914769232273102,
        0.13720469176769257,
        0.1984461098909378,
        0.2206047922372818,
        0.22331808507442474,
        0.2258162647485733,
        0.22567619383335114,
        0.2251615822315216,
        0.22496475279331207,
        0.22249442338943481,
        0.21478813886642456,
        0.205490842461586,
        0.19144563376903534,
        0.188800647854805,
        0.18874111771583557,
        0.190367192029953,
        0.19439123570919037,
        0.198538675904274,
        0.20280276238918304,
        0.2084609866142273,
        0.21558508276939392,
        0.21985429525375366,
        0.22511647641658783,
        0.2311476618051529,
        0.24068132042884827,
        0.2547627091407776,
        0.2762547731399536,
        0.30550503730773926,
        0.34050679206848145,
        0.39206719398498535,
        0.46342816948890686,
        0.575099527835846
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.19442192651331425,
        "first_to_last_ratio": 0.5460704265504183,
        "imbalance_abs_log_ratio": 0.6050073251991793,
        "last_quartile_mean": 0.3560381904244423,
        "mean": 0.2414584644138813,
        "normalized": [
          0.41062007316000754,
          0.5682331000519887,
          0.8218643747803495,
          0.9136345365766327,
          0.9248716362729685,
          0.9352178450099977,
          0.9346377414482436,
          0.932506477990246,
          0.9316913090596918,
          0.9214604421904198,
          0.8895448721907656,
          0.851040128000468,
          0.7928719096004871,
          0.7819177029602238,
          0.7816711589464782,
          0.7884055441669947,
          0.8050711172253067,
          0.8222477368362661,
          0.8399074469452479,
          0.8633409771748842,
          0.8928454146045669,
          0.9105263540353832,
          0.9323196723007322,
          0.9572978208332561,
          0.9967814589274413,
          1.0550995168431603,
          1.144108879390653,
          1.2652488205345183,
          1.410208554481745,
          1.6237459098263265,
          1.9192873217919157,
          2.381774145842633
        ],
        "population_cv": 0.36935148393532197
      },
      "bits_per_byte": 2.073311222061611,
      "branch_output_rms_by_sublayer": [
        0.01824937015771866,
        0.06101369485259056,
        0.05977629870176315,
        0.0147639699280262,
        0.08494830876588821,
        0.003444893751293421,
        0.03928658738732338,
        0.004586480092257261,
        0.025128526613116264,
        0.007272258400917053,
        0.024004649370908737,
        0.006204464007169008,
        0.019449269399046898,
        0.007294370327144861,
        0.02213231660425663,
        0.010247048921883106,
        0.041971735656261444,
        0.012659681960940361,
        0.02208700217306614,
        0.013693951070308685,
        0.029153602197766304,
        0.01590428315103054,
        0.02919897623360157,
        0.017466455698013306,
        0.036670926958322525,
        0.021206999197602272,
        0.02140921540558338,
        0.021084947511553764,
        0.023707622662186623,
        0.022131068632006645,
        0.02047598734498024,
        0.022354252636432648,
        0.022473404183983803,
        0.02369093894958496,
        0.02454194240272045,
        0.02377566508948803,
        0.020227128639817238,
        0.025439493358135223,
        0.02723041921854019,
        0.027927791699767113,
        0.03427847847342491,
        0.028011852875351906,
        0.03502028062939644,
        0.030455492436885834,
        0.029505647718906403,
        0.032503895461559296,
        0.033919259905815125,
        0.03810396417975426,
        0.036724433302879333,
        0.04227631166577339,
        0.033869534730911255,
        0.04830062761902809,
        0.021475639194250107,
        0.057604189962148666,
        0.010480014607310295,
        0.06417365372180939,
        0.010398799553513527,
        0.06654922664165497,
        0.012935257516801357,
        0.0871838852763176,
        0.010905168019235134,
        0.11424379050731659,
        0.02395492233335972,
        0.16713054478168488
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00035888588058483766,
        0.00018018950310225553,
        0.0002078431087578172,
        0.0001449691382379752,
        0.00012177099939920197,
        0.00012306582379429542,
        0.00012204348242118167,
        0.00015411618529826362,
        0.00023712554584861813,
        0.0001667297606409996,
        0.00020124714337423614,
        0.00019657670547036952,
        0.00021589641757185544,
        0.00019691649893044806,
        0.0002141921139315525,
        0.00021492094770207205,
        0.00022724548428173296,
        0.00022961612017464663,
        0.00022356249125846063,
        0.00024496232741861765,
        0.0002428330999302831,
        0.0002472582804061272,
        0.00023198586071939664,
        0.00023918771709558322,
        0.0002286007324212131,
        0.00020791382420134994,
        0.00018978221308207498,
        0.00017514362262291224,
        0.0001585770655414503,
        0.0001491152323589239,
        0.00013960034115182495,
        0.00014044566543596396
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00017661051519947854,
        "first_to_last_ratio": 1.0170643451662862,
        "imbalance_abs_log_ratio": 0.016920384648356147,
        "last_quartile_mean": 0.00017364733710196415,
        "mean": 0.0001978849791614544,
        "normalized": [
          1.813608501795493,
          0.9105769617573595,
          1.0503228170150194,
          0.7325929378383735,
          0.6153625197587583,
          0.6219058380064613,
          0.6167394965416064,
          0.7788169973857398,
          1.1982998752782916,
          0.8425589519099616,
          1.0169904973436037,
          0.9933887165330652,
          1.0910197352357174,
          0.9951058426207471,
          1.0824071379202214,
          1.0860902561316592,
          1.1483715704177997,
          1.160351438232726,
          1.1297597837178734,
          1.2379025859196358,
          1.227142661152444,
          1.2495050481036718,
          1.1723267814588358,
          1.2087209353087376,
          1.155220236472308,
          1.0506801733127658,
          0.959053152423619,
          0.8850779041698386,
          0.8013597909928638,
          0.7535449784556955,
          0.7054620403397319,
          0.7097338364493765
        ],
        "population_cv": 0.24613765810690832
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.0564301572740078,
        0.061963923275470734,
        0.09914769232273102,
        0.12944035232067108,
        0.13720469176769257,
        0.1976913958787918,
        0.1984461098909378,
        0.21992529928684235,
        0.2206047922372818,
        0.22257371246814728,
        0.22331808507442474,
        0.22564378380775452,
        0.2258162647485733,
        0.2261907458305359,
        0.22567619383335114,
        0.22598126530647278,
        0.2251615822315216,
        0.22548764944076538,
        0.22496475279331207,
        0.22340621054172516,
        0.22249442338943481,
        0.21654370427131653,
        0.21478813886642456,
        0.20714853703975677,
        0.205490842461586,
        0.19201314449310303,
        0.19144563376903534,
        0.18731488287448883,
        0.188800647854805,
        0.18594272434711456,
        0.18874111771583557,
        0.18583883345127106,
        0.190367192029953,
        0.18933312594890594,
        0.19439123570919037,
        0.1937587559223175,
        0.198538675904274,
        0.19723950326442719,
        0.20280276238918304,
        0.2025505006313324,
        0.2084609866142273,
        0.2101445198059082,
        0.21558508276939392,
        0.21427761018276215,
        0.21985429525375366,
        0.21906179189682007,
        0.22511647641658783,
        0.22344982624053955,
        0.2311476618051529,
        0.2303154021501541,
        0.24068132042884827,
        0.23970884084701538,
        0.2547627091407776,
        0.25386181473731995,
        0.2762547731399536,
        0.2756711542606354,
        0.30550503730773926,
        0.30545076727867126,
        0.34050679206848145,
        0.34091928601264954,
        0.39206719398498535,
        0.3925974369049072,
        0.46342816948890686,
        0.46681874990463257
      ],
      "loss_nats": 1.4371098279953003,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 2000,
      "stream_state_rms_by_sublayer": [
        0.061963923275470734,
        0.09914769232273102,
        0.12944035232067108,
        0.13720469176769257,
        0.1976913958787918,
        0.1984461098909378,
        0.21992529928684235,
        0.2206047922372818,
        0.22257371246814728,
        0.22331808507442474,
        0.22564378380775452,
        0.2258162647485733,
        0.2261907458305359,
        0.22567619383335114,
        0.22598126530647278,
        0.2251615822315216,
        0.22548764944076538,
        0.22496475279331207,
        0.22340621054172516,
        0.22249442338943481,
        0.21654370427131653,
        0.21478813886642456,
        0.20714853703975677,
        0.205490842461586,
        0.19201314449310303,
        0.19144563376903534,
        0.18731488287448883,
        0.188800647854805,
        0.18594272434711456,
        0.18874111771583557,
        0.18583883345127106,
        0.190367192029953,
        0.18933312594890594,
        0.19439123570919037,
        0.1937587559223175,
        0.198538675904274,
        0.19723950326442719,
        0.20280276238918304,
        0.2025505006313324,
        0.2084609866142273,
        0.2101445198059082,
        0.21558508276939392,
        0.21427761018276215,
        0.21985429525375366,
        0.21906179189682007,
        0.22511647641658783,
        0.22344982624053955,
        0.2311476618051529,
        0.2303154021501541,
        0.24068132042884827,
        0.23970884084701538,
        0.2547627091407776,
        0.25386181473731995,
        0.2762547731399536,
        0.2756711542606354,
        0.30550503730773926,
        0.30545076727867126,
        0.34050679206848145,
        0.34091928601264954,
        0.39206719398498535,
        0.3925974369049072,
        0.46342816948890686,
        0.46681874990463257,
        0.575099527835846
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.00010021305934060365,
        0.00010125436529051512,
        0.000114263420982752,
        0.00011024689592886716,
        0.00010690757335396484,
        0.00010680041305022314,
        0.00010475025192135945,
        0.00010345032933400944,
        9.713075269246474e-05,
        9.490915545029566e-05,
        9.159108594758436e-05,
        8.836394408717752e-05,
        8.501022239215672e-05,
        8.242073090514168e-05,
        7.99787012510933e-05,
        7.76422893977724e-05,
        7.462384382961318e-05,
        7.162441761465743e-05,
        7.150838064262643e-05,
        6.863063026685268e-05,
        6.474268593592569e-05,
        6.149821274448186e-05,
        5.7751010899664834e-05,
        5.37736268597655e-05,
        4.920278661302291e-05,
        4.476245885598473e-05,
        4.037534017697908e-05,
        3.581849159672856e-05,
        3.1086256058188155e-05,
        2.5875511710182764e-05,
        2.1458168703247793e-05,
        1.863625948317349e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00010598578865028685,
        "first_to_last_ratio": 3.1730458332581706,
        "imbalance_abs_log_ratio": 1.1546919572676668,
        "last_quartile_mean": 3.3401909149688436e-05,
        "mean": 7.300941479115863e-05,
        "normalized": [
          1.3726046103404645,
          1.3868672359606,
          1.565050498070684,
          1.5100367020366523,
          1.4642984560247596,
          1.4628306959550723,
          1.434749918500163,
          1.4169450560578547,
          1.330386676434875,
          1.2999577619103044,
          1.2545106163304838,
          1.2103088942698703,
          1.1643734254729483,
          1.12890551363689,
          1.0954573664214418,
          1.0634558518222064,
          1.0221126144220245,
          0.9810298825094963,
          0.9794405399245307,
          0.9400243853915095,
          0.8867717419886232,
          0.8423326350498187,
          0.7910077222897892,
          0.7365300353876748,
          0.6739238597346121,
          0.613105295858438,
          0.5530155294693376,
          0.4906009957645373,
          0.425784210804997,
          0.35441335592401246,
          0.29390961103616947,
          0.255258305199158
        ],
        "population_cv": 0.383084243994172
      },
      "activation_output_rms_by_block": [
        0.11517385393381119,
        0.17733007669448853,
        0.24607165157794952,
        0.2600470185279846,
        0.2621006965637207,
        0.26366734504699707,
        0.2635972201824188,
        0.26321178674697876,
        0.2646143436431885,
        0.2614729404449463,
        0.2522071599960327,
        0.23732660710811615,
        0.2133668214082718,
        0.2109174132347107,
        0.21198545396327972,
        0.21621736884117126,
        0.2226637899875641,
        0.22916270792484283,
        0.23635683953762054,
        0.24468488991260529,
        0.25434690713882446,
        0.26204079389572144,
        0.2696947753429413,
        0.27905911207199097,
        0.29096299409866333,
        0.30846187472343445,
        0.3323265314102173,
        0.36396095156669617,
        0.403833270072937,
        0.4673614203929901,
        0.5606572031974792,
        0.7190070152282715
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.23139995615929365,
        "first_to_last_ratio": 0.5371134119256165,
        "imbalance_abs_log_ratio": 0.6215460113796702,
        "last_quartile_mean": 0.43082140758633614,
        "mean": 0.28637152607552707,
        "normalized": [
          0.4021833298588333,
          0.6192308262090269,
          0.859274156722729,
          0.9080756808880515,
          0.9152470573998156,
          0.9207177426482616,
          0.92047286891539,
          0.9191269479688416,
          0.9240246307637429,
          0.9130549535709983,
          0.8806991513867063,
          0.8287367475407597,
          0.7450699597557016,
          0.7365167065495323,
          0.740246269831209,
          0.7550239781316335,
          0.7775346698709116,
          0.8002286786864554,
          0.8253503509119278,
          0.8544316303572463,
          0.8881710784044343,
          0.9150378792429639,
          0.9417653320456605,
          0.9744652895357776,
          1.0160332561203216,
          1.0771387747610122,
          1.1604733751447416,
          1.2709397353656795,
          1.410172567109311,
          1.6320107896122644,
          1.9577966108600218,
          2.510748973830038
        ],
        "population_cv": 0.3918463566300126
      },
      "bits_per_byte": 1.9262875534052675,
      "branch_output_rms_by_sublayer": [
        0.021954918280243874,
        0.07429475337266922,
        0.09672636538743973,
        0.017902610823512077,
        0.0940650999546051,
        0.0039193108677864075,
        0.036259762942790985,
        0.004245741758495569,
        0.02603325992822647,
        0.008932216092944145,
        0.02122427336871624,
        0.005480592139065266,
        0.02764410711824894,
        0.00817549042403698,
        0.02432539500296116,
        0.009956736117601395,
        0.05137033388018608,
        0.014394877478480339,
        0.02661585807800293,
        0.015817945823073387,
        0.030927525833249092,
        0.018779242411255836,
        0.03579767048358917,
        0.02186598815023899,
        0.05378979817032814,
        0.028890088200569153,
        0.025482553988695145,
        0.027309151366353035,
        0.0287014190107584,
        0.028864959254860878,
        0.027393119409680367,
        0.02912190742790699,
        0.029581313952803612,
        0.03136448562145233,
        0.0320778414607048,
        0.03278070688247681,
        0.028201179578900337,
        0.033851705491542816,
        0.035640619695186615,
        0.03695352375507355,
        0.04368399456143379,
        0.038013845682144165,
        0.04277802258729935,
        0.04267064854502678,
        0.03937495872378349,
        0.04597146809101105,
        0.04081331565976143,
        0.053922027349472046,
        0.05011079087853432,
        0.0602298267185688,
        0.04528728500008583,
        0.06855001300573349,
        0.028651293367147446,
        0.07740508019924164,
        0.014390980824828148,
        0.08345231413841248,
        0.015424116514623165,
        0.08889877796173096,
        0.01913117803633213,
        0.11831489205360413,
        0.01693635806441307,
        0.153398334980011,
        0.032383937388658524,
        0.24250756204128265
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0003320807822012488,
        0.00017565239824668803,
        0.00017577232236880354,
        0.00012480421087066412,
        0.00013414970937401124,
        0.0001166490159724099,
        0.00011491447201106907,
        0.00015823331904477844,
        0.000249176250949333,
        0.0001712841712530518,
        0.00020010962025834788,
        0.00021729074089037623,
        0.0002489951018894741,
        0.00022387049947650086,
        0.00024825867897268754,
        0.000256508706958601,
        0.0002733879022225402,
        0.00028045470195987973,
        0.0002710271146389,
        0.0002966387758490912,
        0.00030658370753967526,
        0.0003196905817287839,
        0.0002949547776417074,
        0.00030211663360406435,
        0.0002930543360663917,
        0.00026176751657805116,
        0.0002415189744957938,
        0.00021641616631898222,
        0.00019981422740886822,
        0.0001881612684369175,
        0.00017039642606795488,
        0.0001626151127852574
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00016653202876120915,
        "first_to_last_ratio": 0.7684272928714567,
        "imbalance_abs_log_ratio": 0.2634093296250088,
        "last_quartile_mean": 0.0002167180035197771,
        "mean": 0.00022582338200252827,
        "normalized": [
          1.4705332072191308,
          0.7778308724679426,
          0.7783619251917662,
          0.5526629251760424,
          0.5940470299594988,
          0.5165497697271398,
          0.5088687937982548,
          0.7006950194511165,
          1.103412094618897,
          0.758487317540635,
          0.8861333068517568,
          0.9622154223513644,
          1.1026099232128512,
          0.991352168634134,
          1.0993488662299287,
          1.1358819652950252,
          1.2106270829806252,
          1.2419205640837485,
          1.2001729503628886,
          1.313587517902686,
          1.3576260563498372,
          1.4156664331827458,
          1.306130370673508,
          1.3378447834984684,
          1.2977147603923083,
          1.159169233304284,
          1.069503841250105,
          0.9583425967668807,
          0.8848252365941058,
          0.8332231444253673,
          0.7545561693254915,
          0.7200986511814654
        ],
        "population_cv": 0.2750960622596733
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.06031418219208717,
        0.06744465231895447,
        0.11517385393381119,
        0.16882504522800446,
        0.17733007669448853,
        0.2454802393913269,
        0.24607165157794952,
        0.25892511010169983,
        0.2600470185279846,
        0.2610686123371124,
        0.2621006965637207,
        0.2635994553565979,
        0.26366734504699707,
        0.26440703868865967,
        0.2635972201824188,
        0.2638950049877167,
        0.26321178674697876,
        0.2655138075351715,
        0.2646143436431885,
        0.2625099718570709,
        0.2614729404449463,
        0.25542622804641724,
        0.2522071599960327,
        0.24079175293445587,
        0.23732660710811615,
        0.21542933583259583,
        0.2133668214082718,
        0.20939168334007263,
        0.2109174132347107,
        0.20824690163135529,
        0.21198545396327972,
        0.21042729914188385,
        0.21621736884117126,
        0.21581067144870758,
        0.2226637899875641,
        0.2222982496023178,
        0.22916270792484283,
        0.22995305061340332,
        0.23635683953762054,
        0.23728354275226593,
        0.24468488991260529,
        0.24780811369419098,
        0.25434690713882446,
        0.25493237376213074,
        0.26204079389572144,
        0.26210466027259827,
        0.2696947753429413,
        0.2699442505836487,
        0.27905911207199097,
        0.27968016266822815,
        0.29096299409866333,
        0.29178300499916077,
        0.30846187472343445,
        0.30848607420921326,
        0.3323265314102173,
        0.33218035101890564,
        0.36396095156669617,
        0.3638138175010681,
        0.403833270072937,
        0.40415704250335693,
        0.4673614203929901,
        0.4678023159503937,
        0.5606572031974792,
        0.5632834434509277
      ],
      "loss_nats": 1.3352007865905762,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 4000,
      "stream_state_rms_by_sublayer": [
        0.06744465231895447,
        0.11517385393381119,
        0.16882504522800446,
        0.17733007669448853,
        0.2454802393913269,
        0.24607165157794952,
        0.25892511010169983,
        0.2600470185279846,
        0.2610686123371124,
        0.2621006965637207,
        0.2635994553565979,
        0.26366734504699707,
        0.26440703868865967,
        0.2635972201824188,
        0.2638950049877167,
        0.26321178674697876,
        0.2655138075351715,
        0.2646143436431885,
        0.2625099718570709,
        0.2614729404449463,
        0.25542622804641724,
        0.2522071599960327,
        0.24079175293445587,
        0.23732660710811615,
        0.21542933583259583,
        0.2133668214082718,
        0.20939168334007263,
        0.2109174132347107,
        0.20824690163135529,
        0.21198545396327972,
        0.21042729914188385,
        0.21621736884117126,
        0.21581067144870758,
        0.2226637899875641,
        0.2222982496023178,
        0.22916270792484283,
        0.22995305061340332,
        0.23635683953762054,
        0.23728354275226593,
        0.24468488991260529,
        0.24780811369419098,
        0.25434690713882446,
        0.25493237376213074,
        0.26204079389572144,
        0.26210466027259827,
        0.2696947753429413,
        0.2699442505836487,
        0.27905911207199097,
        0.27968016266822815,
        0.29096299409866333,
        0.29178300499916077,
        0.30846187472343445,
        0.30848607420921326,
        0.3323265314102173,
        0.33218035101890564,
        0.36396095156669617,
        0.3638138175010681,
        0.403833270072937,
        0.40415704250335693,
        0.4673614203929901,
        0.4678023159503937,
        0.5606572031974792,
        0.5632834434509277,
        0.7190070152282715
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.00018638069741427898,
        0.00018956589337904006,
        0.00020923581905663013,
        0.00020349136320874095,
        0.00019885810615960509,
        0.0001991111203096807,
        0.00019671664631459862,
        0.00019502552459016442,
        0.00018589819956105202,
        0.00018270809960085899,
        0.00017816871695686132,
        0.00017320679035037756,
        0.0001670292258495465,
        0.00016241244156844914,
        0.00015707056445535272,
        0.0001517482742201537,
        0.0001448393304599449,
        0.00013789460354018956,
        0.00013790397497359663,
        0.00013200659304857254,
        0.00012367918679956347,
        0.00011618347343755886,
        0.00010830583050847054,
        9.980145841836929e-05,
        8.979889389593154e-05,
        8.034214988583699e-05,
        7.120905502233654e-05,
        6.192603905219585e-05,
        5.214559132582508e-05,
        4.129187436774373e-05,
        3.1231873435899615e-05,
        2.3505688659497537e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00019729814630409237,
        "first_to_last_ratio": 3.49624785701179,
        "imbalance_abs_log_ratio": 1.2516903525947785,
        "last_quartile_mean": 5.643139570565836e-05,
        "mean": 0.00013714665936959136,
        "normalized": [
          1.3589882412812453,
          1.3822129846282736,
          1.525635549698432,
          1.4837500537316024,
          1.449966824373825,
          1.4518116680706283,
          1.434352445905914,
          1.4220216918634343,
          1.3554701253063843,
          1.3322096246506874,
          1.2991108771867432,
          1.2629311654147486,
          1.217887673074309,
          1.1842245543201309,
          1.1452744469121132,
          1.1064671565292232,
          1.0560908382727927,
          1.0054536083783316,
          1.0055219398524642,
          0.9625213888209478,
          0.9018024016630627,
          0.8471476736772742,
          0.7897081198062669,
          0.7276987924978771,
          0.6547654482340389,
          0.5858119348669365,
          0.519218297767195,
          0.451531516238494,
          0.38021772871113024,
          0.3010782366668357,
          0.22772609686200243,
          0.17139089473665517
        ],
        "population_cv": 0.4031953558435787
      },
      "activation_output_rms_by_block": [
        0.1026567816734314,
        0.17673343420028687,
        0.23246681690216064,
        0.24062344431877136,
        0.2416554093360901,
        0.24258241057395935,
        0.24218299984931946,
        0.24160927534103394,
        0.24130181968212128,
        0.23599836230278015,
        0.22310413420200348,
        0.20329499244689941,
        0.176778644323349,
        0.17310521006584167,
        0.1729687601327896,
        0.17606647312641144,
        0.1812635064125061,
        0.18683874607086182,
        0.19253391027450562,
        0.1985689103603363,
        0.20644955337047577,
        0.2135031372308731,
        0.22015690803527832,
        0.228074312210083,
        0.2383035272359848,
        0.2532499134540558,
        0.27322494983673096,
        0.2992607355117798,
        0.3339390158653259,
        0.3918353319168091,
        0.48234984278678894,
        0.651733934879303
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.21506382152438164,
        "first_to_last_ratio": 0.5884305856918151,
        "imbalance_abs_log_ratio": 0.5302963104844547,
        "last_quartile_mean": 0.3654871564358473,
        "mean": 0.2460754751227796,
        "normalized": [
          0.41717599700746566,
          0.7182082412402355,
          0.9446972185512233,
          0.9778440708027163,
          0.9820377638832796,
          0.9858049058034841,
          0.9841817829610285,
          0.9818502848383519,
          0.9806008484255633,
          0.9590486909962422,
          0.906649206267652,
          0.8261489380258848,
          0.7183919658598449,
          0.7034638863522286,
          0.7029093819548116,
          0.7154978489366439,
          0.7366175207913933,
          0.7592741453720182,
          0.7824181185810598,
          0.8069431147547724,
          0.838968423275288,
          0.8676327339176954,
          0.8946722867259763,
          0.9268469850410127,
          0.9684164060521798,
          1.0291554382967116,
          1.1103298681041054,
          1.2161339335521484,
          1.357059315637638,
          1.5923380081712835,
          1.9601703198830358,
          2.648512349937025
        ],
        "population_cv": 0.40510862299171807
      },
      "bits_per_byte": 1.8611962476373152,
      "branch_output_rms_by_sublayer": [
        0.026356825605034828,
        0.057217150926589966,
        0.10627161711454391,
        0.01789308711886406,
        0.07707435637712479,
        0.0034156402107328176,
        0.02677960880100727,
        0.0038151287008076906,
        0.02153952419757843,
        0.007133069448173046,
        0.01632758229970932,
        0.004304513800889254,
        0.01752857305109501,
        0.006448030937463045,
        0.01845136471092701,
        0.007711905520409346,
        0.044129498302936554,
        0.011710400693118572,
        0.023254960775375366,
        0.01295715756714344,
        0.02913394197821617,
        0.01598021760582924,
        0.03491654247045517,
        0.01944364421069622,
        0.04857943207025528,
        0.02654583752155304,
        0.023438163101673126,
        0.02489112876355648,
        0.025785095989704132,
        0.0264157485216856,
        0.025251252576708794,
        0.02720639854669571,
        0.027512386441230774,
        0.029633820056915283,
        0.02885596640408039,
        0.031438712030649185,
        0.0244715828448534,
        0.03218274191021919,
        0.031743232160806656,
        0.03524785488843918,
        0.038931526243686676,
        0.037403788417577744,
        0.03771914169192314,
        0.041983578354120255,
        0.03549705073237419,
        0.04549537971615791,
        0.0359940230846405,
        0.052616480737924576,
        0.04251093044877052,
        0.059319522231817245,
        0.03818616643548012,
        0.06705058366060257,
        0.024311494082212448,
        0.07359735667705536,
        0.01473314967006445,
        0.07876058667898178,
        0.015392928384244442,
        0.0848938599228859,
        0.018577447161078453,
        0.11232978105545044,
        0.018359949812293053,
        0.1471690982580185,
        0.03667483106255531,
        0.24789714813232422
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0005542142737822822,
        0.0003797328422683109,
        0.000340619215402376,
        0.0001793765938241596,
        0.00020684039436717747,
        0.00016374338231517116,
        0.00018705111879548875,
        0.0002256278252430331,
        0.000426778694633347,
        0.00027155262856832466,
        0.0003392540980461625,
        0.0003897844238340517,
        0.00045204461946776503,
        0.0004208641344247883,
        0.00047738343422760874,
        0.00048450285457785075,
        0.0005178591522769467,
        0.0005253914687520978,
        0.0005086607978429822,
        0.0005610796431247281,
        0.0005682688177114847,
        0.000574059459111366,
        0.0005453911942509562,
        0.0005508569019095279,
        0.0005214428582378581,
        0.000461966711188474,
        0.00041497592078977314,
        0.00037256322877604213,
        0.00033686502618798456,
        0.00030674152201740854,
        0.00025927963978097604,
        0.00021950976285567438
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.0002796507057497499,
        "first_to_last_ratio": 0.7732247282264533,
        "imbalance_abs_log_ratio": 0.25718555049038655,
        "last_quartile_mean": 0.00036166808372927383,
        "mean": 0.00039825883245600557,
        "normalized": [
          1.3915931766397287,
          0.9534825377922003,
          0.8552709636138532,
          0.4504020481302817,
          0.5193617253674505,
          0.41114815032572916,
          0.46967224214958675,
          0.5665356468094339,
          1.0716113739435822,
          0.681849607436697,
          0.8518432496626144,
          0.9787213542266133,
          1.1350523394046785,
          1.056760328024313,
          1.1986763263570401,
          1.2165526916000595,
          1.3003080159789122,
          1.319221134436827,
          1.2772115930389878,
          1.4088316376177517,
          1.4268831508570738,
          1.441423045337684,
          1.3694390426638032,
          1.383163051306287,
          1.3093064503358134,
          1.159966015918821,
          1.0419754365036318,
          0.9354801410893958,
          0.8458444577627716,
          0.7702064512311685,
          0.6510329932472191,
          0.5511736211899907
        ],
        "population_cv": 0.3252608102570666
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.061885204166173935,
        0.07089564949274063,
        0.1026567816734314,
        0.16996370255947113,
        0.17673343420028687,
        0.23203739523887634,
        0.23246681690216064,
        0.23961465060710907,
        0.24062344431877136,
        0.24108263850212097,
        0.2416554093360901,
        0.24260269105434418,
        0.24258241057395935,
        0.2429492026567459,
        0.24218299984931946,
        0.24250134825706482,
        0.24160927534103394,
        0.242857426404953,
        0.24130181968212128,
        0.23791606724262238,
        0.23599836230278015,
        0.22686637938022614,
        0.22310413420200348,
        0.20835906267166138,
        0.20329499244689941,
        0.1809774935245514,
        0.176778644323349,
        0.17327234148979187,
        0.17310521006584167,
        0.17050857841968536,
        0.1729687601327896,
        0.17174123227596283,
        0.17606647312641144,
        0.1756037324666977,
        0.1812635064125061,
        0.1811160296201706,
        0.18683874607086182,
        0.1874236911535263,
        0.19253391027450562,
        0.19280622899532318,
        0.1985689103603363,
        0.20097090303897858,
        0.20644955337047577,
        0.2074177861213684,
        0.2135031372308731,
        0.21357181668281555,
        0.22015690803527832,
        0.22022640705108643,
        0.228074312210083,
        0.22874467074871063,
        0.2383035272359848,
        0.23907209932804108,
        0.2532499134540558,
        0.2534840703010559,
        0.27322494983673096,
        0.27303415536880493,
        0.2992607355117798,
        0.29903557896614075,
        0.3339390158653259,
        0.3341987431049347,
        0.3918353319168091,
        0.3921641707420349,
        0.48234984278678894,
        0.4832552373409271
      ],
      "loss_nats": 1.2900829315185547,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 8000,
      "stream_state_rms_by_sublayer": [
        0.07089564949274063,
        0.1026567816734314,
        0.16996370255947113,
        0.17673343420028687,
        0.23203739523887634,
        0.23246681690216064,
        0.23961465060710907,
        0.24062344431877136,
        0.24108263850212097,
        0.2416554093360901,
        0.24260269105434418,
        0.24258241057395935,
        0.2429492026567459,
        0.24218299984931946,
        0.24250134825706482,
        0.24160927534103394,
        0.242857426404953,
        0.24130181968212128,
        0.23791606724262238,
        0.23599836230278015,
        0.22686637938022614,
        0.22310413420200348,
        0.20835906267166138,
        0.20329499244689941,
        0.1809774935245514,
        0.176778644323349,
        0.17327234148979187,
        0.17310521006584167,
        0.17050857841968536,
        0.1729687601327896,
        0.17174123227596283,
        0.17606647312641144,
        0.1756037324666977,
        0.1812635064125061,
        0.1811160296201706,
        0.18683874607086182,
        0.1874236911535263,
        0.19253391027450562,
        0.19280622899532318,
        0.1985689103603363,
        0.20097090303897858,
        0.20644955337047577,
        0.2074177861213684,
        0.2135031372308731,
        0.21357181668281555,
        0.22015690803527832,
        0.22022640705108643,
        0.228074312210083,
        0.22874467074871063,
        0.2383035272359848,
        0.23907209932804108,
        0.2532499134540558,
        0.2534840703010559,
        0.27322494983673096,
        0.27303415536880493,
        0.2992607355117798,
        0.29903557896614075,
        0.3339390158653259,
        0.3341987431049347,
        0.3918353319168091,
        0.3921641707420349,
        0.48234984278678894,
        0.4832552373409271,
        0.651733934879303
      ]
    }
  ],
  "environment": {
    "autocast": "cuda-bfloat16-forward-fp32-cross-entropy",
    "compile": false,
    "compute_capability": [
      12,
      0
    ],
    "cublas_workspace_config": ":4096:8",
    "cuda": "12.8",
    "deterministic_algorithms": true,
    "gpu": "NVIDIA GeForce RTX 5090",
    "python": "3.10.14",
    "torch": "2.11.0+cu128"
  },
  "evaluations": [
    {
      "bits_per_byte": 8.06581270897968,
      "cross_entropy_nats": 5.590795338153839,
      "step": 0
    },
    {
      "bits_per_byte": 5.193364149696528,
      "cross_entropy_nats": 3.599765717983246,
      "step": 100
    },
    {
      "bits_per_byte": 3.118855971380574,
      "cross_entropy_nats": 2.1618262231349945,
      "step": 500
    },
    {
      "bits_per_byte": 1.9576097217110056,
      "cross_entropy_nats": 1.3569116592407227,
      "step": 2000
    },
    {
      "bits_per_byte": 1.792987261872353,
      "cross_entropy_nats": 1.2428040653467178,
      "step": 4000
    },
    {
      "bits_per_byte": 1.709514213904039,
      "cross_entropy_nats": 1.1849449574947357,
      "step": 8000
    }
  ],
  "gradient_gate": null,
  "hashes": {
    "final_mixer_parameters": null,
    "final_model_state": "411fcebe23a13021e3c3b6a3b03dca473d4b287d53d6c87cd6f8b3e6b7611d22",
    "final_optimizer_state": "e774625d2eec7b16bb949a8bd26f25b05b7288a59bf24a716a88b489cd22bcd6",
    "final_public_parameters": "411fcebe23a13021e3c3b6a3b03dca473d4b287d53d6c87cd6f8b3e6b7611d22",
    "initial_mixer_parameters": null,
    "initial_public_parameter_elements": 18985152,
    "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670",
    "initial_public_parameter_tensors": 227,
    "initial_public_parameters": "3f93c11463780b386fff00c53b0424f687e1136b1b80f7b626f058adef96c764"
  },
  "manifest": {
    "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716",
    "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371",
    "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
    "input_gate_tensor_hashes": {
      "0": "3eff5c07fa0cfbd5ceed5922cbd09525ad514f801cc0a440c03fd2fa15df4c84",
      "1": "e83e4001c5811b19c69a54c7ea7a802af996a34aa57072cde43a76bc8369e2d0",
      "7999": "d04c4ca879f0209183a48ac54a0c4ffe3a0f62a78888479d4641f6c304426442"
    },
    "path": "experiments/k3/attnres_gradient/manifest.json",
    "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338"
  },
  "model": {
    "attnres_aggregation_groups": 8,
    "context": 256,
    "d_ff": 768,
    "d_head": 32,
    "d_model": 192,
    "heads": 6,
    "layers": 32,
    "parameters": {
      "core": 18985152,
      "embedding": 98304,
      "mixer": 0,
      "total": 18985152
    },
    "sublayers": 64,
    "sublayers_per_attnres_group": 8,
    "transformer_blocks_per_attnres_group": 4,
    "vocabulary": 256
  },
  "optimizer": {
    "betas": [
      0.9,
      0.95
    ],
    "epsilon": 1e-08,
    "grad_clip": 1.0,
    "min_lr": 3e-05,
    "name": "AdamW",
    "peak_lr": 0.0003,
    "warmup_steps": 400,
    "weight_decay_ndim_ge_2": 0.1
  },
  "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
  "run_kind": "formal",
  "schema_version": 1,
  "seed": 2026073003,
  "steps": 8000,
  "target_bytes_seen": 65536000,
  "timing": {
    "mean_ms": 42.39461278025134,
    "measured_steps": 7980,
    "median_ms": 42.272048493032344,
    "p95_ms": 43.13050716154976,
    "peak_allocated_bytes": 5938562048,
    "peak_reserved_bytes": 6325010432,
    "warmup_steps_excluded": 20
  },
  "training_history": [
    {
      "bits_per_byte": 8.087939566883062,
      "learning_rate": 7.499999999999999e-07,
      "loss_nats": 5.606132507324219,
      "step": 1,
      "unclipped_grad_norm": 19.928020477294922
    },
    {
      "bits_per_byte": 7.3282798152711734,
      "learning_rate": 7.499999999999999e-06,
      "loss_nats": 5.07957649230957,
      "step": 10,
      "unclipped_grad_norm": 11.731513977050781
    },
    {
      "bits_per_byte": 6.791570117070589,
      "learning_rate": 1.4999999999999999e-05,
      "loss_nats": 4.707557678222656,
      "step": 20,
      "unclipped_grad_norm": 3.350315809249878
    },
    {
      "bits_per_byte": 6.667928349712299,
      "learning_rate": 2.2499999999999998e-05,
      "loss_nats": 4.621855735778809,
      "step": 30,
      "unclipped_grad_norm": 3.0359153747558594
    },
    {
      "bits_per_byte": 6.473404964696806,
      "learning_rate": 2.9999999999999997e-05,
      "loss_nats": 4.487022399902344,
      "step": 40,
      "unclipped_grad_norm": 3.083479404449463
    },
    {
      "bits_per_byte": 6.159992036931666,
      "learning_rate": 3.75e-05,
      "loss_nats": 4.269781112670898,
      "step": 50,
      "unclipped_grad_norm": 2.770655393600464
    },
    {
      "bits_per_byte": 5.940455495174687,
      "learning_rate": 4.4999999999999996e-05,
      "loss_nats": 4.117609977722168,
      "step": 60,
      "unclipped_grad_norm": 2.5750629901885986
    },
    {
      "bits_per_byte": 5.762018679390299,
      "learning_rate": 5.2499999999999995e-05,
      "loss_nats": 3.993927001953125,
      "step": 70,
      "unclipped_grad_norm": 2.5019538402557373
    },
    {
      "bits_per_byte": 5.5944951946990615,
      "learning_rate": 5.9999999999999995e-05,
      "loss_nats": 3.8778085708618164,
      "step": 80,
      "unclipped_grad_norm": 2.5689163208007812
    },
    {
      "bits_per_byte": 5.40584020169631,
      "learning_rate": 6.749999999999999e-05,
      "loss_nats": 3.7470428943634033,
      "step": 90,
      "unclipped_grad_norm": 2.3143606185913086
    },
    {
      "bits_per_byte": 5.179311877251155,
      "learning_rate": 7.5e-05,
      "loss_nats": 3.5900254249572754,
      "step": 100,
      "unclipped_grad_norm": 2.227365255355835
    },
    {
      "bits_per_byte": 4.996660110659421,
      "learning_rate": 8.249999999999999e-05,
      "loss_nats": 3.463420867919922,
      "step": 110,
      "unclipped_grad_norm": 2.0371005535125732
    },
    {
      "bits_per_byte": 4.800645980026251,
      "learning_rate": 8.999999999999999e-05,
      "loss_nats": 3.327554225921631,
      "step": 120,
      "unclipped_grad_norm": 1.8128844499588013
    },
    {
      "bits_per_byte": 4.634309863556569,
      "learning_rate": 9.75e-05,
      "loss_nats": 3.212258815765381,
      "step": 130,
      "unclipped_grad_norm": 1.5838756561279297
    },
    {
      "bits_per_byte": 4.478764626532625,
      "learning_rate": 0.00010499999999999999,
      "loss_nats": 3.104443073272705,
      "step": 140,
      "unclipped_grad_norm": 1.4334924221038818
    },
    {
      "bits_per_byte": 4.278149959988874,
      "learning_rate": 0.0001125,
      "loss_nats": 2.9653875827789307,
      "step": 150,
      "unclipped_grad_norm": 1.2058515548706055
    },
    {
      "bits_per_byte": 4.130972366860008,
      "learning_rate": 0.00011999999999999999,
      "loss_nats": 2.8633718490600586,
      "step": 160,
      "unclipped_grad_norm": 1.057314395904541
    },
    {
      "bits_per_byte": 4.025422830407184,
      "learning_rate": 0.00012749999999999998,
      "loss_nats": 2.790210485458374,
      "step": 170,
      "unclipped_grad_norm": 0.9353183507919312
    },
    {
      "bits_per_byte": 3.9235808959416594,
      "learning_rate": 0.00013499999999999997,
      "loss_nats": 2.719619035720825,
      "step": 180,
      "unclipped_grad_norm": 0.8658975958824158
    },
    {
      "bits_per_byte": 3.837114554428572,
      "learning_rate": 0.0001425,
      "loss_nats": 2.6596851348876953,
      "step": 190,
      "unclipped_grad_norm": 0.8723611235618591
    },
    {
      "bits_per_byte": 3.771249326731737,
      "learning_rate": 0.00015,
      "loss_nats": 2.6140308380126953,
      "step": 200,
      "unclipped_grad_norm": 0.9528701305389404
    },
    {
      "bits_per_byte": 3.687100279456375,
      "learning_rate": 0.0001575,
      "loss_nats": 2.5557031631469727,
      "step": 210,
      "unclipped_grad_norm": 0.7721163034439087
    },
    {
      "bits_per_byte": 3.664338719579984,
      "learning_rate": 0.00016499999999999997,
      "loss_nats": 2.539926052093506,
      "step": 220,
      "unclipped_grad_norm": 1.0300326347351074
    },
    {
      "bits_per_byte": 3.616636923606072,
      "learning_rate": 0.0001725,
      "loss_nats": 2.506861686706543,
      "step": 230,
      "unclipped_grad_norm": 1.116415023803711
    },
    {
      "bits_per_byte": 3.58821025521026,
      "learning_rate": 0.00017999999999999998,
      "loss_nats": 2.4871578216552734,
      "step": 240,
      "unclipped_grad_norm": 0.7745823860168457
    },
    {
      "bits_per_byte": 3.5119345736913976,
      "learning_rate": 0.0001875,
      "loss_nats": 2.4342875480651855,
      "step": 250,
      "unclipped_grad_norm": 0.9986598491668701
    },
    {
      "bits_per_byte": 3.522710662629159,
      "learning_rate": 0.000195,
      "loss_nats": 2.4417569637298584,
      "step": 260,
      "unclipped_grad_norm": 0.8885983228683472
    },
    {
      "bits_per_byte": 3.5021329384534243,
      "learning_rate": 0.00020249999999999996,
      "loss_nats": 2.4274935722351074,
      "step": 270,
      "unclipped_grad_norm": 0.7707400321960449
    },
    {
      "bits_per_byte": 3.4629779922967905,
      "learning_rate": 0.00020999999999999998,
      "loss_nats": 2.40035343170166,
      "step": 280,
      "unclipped_grad_norm": 1.055368423461914
    },
    {
      "bits_per_byte": 3.4474741002868097,
      "learning_rate": 0.00021749999999999997,
      "loss_nats": 2.3896069526672363,
      "step": 290,
      "unclipped_grad_norm": 1.053453803062439
    },
    {
      "bits_per_byte": 3.4491093113312137,
      "learning_rate": 0.000225,
      "loss_nats": 2.390740394592285,
      "step": 300,
      "unclipped_grad_norm": 1.2985762357711792
    },
    {
      "bits_per_byte": 3.334430591805289,
      "learning_rate": 0.00023249999999999996,
      "loss_nats": 2.311251163482666,
      "step": 310,
      "unclipped_grad_norm": 0.8886587619781494
    },
    {
      "bits_per_byte": 3.4024717440008403,
      "learning_rate": 0.00023999999999999998,
      "loss_nats": 2.3584136962890625,
      "step": 320,
      "unclipped_grad_norm": 1.0081987380981445
    },
    {
      "bits_per_byte": 3.413793361907139,
      "learning_rate": 0.0002475,
      "loss_nats": 2.3662612438201904,
      "step": 330,
      "unclipped_grad_norm": 0.8511857986450195
    },
    {
      "bits_per_byte": 3.3420762525320464,
      "learning_rate": 0.00025499999999999996,
      "loss_nats": 2.3165507316589355,
      "step": 340,
      "unclipped_grad_norm": 0.9547706842422485
    },
    {
      "bits_per_byte": 3.3350242759160675,
      "learning_rate": 0.0002625,
      "loss_nats": 2.3116626739501953,
      "step": 350,
      "unclipped_grad_norm": 1.1394811868667603
    },
    {
      "bits_per_byte": 3.3268671387856448,
      "learning_rate": 0.00026999999999999995,
      "loss_nats": 2.3060085773468018,
      "step": 360,
      "unclipped_grad_norm": 1.0605591535568237
    },
    {
      "bits_per_byte": 3.320889365806955,
      "learning_rate": 0.00027749999999999997,
      "loss_nats": 2.3018651008605957,
      "step": 370,
      "unclipped_grad_norm": 0.8489253520965576
    },
    {
      "bits_per_byte": 3.2193453052927254,
      "learning_rate": 0.000285,
      "loss_nats": 2.231480121612549,
      "step": 380,
      "unclipped_grad_norm": 0.923150897026062
    },
    {
      "bits_per_byte": 3.297119987596508,
      "learning_rate": 0.0002925,
      "loss_nats": 2.2853894233703613,
      "step": 390,
      "unclipped_grad_norm": 0.7830708026885986
    },
    {
      "bits_per_byte": 3.2808776020837227,
      "learning_rate": 0.0003,
      "loss_nats": 2.2741310596466064,
      "step": 400,
      "unclipped_grad_norm": 0.7351507544517517
    },
    {
      "bits_per_byte": 3.2985058237971607,
      "learning_rate": 0.0002999988466112388,
      "loss_nats": 2.2863500118255615,
      "step": 410,
      "unclipped_grad_norm": 0.8769013285636902
    },
    {
      "bits_per_byte": 3.225643997897925,
      "learning_rate": 0.00029999538646466355,
      "loss_nats": 2.2358460426330566,
      "step": 420,
      "unclipped_grad_norm": 0.7473549842834473
    },
    {
      "bits_per_byte": 3.2585050669635836,
      "learning_rate": 0.0002999896196193985,
      "loss_nats": 2.2586236000061035,
      "step": 430,
      "unclipped_grad_norm": 0.9111914038658142
    },
    {
      "bits_per_byte": 3.2092602426461303,
      "learning_rate": 0.00029998154617398326,
      "loss_nats": 2.224489688873291,
      "step": 440,
      "unclipped_grad_norm": 1.075931191444397
    },
    {
      "bits_per_byte": 3.2640807445045557,
      "learning_rate": 0.0002999711662663707,
      "loss_nats": 2.26248836517334,
      "step": 450,
      "unclipped_grad_norm": 0.8512128591537476
    },
    {
      "bits_per_byte": 3.207580316112502,
      "learning_rate": 0.0002999584800739247,
      "loss_nats": 2.223325252532959,
      "step": 460,
      "unclipped_grad_norm": 1.2196831703186035
    },
    {
      "bits_per_byte": 3.1660169249131855,
      "learning_rate": 0.0002999434878134173,
      "loss_nats": 2.1945157051086426,
      "step": 470,
      "unclipped_grad_norm": 0.7675222158432007
    },
    {
      "bits_per_byte": 3.1673281206433637,
      "learning_rate": 0.000299926189741025,
      "loss_nats": 2.1954245567321777,
      "step": 480,
      "unclipped_grad_norm": 0.9160394668579102
    },
    {
      "bits_per_byte": 3.113679164602616,
      "learning_rate": 0.000299906586152324,
      "loss_nats": 2.158237934112549,
      "step": 490,
      "unclipped_grad_norm": 0.6955189108848572
    },
    {
      "bits_per_byte": 3.0709621137813112,
      "learning_rate": 0.0002998846773822856,
      "loss_nats": 2.128628730773926,
      "step": 500,
      "unclipped_grad_norm": 0.9366468191146851
    },
    {
      "bits_per_byte": 3.1516054667014854,
      "learning_rate": 0.0002998604638052702,
      "loss_nats": 2.1845264434814453,
      "step": 510,
      "unclipped_grad_norm": 1.3161629438400269
    },
    {
      "bits_per_byte": 3.151479575401054,
      "learning_rate": 0.0002998339458350211,
      "loss_nats": 2.184439182281494,
      "step": 520,
      "unclipped_grad_norm": 0.8699809312820435
    },
    {
      "bits_per_byte": 3.012768688174164,
      "learning_rate": 0.0002998051239246572,
      "loss_nats": 2.088292121887207,
      "step": 530,
      "unclipped_grad_norm": 1.1934181451797485
    },
    {
      "bits_per_byte": 3.0200408025832473,
      "learning_rate": 0.00029977399856666537,
      "loss_nats": 2.0933327674865723,
      "step": 540,
      "unclipped_grad_norm": 0.8321981430053711
    },
    {
      "bits_per_byte": 3.02149439994834,
      "learning_rate": 0.0002997405702928922,
      "loss_nats": 2.0943403244018555,
      "step": 550,
      "unclipped_grad_norm": 0.9657202959060669
    },
    {
      "bits_per_byte": 3.0546815481200342,
      "learning_rate": 0.0002997048396745345,
      "loss_nats": 2.1173439025878906,
      "step": 560,
      "unclipped_grad_norm": 1.2880722284317017
    },
    {
      "bits_per_byte": 2.973202359516667,
      "learning_rate": 0.0002996668073221302,
      "loss_nats": 2.0608668327331543,
      "step": 570,
      "unclipped_grad_norm": 0.8365735411643982
    },
    {
      "bits_per_byte": 2.986164347946895,
      "learning_rate": 0.000299626473885547,
      "loss_nats": 2.0698513984680176,
      "step": 580,
      "unclipped_grad_norm": 0.9132604002952576
    },
    {
      "bits_per_byte": 2.926336399225948,
      "learning_rate": 0.00029958384005397226,
      "loss_nats": 2.028381824493408,
      "step": 590,
      "unclipped_grad_norm": 0.7623230814933777
    },
    {
      "bits_per_byte": 2.903745446139213,
      "learning_rate": 0.0002995389065559004,
      "loss_nats": 2.012722969055176,
      "step": 600,
      "unclipped_grad_norm": 0.8222759366035461
    },
    {
      "bits_per_byte": 2.915261748406288,
      "learning_rate": 0.000299491674159121,
      "loss_nats": 2.020705461502075,
      "step": 610,
      "unclipped_grad_norm": 1.3526110649108887
    },
    {
      "bits_per_byte": 2.8888483089214763,
      "learning_rate": 0.00029944214367070535,
      "loss_nats": 2.002397060394287,
      "step": 620,
      "unclipped_grad_norm": 0.9250975847244263
    },
    {
      "bits_per_byte": 2.806001858243749,
      "learning_rate": 0.00029939031593699284,
      "loss_nats": 1.944972276687622,
      "step": 630,
      "unclipped_grad_norm": 0.7925068140029907
    },
    {
      "bits_per_byte": 2.807606972324252,
      "learning_rate": 0.00029933619184357654,
      "loss_nats": 1.9460848569869995,
      "step": 640,
      "unclipped_grad_norm": 0.8585688471794128
    },
    {
      "bits_per_byte": 2.785753824809718,
      "learning_rate": 0.00029927977231528774,
      "loss_nats": 1.93093740940094,
      "step": 650,
      "unclipped_grad_norm": 0.7446964383125305
    },
    {
      "bits_per_byte": 2.8113040833698495,
      "learning_rate": 0.00029922105831618057,
      "loss_nats": 1.9486474990844727,
      "step": 660,
      "unclipped_grad_norm": 1.026633381843567
    },
    {
      "bits_per_byte": 2.7895744193986354,
      "learning_rate": 0.0002991600508495153,
      "loss_nats": 1.9335856437683105,
      "step": 670,
      "unclipped_grad_norm": 0.8819775581359863
    },
    {
      "bits_per_byte": 2.7608092891459255,
      "learning_rate": 0.00029909675095774114,
      "loss_nats": 1.913647174835205,
      "step": 680,
      "unclipped_grad_norm": 0.845567524433136
    },
    {
      "bits_per_byte": 2.7436651985946665,
      "learning_rate": 0.0002990311597224787,
      "loss_nats": 1.9017637968063354,
      "step": 690,
      "unclipped_grad_norm": 1.0123262405395508
    },
    {
      "bits_per_byte": 2.7268093509662537,
      "learning_rate": 0.00029896327826450103,
      "loss_nats": 1.890080213546753,
      "step": 700,
      "unclipped_grad_norm": 0.8187565803527832
    },
    {
      "bits_per_byte": 2.696082242632501,
      "learning_rate": 0.0002988931077437152,
      "loss_nats": 1.8687818050384521,
      "step": 710,
      "unclipped_grad_norm": 0.9888712763786316
    },
    {
      "bits_per_byte": 2.7032793726058073,
      "learning_rate": 0.00029882064935914164,
      "loss_nats": 1.8737704753875732,
      "step": 720,
      "unclipped_grad_norm": 0.8442726731300354
    },
    {
      "bits_per_byte": 2.680898066370474,
      "learning_rate": 0.0002987459043488943,
      "loss_nats": 1.8582569360733032,
      "step": 730,
      "unclipped_grad_norm": 1.1784851551055908
    },
    {
      "bits_per_byte": 2.619564409541223,
      "learning_rate": 0.0002986688739901591,
      "loss_nats": 1.8157436847686768,
      "step": 740,
      "unclipped_grad_norm": 0.8383970260620117
    },
    {
      "bits_per_byte": 2.6352585143630582,
      "learning_rate": 0.0002985895595991723,
      "loss_nats": 1.8266220092773438,
      "step": 750,
      "unclipped_grad_norm": 1.062637448310852
    },
    {
      "bits_per_byte": 2.547899926857294,
      "learning_rate": 0.00029850796253119824,
      "loss_nats": 1.7660696506500244,
      "step": 760,
      "unclipped_grad_norm": 0.8228002786636353
    },
    {
      "bits_per_byte": 2.573545291823353,
      "learning_rate": 0.00029842408418050544,
      "loss_nats": 1.7838456630706787,
      "step": 770,
      "unclipped_grad_norm": 0.8357667326927185
    },
    {
      "bits_per_byte": 2.523504187832405,
      "learning_rate": 0.0002983379259803436,
      "loss_nats": 1.749159812927246,
      "step": 780,
      "unclipped_grad_norm": 0.7918128967285156
    },
    {
      "bits_per_byte": 2.5308719245953677,
      "learning_rate": 0.0002982494894029186,
      "loss_nats": 1.7542667388916016,
      "step": 790,
      "unclipped_grad_norm": 1.3666422367095947
    },
    {
      "bits_per_byte": 2.443318996694129,
      "learning_rate": 0.0002981587759593675,
      "loss_nats": 1.6935796737670898,
      "step": 800,
      "unclipped_grad_norm": 0.7999250888824463
    },
    {
      "bits_per_byte": 2.500851320991359,
      "learning_rate": 0.00029806578719973283,
      "loss_nats": 1.7334580421447754,
      "step": 810,
      "unclipped_grad_norm": 1.1067378520965576
    },
    {
      "bits_per_byte": 2.4739762800716316,
      "learning_rate": 0.00029797052471293596,
      "loss_nats": 1.714829683303833,
      "step": 820,
      "unclipped_grad_norm": 1.090567708015442
    },
    {
      "bits_per_byte": 2.467352884221603,
      "learning_rate": 0.0002978729901267499,
      "loss_nats": 1.7102386951446533,
      "step": 830,
      "unclipped_grad_norm": 0.9818849563598633
    },
    {
      "bits_per_byte": 2.471093850843171,
      "learning_rate": 0.0002977731851077716,
      "loss_nats": 1.712831735610962,
      "step": 840,
      "unclipped_grad_norm": 0.9710109829902649
    },
    {
      "bits_per_byte": 2.396449768733044,
      "learning_rate": 0.0002976711113613935,
      "loss_nats": 1.6610924005508423,
      "step": 850,
      "unclipped_grad_norm": 1.0234057903289795
    },
    {
      "bits_per_byte": 2.473427999380681,
      "learning_rate": 0.0002975667706317743,
      "loss_nats": 1.7144496440887451,
      "step": 860,
      "unclipped_grad_norm": 0.9857868552207947
    },
    {
      "bits_per_byte": 2.465926632097998,
      "learning_rate": 0.00029746016470180915,
      "loss_nats": 1.7092500925064087,
      "step": 870,
      "unclipped_grad_norm": 1.1294745206832886
    },
    {
      "bits_per_byte": 2.391602093753174,
      "learning_rate": 0.00029735129539309926,
      "loss_nats": 1.6577322483062744,
      "step": 880,
      "unclipped_grad_norm": 0.8081791400909424
    },
    {
      "bits_per_byte": 2.354239550783832,
      "learning_rate": 0.0002972401645659207,
      "loss_nats": 1.6318345069885254,
      "step": 890,
      "unclipped_grad_norm": 0.8606147170066833
    },
    {
      "bits_per_byte": 2.395512979233794,
      "learning_rate": 0.00029712677411919283,
      "loss_nats": 1.6604430675506592,
      "step": 900,
      "unclipped_grad_norm": 0.8592826128005981
    },
    {
      "bits_per_byte": 2.3758491708647442,
      "learning_rate": 0.0002970111259904454,
      "loss_nats": 1.646813154220581,
      "step": 910,
      "unclipped_grad_norm": 0.8169032335281372
    },
    {
      "bits_per_byte": 2.353533905967342,
      "learning_rate": 0.0002968932221557859,
      "loss_nats": 1.6313453912734985,
      "step": 920,
      "unclipped_grad_norm": 0.7358964085578918
    },
    {
      "bits_per_byte": 2.2994974730142004,
      "learning_rate": 0.0002967730646298655,
      "loss_nats": 1.5938901901245117,
      "step": 930,
      "unclipped_grad_norm": 0.8204946517944336
    },
    {
      "bits_per_byte": 2.3548098452540933,
      "learning_rate": 0.0002966506554658449,
      "loss_nats": 1.6322298049926758,
      "step": 940,
      "unclipped_grad_norm": 0.8031407594680786
    },
    {
      "bits_per_byte": 2.362864480724602,
      "learning_rate": 0.00029652599675535876,
      "loss_nats": 1.637812852859497,
      "step": 950,
      "unclipped_grad_norm": 0.8955363035202026
    },
    {
      "bits_per_byte": 2.311957100121258,
      "learning_rate": 0.0002963990906284808,
      "loss_nats": 1.6025265455245972,
      "step": 960,
      "unclipped_grad_norm": 0.8827840089797974
    },
    {
      "bits_per_byte": 2.2663949403067303,
      "learning_rate": 0.0002962699392536863,
      "loss_nats": 1.5709452629089355,
      "step": 970,
      "unclipped_grad_norm": 1.4933069944381714
    },
    {
      "bits_per_byte": 2.401434685868302,
      "learning_rate": 0.0002961385448378161,
      "loss_nats": 1.6645476818084717,
      "step": 980,
      "unclipped_grad_norm": 1.0056283473968506
    },
    {
      "bits_per_byte": 2.23815401317385,
      "learning_rate": 0.00029600490962603817,
      "loss_nats": 1.5513701438903809,
      "step": 990,
      "unclipped_grad_norm": 0.7972294092178345
    },
    {
      "bits_per_byte": 2.255509814368321,
      "learning_rate": 0.00029586903590180956,
      "loss_nats": 1.5634002685546875,
      "step": 1000,
      "unclipped_grad_norm": 0.7948418855667114
    },
    {
      "bits_per_byte": 2.249676162851055,
      "learning_rate": 0.00029573092598683735,
      "loss_nats": 1.559356689453125,
      "step": 1010,
      "unclipped_grad_norm": 0.9423647522926331
    },
    {
      "bits_per_byte": 2.2157856056373317,
      "learning_rate": 0.00029559058224103895,
      "loss_nats": 1.5358655452728271,
      "step": 1020,
      "unclipped_grad_norm": 1.3369964361190796
    },
    {
      "bits_per_byte": 2.21131629248936,
      "learning_rate": 0.0002954480070625017,
      "loss_nats": 1.532767653465271,
      "step": 1030,
      "unclipped_grad_norm": 0.7792882919311523
    },
    {
      "bits_per_byte": 2.300165797595453,
      "learning_rate": 0.00029530320288744205,
      "loss_nats": 1.594353437423706,
      "step": 1040,
      "unclipped_grad_norm": 0.9419035911560059
    },
    {
      "bits_per_byte": 2.21666237319143,
      "learning_rate": 0.0002951561721901638,
      "loss_nats": 1.536473274230957,
      "step": 1050,
      "unclipped_grad_norm": 0.9680492281913757
    },
    {
      "bits_per_byte": 2.242138163263738,
      "learning_rate": 0.00029500691748301594,
      "loss_nats": 1.5541317462921143,
      "step": 1060,
      "unclipped_grad_norm": 0.9887726306915283
    },
    {
      "bits_per_byte": 2.276496685270459,
      "learning_rate": 0.00029485544131634963,
      "loss_nats": 1.5779472589492798,
      "step": 1070,
      "unclipped_grad_norm": 0.8514108657836914
    },
    {
      "bits_per_byte": 2.2430374605450996,
      "learning_rate": 0.0002947017462784745,
      "loss_nats": 1.5547550916671753,
      "step": 1080,
      "unclipped_grad_norm": 0.800697386264801
    },
    {
      "bits_per_byte": 2.1823284447037823,
      "learning_rate": 0.00029454583499561476,
      "loss_nats": 1.5126748085021973,
      "step": 1090,
      "unclipped_grad_norm": 0.9124817848205566
    },
    {
      "bits_per_byte": 2.1899661942286,
      "learning_rate": 0.00029438771013186413,
      "loss_nats": 1.5179688930511475,
      "step": 1100,
      "unclipped_grad_norm": 0.8022184371948242
    },
    {
      "bits_per_byte": 2.145753238310091,
      "learning_rate": 0.0002942273743891401,
      "loss_nats": 1.4873228073120117,
      "step": 1110,
      "unclipped_grad_norm": 1.139209508895874
    },
    {
      "bits_per_byte": 2.188020210534087,
      "learning_rate": 0.00029406483050713824,
      "loss_nats": 1.5166200399398804,
      "step": 1120,
      "unclipped_grad_norm": 0.8927217721939087
    },
    {
      "bits_per_byte": 2.2131649340034842,
      "learning_rate": 0.00029390008126328504,
      "loss_nats": 1.5340490341186523,
      "step": 1130,
      "unclipped_grad_norm": 0.8271659016609192
    },
    {
      "bits_per_byte": 2.1401269978697646,
      "learning_rate": 0.00029373312947269045,
      "loss_nats": 1.4834229946136475,
      "step": 1140,
      "unclipped_grad_norm": 0.9781836271286011
    },
    {
      "bits_per_byte": 2.1955361963420934,
      "learning_rate": 0.00029356397798809995,
      "loss_nats": 1.5218297243118286,
      "step": 1150,
      "unclipped_grad_norm": 0.8088869452476501
    },
    {
      "bits_per_byte": 2.103003854750424,
      "learning_rate": 0.0002933926296998457,
      "loss_nats": 1.4576911926269531,
      "step": 1160,
      "unclipped_grad_norm": 0.761218786239624
    },
    {
      "bits_per_byte": 2.164645876472672,
      "learning_rate": 0.0002932190875357972,
      "loss_nats": 1.5004181861877441,
      "step": 1170,
      "unclipped_grad_norm": 0.8793766498565674
    },
    {
      "bits_per_byte": 2.159191274717907,
      "learning_rate": 0.00029304335446131126,
      "loss_nats": 1.4966373443603516,
      "step": 1180,
      "unclipped_grad_norm": 0.8019644618034363
    },
    {
      "bits_per_byte": 2.134348724766076,
      "learning_rate": 0.00029286543347918116,
      "loss_nats": 1.4794178009033203,
      "step": 1190,
      "unclipped_grad_norm": 0.7809260487556458
    },
    {
      "bits_per_byte": 2.1456660431061034,
      "learning_rate": 0.0002926853276295856,
      "loss_nats": 1.4872623682022095,
      "step": 1200,
      "unclipped_grad_norm": 0.896268904209137
    },
    {
      "bits_per_byte": 2.1662845271700926,
      "learning_rate": 0.0002925030399900368,
      "loss_nats": 1.501554012298584,
      "step": 1210,
      "unclipped_grad_norm": 0.8574862480163574
    },
    {
      "bits_per_byte": 2.1378967268533757,
      "learning_rate": 0.00029231857367532726,
      "loss_nats": 1.481877088546753,
      "step": 1220,
      "unclipped_grad_norm": 0.8502950668334961
    },
    {
      "bits_per_byte": 2.128576127089866,
      "learning_rate": 0.0002921319318374776,
      "loss_nats": 1.4754165410995483,
      "step": 1230,
      "unclipped_grad_norm": 0.7498336434364319
    },
    {
      "bits_per_byte": 2.0970609942498597,
      "learning_rate": 0.00029194311766568174,
      "loss_nats": 1.4535719156265259,
      "step": 1240,
      "unclipped_grad_norm": 0.761203944683075
    },
    {
      "bits_per_byte": 2.0385041693014396,
      "learning_rate": 0.000291752134386253,
      "loss_nats": 1.4129834175109863,
      "step": 1250,
      "unclipped_grad_norm": 0.7150603532791138
    },
    {
      "bits_per_byte": 2.0885559362167414,
      "learning_rate": 0.00029155898526256856,
      "loss_nats": 1.447676658630371,
      "step": 1260,
      "unclipped_grad_norm": 0.7949564456939697
    },
    {
      "bits_per_byte": 2.1179862953629884,
      "learning_rate": 0.0002913636735950141,
      "loss_nats": 1.468076229095459,
      "step": 1270,
      "unclipped_grad_norm": 0.862070620059967
    },
    {
      "bits_per_byte": 2.1475409979658098,
      "learning_rate": 0.0002911662027209272,
      "loss_nats": 1.488561987876892,
      "step": 1280,
      "unclipped_grad_norm": 0.7539991140365601
    },
    {
      "bits_per_byte": 2.1029943957046267,
      "learning_rate": 0.00029096657601454013,
      "loss_nats": 1.4576846361160278,
      "step": 1290,
      "unclipped_grad_norm": 0.8074342608451843
    },
    {
      "bits_per_byte": 2.049033979083166,
      "learning_rate": 0.00029076479688692266,
      "loss_nats": 1.4202821254730225,
      "step": 1300,
      "unclipped_grad_norm": 0.7586180567741394
    },
    {
      "bits_per_byte": 2.076369761524555,
      "learning_rate": 0.00029056086878592323,
      "loss_nats": 1.4392298460006714,
      "step": 1310,
      "unclipped_grad_norm": 0.9284558892250061
    },
    {
      "bits_per_byte": 2.051258058724124,
      "learning_rate": 0.00029035479519611045,
      "loss_nats": 1.4218237400054932,
      "step": 1320,
      "unclipped_grad_norm": 0.7597900032997131
    },
    {
      "bits_per_byte": 2.060438148661879,
      "learning_rate": 0.0002901465796387134,
      "loss_nats": 1.4281868934631348,
      "step": 1330,
      "unclipped_grad_norm": 0.7334625720977783
    },
    {
      "bits_per_byte": 2.04908454198252,
      "learning_rate": 0.00028993622567156154,
      "loss_nats": 1.4203171730041504,
      "step": 1340,
      "unclipped_grad_norm": 0.7958364486694336
    },
    {
      "bits_per_byte": 2.0721673654507224,
      "learning_rate": 0.0002897237368890237,
      "loss_nats": 1.436316967010498,
      "step": 1350,
      "unclipped_grad_norm": 0.7738369107246399
    },
    {
      "bits_per_byte": 2.034246910761982,
      "learning_rate": 0.0002895091169219469,
      "loss_nats": 1.4100325107574463,
      "step": 1360,
      "unclipped_grad_norm": 0.8466383218765259
    },
    {
      "bits_per_byte": 2.0554857362476335,
      "learning_rate": 0.00028929236943759425,
      "loss_nats": 1.4247541427612305,
      "step": 1370,
      "unclipped_grad_norm": 0.7919691205024719
    },
    {
      "bits_per_byte": 2.020279167789508,
      "learning_rate": 0.00028907349813958225,
      "loss_nats": 1.40035080909729,
      "step": 1380,
      "unclipped_grad_norm": 0.8231279850006104
    },
    {
      "bits_per_byte": 2.070644631059983,
      "learning_rate": 0.0002888525067678175,
      "loss_nats": 1.4352614879608154,
      "step": 1390,
      "unclipped_grad_norm": 0.7695969343185425
    },
    {
      "bits_per_byte": 2.174483628067326,
      "learning_rate": 0.00028862939909843273,
      "loss_nats": 1.507237195968628,
      "step": 1400,
      "unclipped_grad_norm": 0.8150069713592529
    },
    {
      "bits_per_byte": 2.0594870846026074,
      "learning_rate": 0.0002884041789437225,
      "loss_nats": 1.427527666091919,
      "step": 1410,
      "unclipped_grad_norm": 0.7077280879020691
    },
    {
      "bits_per_byte": 2.048123158964197,
      "learning_rate": 0.0002881768501520778,
      "loss_nats": 1.4196507930755615,
      "step": 1420,
      "unclipped_grad_norm": 0.7647379040718079
    },
    {
      "bits_per_byte": 2.0019392818668,
      "learning_rate": 0.0002879474166079205,
      "loss_nats": 1.3876385688781738,
      "step": 1430,
      "unclipped_grad_norm": 0.6688193082809448
    },
    {
      "bits_per_byte": 2.0332329010524948,
      "learning_rate": 0.00028771588223163694,
      "loss_nats": 1.4093296527862549,
      "step": 1440,
      "unclipped_grad_norm": 0.8427934646606445
    },
    {
      "bits_per_byte": 2.0070671165849254,
      "learning_rate": 0.00028748225097951054,
      "loss_nats": 1.39119291305542,
      "step": 1450,
      "unclipped_grad_norm": 0.6920660138130188
    },
    {
      "bits_per_byte": 2.1256969655317715,
      "learning_rate": 0.000287246526843655,
      "loss_nats": 1.4734208583831787,
      "step": 1460,
      "unclipped_grad_norm": 0.799986720085144
    },
    {
      "bits_per_byte": 2.0930381480635276,
      "learning_rate": 0.00028700871385194533,
      "loss_nats": 1.4507834911346436,
      "step": 1470,
      "unclipped_grad_norm": 0.6732411980628967
    },
    {
      "bits_per_byte": 2.0451248134290543,
      "learning_rate": 0.00028676881606794945,
      "loss_nats": 1.4175724983215332,
      "step": 1480,
      "unclipped_grad_norm": 0.6982382535934448
    },
    {
      "bits_per_byte": 2.0566264971708064,
      "learning_rate": 0.00028652683759085854,
      "loss_nats": 1.4255448579788208,
      "step": 1490,
      "unclipped_grad_norm": 0.7266837358474731
    },
    {
      "bits_per_byte": 1.9341151717766374,
      "learning_rate": 0.0002862827825554171,
      "loss_nats": 1.3406264781951904,
      "step": 1500,
      "unclipped_grad_norm": 0.6626343727111816
    },
    {
      "bits_per_byte": 1.963616731740342,
      "learning_rate": 0.00028603665513185245,
      "loss_nats": 1.3610754013061523,
      "step": 1510,
      "unclipped_grad_norm": 0.789225697517395
    },
    {
      "bits_per_byte": 2.0151121210269856,
      "learning_rate": 0.000285788459525803,
      "loss_nats": 1.3967692852020264,
      "step": 1520,
      "unclipped_grad_norm": 0.7025586366653442
    },
    {
      "bits_per_byte": 2.0371694119480934,
      "learning_rate": 0.00028553819997824707,
      "loss_nats": 1.4120582342147827,
      "step": 1530,
      "unclipped_grad_norm": 0.750766396522522
    },
    {
      "bits_per_byte": 2.0209939276864937,
      "learning_rate": 0.00028528588076542964,
      "loss_nats": 1.400846242904663,
      "step": 1540,
      "unclipped_grad_norm": 0.7461745738983154
    },
    {
      "bits_per_byte": 2.082936231117148,
      "learning_rate": 0.00028503150619878996,
      "loss_nats": 1.4437813758850098,
      "step": 1550,
      "unclipped_grad_norm": 0.765493631362915
    },
    {
      "bits_per_byte": 1.954267066908836,
      "learning_rate": 0.0002847750806248876,
      "loss_nats": 1.3545947074890137,
      "step": 1560,
      "unclipped_grad_norm": 0.727699875831604
    },
    {
      "bits_per_byte": 1.9578143810655324,
      "learning_rate": 0.000284516608425328,
      "loss_nats": 1.357053518295288,
      "step": 1570,
      "unclipped_grad_norm": 0.7358320951461792
    },
    {
      "bits_per_byte": 1.9655836973182597,
      "learning_rate": 0.000284256094016688,
      "loss_nats": 1.3624387979507446,
      "step": 1580,
      "unclipped_grad_norm": 0.6977834701538086
    },
    {
      "bits_per_byte": 1.9752479184181935,
      "learning_rate": 0.0002839935418504399,
      "loss_nats": 1.3691375255584717,
      "step": 1590,
      "unclipped_grad_norm": 0.6583591103553772
    },
    {
      "bits_per_byte": 1.9806421542625054,
      "learning_rate": 0.000283728956412876,
      "loss_nats": 1.372876524925232,
      "step": 1600,
      "unclipped_grad_norm": 0.727998673915863
    },
    {
      "bits_per_byte": 1.9474555220387637,
      "learning_rate": 0.00028346234222503136,
      "loss_nats": 1.3498733043670654,
      "step": 1610,
      "unclipped_grad_norm": 0.7535929083824158
    },
    {
      "bits_per_byte": 1.942356580405983,
      "learning_rate": 0.00028319370384260675,
      "loss_nats": 1.3463389873504639,
      "step": 1620,
      "unclipped_grad_norm": 0.6403960585594177
    },
    {
      "bits_per_byte": 1.9826001767425787,
      "learning_rate": 0.00028292304585589105,
      "loss_nats": 1.3742337226867676,
      "step": 1630,
      "unclipped_grad_norm": 0.7281255125999451
    },
    {
      "bits_per_byte": 1.9377553565647996,
      "learning_rate": 0.0002826503728896824,
      "loss_nats": 1.3431496620178223,
      "step": 1640,
      "unclipped_grad_norm": 0.7114111185073853
    },
    {
      "bits_per_byte": 1.9278453722567828,
      "learning_rate": 0.0002823756896032095,
      "loss_nats": 1.3362805843353271,
      "step": 1650,
      "unclipped_grad_norm": 0.6373128890991211
    },
    {
      "bits_per_byte": 1.9498221752972869,
      "learning_rate": 0.0002820990006900518,
      "loss_nats": 1.3515137434005737,
      "step": 1660,
      "unclipped_grad_norm": 0.6859769225120544
    },
    {
      "bits_per_byte": 1.9095433225172644,
      "learning_rate": 0.00028182031087805947,
      "loss_nats": 1.323594570159912,
      "step": 1670,
      "unclipped_grad_norm": 0.7934040427207947
    },
    {
      "bits_per_byte": 1.938967146322779,
      "learning_rate": 0.0002815396249292725,
      "loss_nats": 1.343989610671997,
      "step": 1680,
      "unclipped_grad_norm": 0.7104787230491638
    },
    {
      "bits_per_byte": 1.9591448388526072,
      "learning_rate": 0.0002812569476398391,
      "loss_nats": 1.357975721359253,
      "step": 1690,
      "unclipped_grad_norm": 0.6892858743667603
    },
    {
      "bits_per_byte": 1.9072468381802934,
      "learning_rate": 0.00028097228383993446,
      "loss_nats": 1.3220027685165405,
      "step": 1700,
      "unclipped_grad_norm": 0.7498061656951904
    },
    {
      "bits_per_byte": 1.9434417909329273,
      "learning_rate": 0.0002806856383936774,
      "loss_nats": 1.3470911979675293,
      "step": 1710,
      "unclipped_grad_norm": 0.6478108763694763
    },
    {
      "bits_per_byte": 1.8728540916260505,
      "learning_rate": 0.00028039701619904774,
      "loss_nats": 1.2981635332107544,
      "step": 1720,
      "unclipped_grad_norm": 0.7232697010040283
    },
    {
      "bits_per_byte": 1.9589699324966796,
      "learning_rate": 0.0002801064221878024,
      "loss_nats": 1.3578544855117798,
      "step": 1730,
      "unclipped_grad_norm": 0.7008518576622009
    },
    {
      "bits_per_byte": 1.9590435410712488,
      "learning_rate": 0.00027981386132539134,
      "loss_nats": 1.3579055070877075,
      "step": 1740,
      "unclipped_grad_norm": 0.6955974698066711
    },
    {
      "bits_per_byte": 1.919930042733473,
      "learning_rate": 0.0002795193386108724,
      "loss_nats": 1.330794095993042,
      "step": 1750,
      "unclipped_grad_norm": 0.6556993126869202
    },
    {
      "bits_per_byte": 1.9569465566092785,
      "learning_rate": 0.00027922285907682625,
      "loss_nats": 1.3564519882202148,
      "step": 1760,
      "unclipped_grad_norm": 0.698861837387085
    },
    {
      "bits_per_byte": 1.9512123110642108,
      "learning_rate": 0.00027892442778927005,
      "loss_nats": 1.3524773120880127,
      "step": 1770,
      "unclipped_grad_norm": 0.6838504672050476
    },
    {
      "bits_per_byte": 2.0216474617597724,
      "learning_rate": 0.00027862404984757116,
      "loss_nats": 1.401299238204956,
      "step": 1780,
      "unclipped_grad_norm": 0.7214158773422241
    },
    {
      "bits_per_byte": 1.8826208143858987,
      "learning_rate": 0.0002783217303843597,
      "loss_nats": 1.3049333095550537,
      "step": 1790,
      "unclipped_grad_norm": 0.7236432433128357
    },
    {
      "bits_per_byte": 2.013814339943575,
      "learning_rate": 0.00027801747456544134,
      "loss_nats": 1.3958697319030762,
      "step": 1800,
      "unclipped_grad_norm": 0.7153979539871216
    },
    {
      "bits_per_byte": 1.8985195785622602,
      "learning_rate": 0.00027771128758970843,
      "loss_nats": 1.3159534931182861,
      "step": 1810,
      "unclipped_grad_norm": 0.7165744304656982
    },
    {
      "bits_per_byte": 1.956814645916067,
      "learning_rate": 0.00027740317468905165,
      "loss_nats": 1.3563605546951294,
      "step": 1820,
      "unclipped_grad_norm": 0.7207518815994263
    },
    {
      "bits_per_byte": 1.973846947744265,
      "learning_rate": 0.0002770931411282704,
      "loss_nats": 1.368166446685791,
      "step": 1830,
      "unclipped_grad_norm": 0.6764873266220093
    },
    {
      "bits_per_byte": 1.953527541510126,
      "learning_rate": 0.0002767811922049827,
      "loss_nats": 1.3540821075439453,
      "step": 1840,
      "unclipped_grad_norm": 0.6510032415390015
    },
    {
      "bits_per_byte": 1.8838981295338566,
      "learning_rate": 0.00027646733324953503,
      "loss_nats": 1.3058186769485474,
      "step": 1850,
      "unclipped_grad_norm": 0.6467540860176086
    },
    {
      "bits_per_byte": 1.9116946534969073,
      "learning_rate": 0.00027615156962491097,
      "loss_nats": 1.3250857591629028,
      "step": 1860,
      "unclipped_grad_norm": 0.6336910128593445
    },
    {
      "bits_per_byte": 1.996913260877985,
      "learning_rate": 0.0002758339067266396,
      "loss_nats": 1.3841547966003418,
      "step": 1870,
      "unclipped_grad_norm": 0.7375073432922363
    },
    {
      "bits_per_byte": 1.8879835774051028,
      "learning_rate": 0.0002755143499827035,
      "loss_nats": 1.3086504936218262,
      "step": 1880,
      "unclipped_grad_norm": 0.6048603057861328
    },
    {
      "bits_per_byte": 1.9029378128629257,
      "learning_rate": 0.0002751929048534456,
      "loss_nats": 1.3190159797668457,
      "step": 1890,
      "unclipped_grad_norm": 0.6331550478935242
    },
    {
      "bits_per_byte": 1.9156196415548983,
      "learning_rate": 0.0002748695768314763,
      "loss_nats": 1.3278063535690308,
      "step": 1900,
      "unclipped_grad_norm": 0.7952298521995544
    },
    {
      "bits_per_byte": 1.9644835243006904,
      "learning_rate": 0.00027454437144157947,
      "loss_nats": 1.3616762161254883,
      "step": 1910,
      "unclipped_grad_norm": 0.6600897312164307
    },
    {
      "bits_per_byte": 1.8776484519841532,
      "learning_rate": 0.00027421729424061787,
      "loss_nats": 1.3014867305755615,
      "step": 1920,
      "unclipped_grad_norm": 0.645132303237915
    },
    {
      "bits_per_byte": 1.9490762865404947,
      "learning_rate": 0.0002738883508174385,
      "loss_nats": 1.350996732711792,
      "step": 1930,
      "unclipped_grad_norm": 0.6212714314460754
    },
    {
      "bits_per_byte": 1.942025857768374,
      "learning_rate": 0.0002735575467927768,
      "loss_nats": 1.3461097478866577,
      "step": 1940,
      "unclipped_grad_norm": 0.6636208295822144
    },
    {
      "bits_per_byte": 1.8837596834999122,
      "learning_rate": 0.0002732248878191609,
      "loss_nats": 1.305722713470459,
      "step": 1950,
      "unclipped_grad_norm": 0.7458274960517883
    },
    {
      "bits_per_byte": 1.8770238109962194,
      "learning_rate": 0.0002728903795808148,
      "loss_nats": 1.301053762435913,
      "step": 1960,
      "unclipped_grad_norm": 0.6796664595603943
    },
    {
      "bits_per_byte": 1.873630765277347,
      "learning_rate": 0.0002725540277935614,
      "loss_nats": 1.2987018823623657,
      "step": 1970,
      "unclipped_grad_norm": 0.7089046239852905
    },
    {
      "bits_per_byte": 1.8950018454215123,
      "learning_rate": 0.00027221583820472456,
      "loss_nats": 1.3135151863098145,
      "step": 1980,
      "unclipped_grad_norm": 0.6426982283592224
    },
    {
      "bits_per_byte": 1.8681299001895,
      "learning_rate": 0.0002718758165930314,
      "loss_nats": 1.294888973236084,
      "step": 1990,
      "unclipped_grad_norm": 0.7099292278289795
    },
    {
      "bits_per_byte": 1.964639340582372,
      "learning_rate": 0.0002715339687685131,
      "loss_nats": 1.3617842197418213,
      "step": 2000,
      "unclipped_grad_norm": 0.7598643898963928
    },
    {
      "bits_per_byte": 1.939805561745735,
      "learning_rate": 0.00027119030057240576,
      "loss_nats": 1.3445707559585571,
      "step": 2010,
      "unclipped_grad_norm": 0.6349508166313171
    },
    {
      "bits_per_byte": 1.851296754271001,
      "learning_rate": 0.0002708448178770506,
      "loss_nats": 1.2832211256027222,
      "step": 2020,
      "unclipped_grad_norm": 0.6008884310722351
    },
    {
      "bits_per_byte": 1.890781391169339,
      "learning_rate": 0.00027049752658579376,
      "loss_nats": 1.3105897903442383,
      "step": 2030,
      "unclipped_grad_norm": 0.637248158454895
    },
    {
      "bits_per_byte": 1.882627693691933,
      "learning_rate": 0.00027014843263288514,
      "loss_nats": 1.3049380779266357,
      "step": 2040,
      "unclipped_grad_norm": 0.6375738382339478
    },
    {
      "bits_per_byte": 1.848167701921203,
      "learning_rate": 0.00026979754198337724,
      "loss_nats": 1.2810522317886353,
      "step": 2050,
      "unclipped_grad_norm": 0.6515361070632935
    },
    {
      "bits_per_byte": 1.833815233758749,
      "learning_rate": 0.00026944486063302305,
      "loss_nats": 1.271103858947754,
      "step": 2060,
      "unclipped_grad_norm": 0.6752470135688782
    },
    {
      "bits_per_byte": 1.960125139962525,
      "learning_rate": 0.0002690903946081738,
      "loss_nats": 1.3586552143096924,
      "step": 2070,
      "unclipped_grad_norm": 0.7093353867530823
    },
    {
      "bits_per_byte": 1.93064937739645,
      "learning_rate": 0.00026873414996567596,
      "loss_nats": 1.338224172592163,
      "step": 2080,
      "unclipped_grad_norm": 0.7233569622039795
    },
    {
      "bits_per_byte": 1.8411410067549012,
      "learning_rate": 0.00026837613279276736,
      "loss_nats": 1.276181697845459,
      "step": 2090,
      "unclipped_grad_norm": 0.6208991408348083
    },
    {
      "bits_per_byte": 1.7919973082361553,
      "learning_rate": 0.0002680163492069739,
      "loss_nats": 1.2421178817749023,
      "step": 2100,
      "unclipped_grad_norm": 0.6259884834289551
    },
    {
      "bits_per_byte": 1.830389339353562,
      "learning_rate": 0.00026765480535600414,
      "loss_nats": 1.2687292098999023,
      "step": 2110,
      "unclipped_grad_norm": 0.633368968963623
    },
    {
      "bits_per_byte": 1.8446745622995284,
      "learning_rate": 0.0002672915074176452,
      "loss_nats": 1.2786309719085693,
      "step": 2120,
      "unclipped_grad_norm": 0.6770470142364502
    },
    {
      "bits_per_byte": 1.8875441617321482,
      "learning_rate": 0.00026692646159965624,
      "loss_nats": 1.308345913887024,
      "step": 2130,
      "unclipped_grad_norm": 0.7079503536224365
    },
    {
      "bits_per_byte": 1.912643825747019,
      "learning_rate": 0.0002665596741396633,
      "loss_nats": 1.3257436752319336,
      "step": 2140,
      "unclipped_grad_norm": 0.6220520734786987
    },
    {
      "bits_per_byte": 1.8089860984536854,
      "learning_rate": 0.000266191151305052,
      "loss_nats": 1.2538936138153076,
      "step": 2150,
      "unclipped_grad_norm": 0.6506339907646179
    },
    {
      "bits_per_byte": 1.9026210208200363,
      "learning_rate": 0.00026582089939286075,
      "loss_nats": 1.3187963962554932,
      "step": 2160,
      "unclipped_grad_norm": 0.6963920593261719
    },
    {
      "bits_per_byte": 1.8293292382936435,
      "learning_rate": 0.0002654489247296732,
      "loss_nats": 1.2679944038391113,
      "step": 2170,
      "unclipped_grad_norm": 0.6530651450157166
    },
    {
      "bits_per_byte": 1.90049290749826,
      "learning_rate": 0.00026507523367151007,
      "loss_nats": 1.3173213005065918,
      "step": 2180,
      "unclipped_grad_norm": 0.6189562678337097
    },
    {
      "bits_per_byte": 1.8805790363548553,
      "learning_rate": 0.00026469983260372037,
      "loss_nats": 1.3035180568695068,
      "step": 2190,
      "unclipped_grad_norm": 0.6570361852645874
    },
    {
      "bits_per_byte": 1.8344993807438814,
      "learning_rate": 0.0002643227279408727,
      "loss_nats": 1.271578073501587,
      "step": 2200,
      "unclipped_grad_norm": 0.5929612517356873
    },
    {
      "bits_per_byte": 1.837753980428809,
      "learning_rate": 0.0002639439261266453,
      "loss_nats": 1.273833990097046,
      "step": 2210,
      "unclipped_grad_norm": 0.6836420893669128
    },
    {
      "bits_per_byte": 1.8346952689832141,
      "learning_rate": 0.000263563433633716,
      "loss_nats": 1.2717138528823853,
      "step": 2220,
      "unclipped_grad_norm": 0.6392546892166138
    },
    {
      "bits_per_byte": 1.7850062134785822,
      "learning_rate": 0.00026318125696365156,
      "loss_nats": 1.237272024154663,
      "step": 2230,
      "unclipped_grad_norm": 0.5976205468177795
    },
    {
      "bits_per_byte": 1.8807291772090586,
      "learning_rate": 0.000262797402646797,
      "loss_nats": 1.3036221265792847,
      "step": 2240,
      "unclipped_grad_norm": 0.6432849168777466
    },
    {
      "bits_per_byte": 1.861241479074492,
      "learning_rate": 0.00026241187724216334,
      "loss_nats": 1.2901142835617065,
      "step": 2250,
      "unclipped_grad_norm": 0.6619172096252441
    },
    {
      "bits_per_byte": 1.9088667427687702,
      "learning_rate": 0.0002620246873373162,
      "loss_nats": 1.3231256008148193,
      "step": 2260,
      "unclipped_grad_norm": 0.6604200601577759
    },
    {
      "bits_per_byte": 1.8085772956925845,
      "learning_rate": 0.00026163583954826264,
      "loss_nats": 1.2536102533340454,
      "step": 2270,
      "unclipped_grad_norm": 0.6277530193328857
    },
    {
      "bits_per_byte": 1.8640159031982109,
      "learning_rate": 0.0002612453405193386,
      "loss_nats": 1.2920373678207397,
      "step": 2280,
      "unclipped_grad_norm": 0.6375154256820679
    },
    {
      "bits_per_byte": 1.8154523021608253,
      "learning_rate": 0.000260853196923095,
      "loss_nats": 1.258375644683838,
      "step": 2290,
      "unclipped_grad_norm": 0.6188551187515259
    },
    {
      "bits_per_byte": 1.8127673090155552,
      "learning_rate": 0.0002604594154601839,
      "loss_nats": 1.256514549255371,
      "step": 2300,
      "unclipped_grad_norm": 0.6347740292549133
    },
    {
      "bits_per_byte": 1.811916166876435,
      "learning_rate": 0.0002600640028592441,
      "loss_nats": 1.2559245824813843,
      "step": 2310,
      "unclipped_grad_norm": 0.621824324131012
    },
    {
      "bits_per_byte": 1.866308603916863,
      "learning_rate": 0.0002596669658767859,
      "loss_nats": 1.2936265468597412,
      "step": 2320,
      "unclipped_grad_norm": 0.6393704414367676
    },
    {
      "bits_per_byte": 1.8101679632304146,
      "learning_rate": 0.000259268311297076,
      "loss_nats": 1.2547128200531006,
      "step": 2330,
      "unclipped_grad_norm": 0.6721166372299194
    },
    {
      "bits_per_byte": 1.82719372971788,
      "learning_rate": 0.00025886804593202114,
      "loss_nats": 1.2665141820907593,
      "step": 2340,
      "unclipped_grad_norm": 0.646355152130127
    },
    {
      "bits_per_byte": 1.8870428323048831,
      "learning_rate": 0.00025846617662105193,
      "loss_nats": 1.3079984188079834,
      "step": 2350,
      "unclipped_grad_norm": 0.6942237019538879
    },
    {
      "bits_per_byte": 1.804568896048925,
      "learning_rate": 0.00025806271023100637,
      "loss_nats": 1.2508318424224854,
      "step": 2360,
      "unclipped_grad_norm": 0.6387670040130615
    },
    {
      "bits_per_byte": 1.739788190948431,
      "learning_rate": 0.00025765765365601165,
      "loss_nats": 1.2059292793273926,
      "step": 2370,
      "unclipped_grad_norm": 0.6083642840385437
    },
    {
      "bits_per_byte": 1.8484354789085964,
      "learning_rate": 0.00025725101381736707,
      "loss_nats": 1.2812378406524658,
      "step": 2380,
      "unclipped_grad_norm": 0.6751434803009033
    },
    {
      "bits_per_byte": 1.8404750899307605,
      "learning_rate": 0.00025684279766342567,
      "loss_nats": 1.2757201194763184,
      "step": 2390,
      "unclipped_grad_norm": 0.6037119626998901
    },
    {
      "bits_per_byte": 1.8757652419572053,
      "learning_rate": 0.00025643301216947504,
      "loss_nats": 1.3001813888549805,
      "step": 2400,
      "unclipped_grad_norm": 0.6957288980484009
    },
    {
      "bits_per_byte": 1.801620425482533,
      "learning_rate": 0.00025602166433761866,
      "loss_nats": 1.2487881183624268,
      "step": 2410,
      "unclipped_grad_norm": 0.626511812210083
    },
    {
      "bits_per_byte": 1.8236299052267009,
      "learning_rate": 0.0002556087611966561,
      "loss_nats": 1.264043927192688,
      "step": 2420,
      "unclipped_grad_norm": 0.6481806635856628
    },
    {
      "bits_per_byte": 1.8322460640522769,
      "learning_rate": 0.00025519430980196277,
      "loss_nats": 1.2700161933898926,
      "step": 2430,
      "unclipped_grad_norm": 0.634190559387207
    },
    {
      "bits_per_byte": 1.778641823500753,
      "learning_rate": 0.0002547783172353695,
      "loss_nats": 1.2328605651855469,
      "step": 2440,
      "unclipped_grad_norm": 0.6495575308799744
    },
    {
      "bits_per_byte": 1.9154751761281736,
      "learning_rate": 0.0002543607906050415,
      "loss_nats": 1.327706217765808,
      "step": 2450,
      "unclipped_grad_norm": 0.682089626789093
    },
    {
      "bits_per_byte": 1.7848875454494868,
      "learning_rate": 0.0002539417370453568,
      "loss_nats": 1.237189769744873,
      "step": 2460,
      "unclipped_grad_norm": 0.6699327230453491
    },
    {
      "bits_per_byte": 1.784353367335907,
      "learning_rate": 0.00025352116371678464,
      "loss_nats": 1.2368195056915283,
      "step": 2470,
      "unclipped_grad_norm": 0.6535210609436035
    },
    {
      "bits_per_byte": 1.865447314801342,
      "learning_rate": 0.0002530990778057627,
      "loss_nats": 1.293029546737671,
      "step": 2480,
      "unclipped_grad_norm": 0.6640622019767761
    },
    {
      "bits_per_byte": 1.857090849778569,
      "learning_rate": 0.00025267548652457475,
      "loss_nats": 1.287237286567688,
      "step": 2490,
      "unclipped_grad_norm": 0.6223002076148987
    },
    {
      "bits_per_byte": 1.8122928088818246,
      "learning_rate": 0.00025225039711122696,
      "loss_nats": 1.2561856508255005,
      "step": 2500,
      "unclipped_grad_norm": 0.6052747368812561
    },
    {
      "bits_per_byte": 1.76881026328153,
      "learning_rate": 0.00025182381682932464,
      "loss_nats": 1.226045846939087,
      "step": 2510,
      "unclipped_grad_norm": 0.6596174836158752
    },
    {
      "bits_per_byte": 1.8006248179166882,
      "learning_rate": 0.0002513957529679477,
      "loss_nats": 1.2480980157852173,
      "step": 2520,
      "unclipped_grad_norm": 0.6490770578384399
    },
    {
      "bits_per_byte": 1.8232907554391995,
      "learning_rate": 0.00025096621284152656,
      "loss_nats": 1.2638088464736938,
      "step": 2530,
      "unclipped_grad_norm": 0.6303134560585022
    },
    {
      "bits_per_byte": 1.8125131186575798,
      "learning_rate": 0.0002505352037897168,
      "loss_nats": 1.256338357925415,
      "step": 2540,
      "unclipped_grad_norm": 0.6458812952041626
    },
    {
      "bits_per_byte": 1.7674137641565237,
      "learning_rate": 0.0002501027331772739,
      "loss_nats": 1.2250778675079346,
      "step": 2550,
      "unclipped_grad_norm": 0.6546228528022766
    },
    {
      "bits_per_byte": 1.7342415784754543,
      "learning_rate": 0.00024966880839392746,
      "loss_nats": 1.2020846605300903,
      "step": 2560,
      "unclipped_grad_norm": 0.602087676525116
    },
    {
      "bits_per_byte": 1.774496353684356,
      "learning_rate": 0.0002492334368542548,
      "loss_nats": 1.2299871444702148,
      "step": 2570,
      "unclipped_grad_norm": 0.6244005560874939
    },
    {
      "bits_per_byte": 1.762712446412538,
      "learning_rate": 0.00024879662599755416,
      "loss_nats": 1.2218191623687744,
      "step": 2580,
      "unclipped_grad_norm": 0.6090607643127441
    },
    {
      "bits_per_byte": 1.7589357073995908,
      "learning_rate": 0.00024835838328771803,
      "loss_nats": 1.2192013263702393,
      "step": 2590,
      "unclipped_grad_norm": 0.6631695032119751
    },
    {
      "bits_per_byte": 1.7746397872151756,
      "learning_rate": 0.0002479187162131051,
      "loss_nats": 1.2300865650177002,
      "step": 2600,
      "unclipped_grad_norm": 0.6302726864814758
    },
    {
      "bits_per_byte": 1.8022297600145398,
      "learning_rate": 0.00024747763228641264,
      "loss_nats": 1.2492104768753052,
      "step": 2610,
      "unclipped_grad_norm": 0.6297239065170288
    },
    {
      "bits_per_byte": 1.7449476704743152,
      "learning_rate": 0.00024703513904454795,
      "loss_nats": 1.209505558013916,
      "step": 2620,
      "unclipped_grad_norm": 0.617965042591095
    },
    {
      "bits_per_byte": 1.738682342503383,
      "learning_rate": 0.00024659124404849956,
      "loss_nats": 1.205162763595581,
      "step": 2630,
      "unclipped_grad_norm": 0.6048376560211182
    },
    {
      "bits_per_byte": 1.7097366304672672,
      "learning_rate": 0.00024614595488320833,
      "loss_nats": 1.1850991249084473,
      "step": 2640,
      "unclipped_grad_norm": 0.5914375185966492
    },
    {
      "bits_per_byte": 1.790221415383346,
      "learning_rate": 0.00024569927915743745,
      "loss_nats": 1.240886926651001,
      "step": 2650,
      "unclipped_grad_norm": 0.6316050887107849
    },
    {
      "bits_per_byte": 1.7080907564985102,
      "learning_rate": 0.0002452512245036427,
      "loss_nats": 1.1839582920074463,
      "step": 2660,
      "unclipped_grad_norm": 0.6427422761917114
    },
    {
      "bits_per_byte": 1.776176280217984,
      "learning_rate": 0.00024480179857784177,
      "loss_nats": 1.2311515808105469,
      "step": 2670,
      "unclipped_grad_norm": 0.695833146572113
    },
    {
      "bits_per_byte": 1.8751694940546166,
      "learning_rate": 0.00024435100905948387,
      "loss_nats": 1.2997684478759766,
      "step": 2680,
      "unclipped_grad_norm": 0.7215304374694824
    },
    {
      "bits_per_byte": 1.7935356930481232,
      "learning_rate": 0.00024389886365131793,
      "loss_nats": 1.243184208869934,
      "step": 2690,
      "unclipped_grad_norm": 0.6506277322769165
    },
    {
      "bits_per_byte": 1.7071520751901008,
      "learning_rate": 0.00024344537007926163,
      "loss_nats": 1.1833076477050781,
      "step": 2700,
      "unclipped_grad_norm": 0.6119946241378784
    },
    {
      "bits_per_byte": 1.7912357690581349,
      "learning_rate": 0.00024299053609226878,
      "loss_nats": 1.2415900230407715,
      "step": 2710,
      "unclipped_grad_norm": 0.6407736539840698
    },
    {
      "bits_per_byte": 1.8449512823847667,
      "learning_rate": 0.00024253436946219733,
      "loss_nats": 1.2788227796554565,
      "step": 2720,
      "unclipped_grad_norm": 0.699180543422699
    },
    {
      "bits_per_byte": 1.7188499911364874,
      "learning_rate": 0.0002420768779836764,
      "loss_nats": 1.1914160251617432,
      "step": 2730,
      "unclipped_grad_norm": 0.6055090427398682
    },
    {
      "bits_per_byte": 1.7885469922945458,
      "learning_rate": 0.00024161806947397305,
      "loss_nats": 1.2397263050079346,
      "step": 2740,
      "unclipped_grad_norm": 0.6273350715637207
    },
    {
      "bits_per_byte": 1.7444217475279766,
      "learning_rate": 0.00024115795177285886,
      "loss_nats": 1.2091410160064697,
      "step": 2750,
      "unclipped_grad_norm": 0.623899519443512
    },
    {
      "bits_per_byte": 1.855314440977807,
      "learning_rate": 0.0002406965327424758,
      "loss_nats": 1.286005973815918,
      "step": 2760,
      "unclipped_grad_norm": 0.6717104315757751
    },
    {
      "bits_per_byte": 1.79349716893433,
      "learning_rate": 0.00024023382026720197,
      "loss_nats": 1.2431575059890747,
      "step": 2770,
      "unclipped_grad_norm": 0.6304712891578674
    },
    {
      "bits_per_byte": 1.8011758503300526,
      "learning_rate": 0.00023976982225351698,
      "loss_nats": 1.248479962348938,
      "step": 2780,
      "unclipped_grad_norm": 0.6547600626945496
    },
    {
      "bits_per_byte": 1.7772979510669111,
      "learning_rate": 0.0002393045466298666,
      "loss_nats": 1.231929063796997,
      "step": 2790,
      "unclipped_grad_norm": 0.6679553389549255
    },
    {
      "bits_per_byte": 1.784543924113063,
      "learning_rate": 0.00023883800134652763,
      "loss_nats": 1.2369515895843506,
      "step": 2800,
      "unclipped_grad_norm": 0.6488308310508728
    },
    {
      "bits_per_byte": 1.723108969502454,
      "learning_rate": 0.0002383701943754715,
      "loss_nats": 1.1943681240081787,
      "step": 2810,
      "unclipped_grad_norm": 0.5890340209007263
    },
    {
      "bits_per_byte": 1.7962786443467342,
      "learning_rate": 0.00023790113371022884,
      "loss_nats": 1.2450854778289795,
      "step": 2820,
      "unclipped_grad_norm": 0.678447961807251
    },
    {
      "bits_per_byte": 1.7361824026904413,
      "learning_rate": 0.0002374308273657521,
      "loss_nats": 1.203429937362671,
      "step": 2830,
      "unclipped_grad_norm": 0.5870603322982788
    },
    {
      "bits_per_byte": 1.7695838412451108,
      "learning_rate": 0.00023695928337827911,
      "loss_nats": 1.2265820503234863,
      "step": 2840,
      "unclipped_grad_norm": 0.6424716711044312
    },
    {
      "bits_per_byte": 1.7215556221998611,
      "learning_rate": 0.00023648650980519564,
      "loss_nats": 1.193291425704956,
      "step": 2850,
      "unclipped_grad_norm": 0.621953010559082
    },
    {
      "bits_per_byte": 1.750462466156882,
      "learning_rate": 0.00023601251472489757,
      "loss_nats": 1.2133281230926514,
      "step": 2860,
      "unclipped_grad_norm": 0.6744791269302368
    },
    {
      "bits_per_byte": 1.7458160108785215,
      "learning_rate": 0.00023553730623665305,
      "loss_nats": 1.210107445716858,
      "step": 2870,
      "unclipped_grad_norm": 0.6091592311859131
    },
    {
      "bits_per_byte": 1.7563069525811528,
      "learning_rate": 0.00023506089246046403,
      "loss_nats": 1.2173792123794556,
      "step": 2880,
      "unclipped_grad_norm": 0.6055625081062317
    },
    {
      "bits_per_byte": 1.7055577960166026,
      "learning_rate": 0.00023458328153692736,
      "loss_nats": 1.1822025775909424,
      "step": 2890,
      "unclipped_grad_norm": 0.5587822794914246
    },
    {
      "bits_per_byte": 1.7494334939567697,
      "learning_rate": 0.0002341044816270961,
      "loss_nats": 1.212614893913269,
      "step": 2900,
      "unclipped_grad_norm": 0.7476353049278259
    },
    {
      "bits_per_byte": 1.7069099236176861,
      "learning_rate": 0.00023362450091233953,
      "loss_nats": 1.1831398010253906,
      "step": 2910,
      "unclipped_grad_norm": 0.6083102226257324
    },
    {
      "bits_per_byte": 1.7573092674703812,
      "learning_rate": 0.00023314334759420388,
      "loss_nats": 1.2180739641189575,
      "step": 2920,
      "unclipped_grad_norm": 0.594536542892456
    },
    {
      "bits_per_byte": 1.7148656690639488,
      "learning_rate": 0.0002326610298942717,
      "loss_nats": 1.1886543035507202,
      "step": 2930,
      "unclipped_grad_norm": 0.6017132997512817
    },
    {
      "bits_per_byte": 1.7206826382640814,
      "learning_rate": 0.00023217755605402175,
      "loss_nats": 1.1926863193511963,
      "step": 2940,
      "unclipped_grad_norm": 0.6191802620887756
    },
    {
      "bits_per_byte": 1.7777722792179909,
      "learning_rate": 0.00023169293433468802,
      "loss_nats": 1.2322578430175781,
      "step": 2950,
      "unclipped_grad_norm": 0.6343991756439209
    },
    {
      "bits_per_byte": 1.7965679191654853,
      "learning_rate": 0.00023120717301711857,
      "loss_nats": 1.245285987854004,
      "step": 2960,
      "unclipped_grad_norm": 0.6511562466621399
    },
    {
      "bits_per_byte": 1.750740734085978,
      "learning_rate": 0.00023072028040163408,
      "loss_nats": 1.2135210037231445,
      "step": 2970,
      "unclipped_grad_norm": 0.6213991641998291
    },
    {
      "bits_per_byte": 1.7817877301503355,
      "learning_rate": 0.00023023226480788593,
      "loss_nats": 1.2350411415100098,
      "step": 2980,
      "unclipped_grad_norm": 0.6211501359939575
    },
    {
      "bits_per_byte": 1.724867836072828,
      "learning_rate": 0.00022974313457471416,
      "loss_nats": 1.1955872774124146,
      "step": 2990,
      "unclipped_grad_norm": 0.642371416091919
    },
    {
      "bits_per_byte": 1.795713509355999,
      "learning_rate": 0.0002292528980600049,
      "loss_nats": 1.2446937561035156,
      "step": 3000,
      "unclipped_grad_norm": 0.6221276521682739
    },
    {
      "bits_per_byte": 1.6848799779380654,
      "learning_rate": 0.00022876156364054758,
      "loss_nats": 1.1678698062896729,
      "step": 3010,
      "unclipped_grad_norm": 0.6145718693733215
    },
    {
      "bits_per_byte": 1.752584732068529,
      "learning_rate": 0.0002282691397118918,
      "loss_nats": 1.214799165725708,
      "step": 3020,
      "unclipped_grad_norm": 0.6433340907096863
    },
    {
      "bits_per_byte": 1.6945199494842276,
      "learning_rate": 0.00022777563468820376,
      "loss_nats": 1.1745517253875732,
      "step": 3030,
      "unclipped_grad_norm": 0.6049435138702393
    },
    {
      "bits_per_byte": 1.6808318503020565,
      "learning_rate": 0.00022728105700212282,
      "loss_nats": 1.1650638580322266,
      "step": 3040,
      "unclipped_grad_norm": 0.5896543860435486
    },
    {
      "bits_per_byte": 1.8340489581812716,
      "learning_rate": 0.00022678541510461698,
      "loss_nats": 1.2712658643722534,
      "step": 3050,
      "unclipped_grad_norm": 0.6642063856124878
    },
    {
      "bits_per_byte": 1.7718811854953362,
      "learning_rate": 0.00022628871746483876,
      "loss_nats": 1.2281744480133057,
      "step": 3060,
      "unclipped_grad_norm": 0.6303152441978455
    },
    {
      "bits_per_byte": 1.6793462641639036,
      "learning_rate": 0.0002257909725699804,
      "loss_nats": 1.164034128189087,
      "step": 3070,
      "unclipped_grad_norm": 0.6531770825386047
    },
    {
      "bits_per_byte": 1.767122941493915,
      "learning_rate": 0.00022529218892512886,
      "loss_nats": 1.2248762845993042,
      "step": 3080,
      "unclipped_grad_norm": 0.6228736042976379
    },
    {
      "bits_per_byte": 1.7547854220689696,
      "learning_rate": 0.00022479237505312034,
      "loss_nats": 1.2163245677947998,
      "step": 3090,
      "unclipped_grad_norm": 0.6808046102523804
    },
    {
      "bits_per_byte": 1.7115178547822532,
      "learning_rate": 0.00022429153949439496,
      "loss_nats": 1.1863337755203247,
      "step": 3100,
      "unclipped_grad_norm": 0.6307847499847412
    },
    {
      "bits_per_byte": 1.8621731090942157,
      "learning_rate": 0.00022378969080685053,
      "loss_nats": 1.2907600402832031,
      "step": 3110,
      "unclipped_grad_norm": 0.652417004108429
    },
    {
      "bits_per_byte": 1.7445247651358435,
      "learning_rate": 0.0002232868375656965,
      "loss_nats": 1.2092124223709106,
      "step": 3120,
      "unclipped_grad_norm": 0.6338313221931458
    },
    {
      "bits_per_byte": 1.6854237870800937,
      "learning_rate": 0.00022278298836330733,
      "loss_nats": 1.1682467460632324,
      "step": 3130,
      "unclipped_grad_norm": 0.6034438610076904
    },
    {
      "bits_per_byte": 1.717318657613205,
      "learning_rate": 0.00022227815180907572,
      "loss_nats": 1.190354585647583,
      "step": 3140,
      "unclipped_grad_norm": 0.6048492193222046
    },
    {
      "bits_per_byte": 1.7027365926118492,
      "learning_rate": 0.00022177233652926546,
      "loss_nats": 1.1802470684051514,
      "step": 3150,
      "unclipped_grad_norm": 0.5864679217338562
    },
    {
      "bits_per_byte": 1.705547477057551,
      "learning_rate": 0.00022126555116686416,
      "loss_nats": 1.1821954250335693,
      "step": 3160,
      "unclipped_grad_norm": 0.6051758527755737
    },
    {
      "bits_per_byte": 1.7744621291368343,
      "learning_rate": 0.00022075780438143532,
      "loss_nats": 1.2299634218215942,
      "step": 3170,
      "unclipped_grad_norm": 0.6531821489334106
    },
    {
      "bits_per_byte": 1.7458908233316468,
      "learning_rate": 0.00022024910484897065,
      "loss_nats": 1.2101593017578125,
      "step": 3180,
      "unclipped_grad_norm": 0.6303388476371765
    },
    {
      "bits_per_byte": 1.7361681281304195,
      "learning_rate": 0.00021973946126174158,
      "loss_nats": 1.2034200429916382,
      "step": 3190,
      "unclipped_grad_norm": 0.678484320640564
    },
    {
      "bits_per_byte": 1.7289813171161645,
      "learning_rate": 0.00021922888232815086,
      "loss_nats": 1.1984385251998901,
      "step": 3200,
      "unclipped_grad_norm": 0.6228032112121582
    },
    {
      "bits_per_byte": 1.6986354943193747,
      "learning_rate": 0.0002187173767725837,
      "loss_nats": 1.1774044036865234,
      "step": 3210,
      "unclipped_grad_norm": 0.6199705004692078
    },
    {
      "bits_per_byte": 1.6704626723162361,
      "learning_rate": 0.0002182049533352588,
      "loss_nats": 1.1578764915466309,
      "step": 3220,
      "unclipped_grad_norm": 0.6396132111549377
    },
    {
      "bits_per_byte": 1.7446800654695727,
      "learning_rate": 0.00021769162077207884,
      "loss_nats": 1.209320068359375,
      "step": 3230,
      "unclipped_grad_norm": 0.6783336400985718
    },
    {
      "bits_per_byte": 1.7503792265538642,
      "learning_rate": 0.00021717738785448102,
      "loss_nats": 1.2132704257965088,
      "step": 3240,
      "unclipped_grad_norm": 0.6652441620826721
    },
    {
      "bits_per_byte": 1.7183588086856232,
      "learning_rate": 0.00021666226336928708,
      "loss_nats": 1.1910755634307861,
      "step": 3250,
      "unclipped_grad_norm": 0.63749098777771
    },
    {
      "bits_per_byte": 1.7194732562632142,
      "learning_rate": 0.0002161462561185532,
      "loss_nats": 1.1918480396270752,
      "step": 3260,
      "unclipped_grad_norm": 0.6323399543762207
    },
    {
      "bits_per_byte": 1.76014147776479,
      "learning_rate": 0.00021562937491941957,
      "loss_nats": 1.2200371026992798,
      "step": 3270,
      "unclipped_grad_norm": 0.668753445148468
    },
    {
      "bits_per_byte": 1.7466587258677493,
      "learning_rate": 0.00021511162860395975,
      "loss_nats": 1.2106915712356567,
      "step": 3280,
      "unclipped_grad_norm": 0.5981830954551697
    },
    {
      "bits_per_byte": 1.7077491989538964,
      "learning_rate": 0.0002145930260190298,
      "loss_nats": 1.1837215423583984,
      "step": 3290,
      "unclipped_grad_norm": 0.6258406639099121
    },
    {
      "bits_per_byte": 1.7122833495612435,
      "learning_rate": 0.00021407357602611698,
      "loss_nats": 1.1868643760681152,
      "step": 3300,
      "unclipped_grad_norm": 0.6118162870407104
    },
    {
      "bits_per_byte": 1.6971689982554674,
      "learning_rate": 0.0002135532875011885,
      "loss_nats": 1.176387906074524,
      "step": 3310,
      "unclipped_grad_norm": 0.614425003528595
    },
    {
      "bits_per_byte": 1.7078183359795434,
      "learning_rate": 0.0002130321693345397,
      "loss_nats": 1.1837694644927979,
      "step": 3320,
      "unclipped_grad_norm": 0.6319419741630554
    },
    {
      "bits_per_byte": 1.7065344854908524,
      "learning_rate": 0.00021251023043064224,
      "loss_nats": 1.1828795671463013,
      "step": 3330,
      "unclipped_grad_norm": 0.6474409699440002
    },
    {
      "bits_per_byte": 1.6476438422171078,
      "learning_rate": 0.00021198747970799188,
      "loss_nats": 1.1420596837997437,
      "step": 3340,
      "unclipped_grad_norm": 0.6128159165382385
    },
    {
      "bits_per_byte": 1.693542744062025,
      "learning_rate": 0.00021146392609895618,
      "loss_nats": 1.1738743782043457,
      "step": 3350,
      "unclipped_grad_norm": 0.6108464002609253
    },
    {
      "bits_per_byte": 1.7204735073606323,
      "learning_rate": 0.00021093957854962178,
      "loss_nats": 1.1925413608551025,
      "step": 3360,
      "unclipped_grad_norm": 0.6041978597640991
    },
    {
      "bits_per_byte": 1.6622183399818258,
      "learning_rate": 0.00021041444601964157,
      "loss_nats": 1.152161955833435,
      "step": 3370,
      "unclipped_grad_norm": 0.5887104272842407
    },
    {
      "bits_per_byte": 1.7058931621857851,
      "learning_rate": 0.00020988853748208164,
      "loss_nats": 1.1824350357055664,
      "step": 3380,
      "unclipped_grad_norm": 0.6352498531341553
    },
    {
      "bits_per_byte": 1.707757282138487,
      "learning_rate": 0.00020936186192326782,
      "loss_nats": 1.1837271451950073,
      "step": 3390,
      "unclipped_grad_norm": 0.6421219706535339
    },
    {
      "bits_per_byte": 1.6979798964542856,
      "learning_rate": 0.00020883442834263225,
      "loss_nats": 1.1769499778747559,
      "step": 3400,
      "unclipped_grad_norm": 0.6395401954650879
    },
    {
      "bits_per_byte": 1.6959995162296002,
      "learning_rate": 0.0002083062457525596,
      "loss_nats": 1.1755772829055786,
      "step": 3410,
      "unclipped_grad_norm": 0.6283812522888184
    },
    {
      "bits_per_byte": 1.6799083034669233,
      "learning_rate": 0.00020777732317823306,
      "loss_nats": 1.1644237041473389,
      "step": 3420,
      "unclipped_grad_norm": 0.6197249293327332
    },
    {
      "bits_per_byte": 1.71363410130112,
      "learning_rate": 0.00020724766965748005,
      "loss_nats": 1.187800645828247,
      "step": 3430,
      "unclipped_grad_norm": 0.6102274060249329
    },
    {
      "bits_per_byte": 1.7091876618457131,
      "learning_rate": 0.00020671729424061788,
      "loss_nats": 1.1847186088562012,
      "step": 3440,
      "unclipped_grad_norm": 0.6624795198440552
    },
    {
      "bits_per_byte": 1.7982115573597812,
      "learning_rate": 0.00020618620599029906,
      "loss_nats": 1.2464252710342407,
      "step": 3450,
      "unclipped_grad_norm": 0.6986591815948486
    },
    {
      "bits_per_byte": 1.7759950105039746,
      "learning_rate": 0.00020565441398135649,
      "loss_nats": 1.2310259342193604,
      "step": 3460,
      "unclipped_grad_norm": 0.6196012496948242
    },
    {
      "bits_per_byte": 1.7475674821949083,
      "learning_rate": 0.00020512192730064827,
      "loss_nats": 1.211321473121643,
      "step": 3470,
      "unclipped_grad_norm": 0.6346544623374939
    },
    {
      "bits_per_byte": 1.7169992858305527,
      "learning_rate": 0.00020458875504690273,
      "loss_nats": 1.1901332139968872,
      "step": 3480,
      "unclipped_grad_norm": 0.6380975246429443
    },
    {
      "bits_per_byte": 1.6316707815355735,
      "learning_rate": 0.00020405490633056245,
      "loss_nats": 1.1309880018234253,
      "step": 3490,
      "unclipped_grad_norm": 0.6554663181304932
    },
    {
      "bits_per_byte": 1.641168867394774,
      "learning_rate": 0.00020352039027362917,
      "loss_nats": 1.1375715732574463,
      "step": 3500,
      "unclipped_grad_norm": 0.6088140606880188
    },
    {
      "bits_per_byte": 1.6880614849963764,
      "learning_rate": 0.0002029852160095075,
      "loss_nats": 1.1700750589370728,
      "step": 3510,
      "unclipped_grad_norm": 0.6358106732368469
    },
    {
      "bits_per_byte": 1.6616972325497115,
      "learning_rate": 0.000202449392682849,
      "loss_nats": 1.1518007516860962,
      "step": 3520,
      "unclipped_grad_norm": 0.6153261661529541
    },
    {
      "bits_per_byte": 1.6712895649015846,
      "learning_rate": 0.0002019129294493959,
      "loss_nats": 1.158449649810791,
      "step": 3530,
      "unclipped_grad_norm": 0.6411516666412354
    },
    {
      "bits_per_byte": 1.6785265948498915,
      "learning_rate": 0.00020137583547582478,
      "loss_nats": 1.163465976715088,
      "step": 3540,
      "unclipped_grad_norm": 0.6034340858459473
    },
    {
      "bits_per_byte": 1.7376134703282706,
      "learning_rate": 0.00020083811993958973,
      "loss_nats": 1.204421877861023,
      "step": 3550,
      "unclipped_grad_norm": 0.6545450687408447
    },
    {
      "bits_per_byte": 1.6707187544833708,
      "learning_rate": 0.00020029979202876563,
      "loss_nats": 1.158053994178772,
      "step": 3560,
      "unclipped_grad_norm": 0.6340785026550293
    },
    {
      "bits_per_byte": 1.7138805524398064,
      "learning_rate": 0.00019976086094189118,
      "loss_nats": 1.1879714727401733,
      "step": 3570,
      "unclipped_grad_norm": 0.6533586382865906
    },
    {
      "bits_per_byte": 1.7826314770354685,
      "learning_rate": 0.00019922133588781175,
      "loss_nats": 1.235625982284546,
      "step": 3580,
      "unclipped_grad_norm": 0.7123501896858215
    },
    {
      "bits_per_byte": 1.7131767994324758,
      "learning_rate": 0.0001986812260855219,
      "loss_nats": 1.1874836683273315,
      "step": 3590,
      "unclipped_grad_norm": 0.6637008786201477
    },
    {
      "bits_per_byte": 1.6959864455481348,
      "learning_rate": 0.00019814054076400787,
      "loss_nats": 1.1755682229995728,
      "step": 3600,
      "unclipped_grad_norm": 0.6217906475067139
    },
    {
      "bits_per_byte": 1.6716725702650561,
      "learning_rate": 0.00019759928916209004,
      "loss_nats": 1.1587151288986206,
      "step": 3610,
      "unclipped_grad_norm": 0.6237639784812927
    },
    {
      "bits_per_byte": 1.6746752153664697,
      "learning_rate": 0.00019705748052826493,
      "loss_nats": 1.1607964038848877,
      "step": 3620,
      "unclipped_grad_norm": 0.6187261343002319
    },
    {
      "bits_per_byte": 1.6883108598401275,
      "learning_rate": 0.00019651512412054723,
      "loss_nats": 1.1702479124069214,
      "step": 3630,
      "unclipped_grad_norm": 0.6150439381599426
    },
    {
      "bits_per_byte": 1.6436062055228018,
      "learning_rate": 0.00019597222920631144,
      "loss_nats": 1.13926100730896,
      "step": 3640,
      "unclipped_grad_norm": 0.6298535466194153
    },
    {
      "bits_per_byte": 1.7182858880416574,
      "learning_rate": 0.0001954288050621338,
      "loss_nats": 1.1910250186920166,
      "step": 3650,
      "unclipped_grad_norm": 0.6331585049629211
    },
    {
      "bits_per_byte": 1.6224437403341332,
      "learning_rate": 0.00019488486097363363,
      "loss_nats": 1.1245923042297363,
      "step": 3660,
      "unclipped_grad_norm": 0.6383513808250427
    },
    {
      "bits_per_byte": 1.644355533932611,
      "learning_rate": 0.00019434040623531452,
      "loss_nats": 1.1397804021835327,
      "step": 3670,
      "unclipped_grad_norm": 0.6061356067657471
    },
    {
      "bits_per_byte": 1.7035681287287707,
      "learning_rate": 0.0001937954501504058,
      "loss_nats": 1.1808234453201294,
      "step": 3680,
      "unclipped_grad_norm": 0.6479222774505615
    },
    {
      "bits_per_byte": 1.6706625161565387,
      "learning_rate": 0.00019325000203070333,
      "loss_nats": 1.1580150127410889,
      "step": 3690,
      "unclipped_grad_norm": 0.6192170977592468
    },
    {
      "bits_per_byte": 1.6109532354646872,
      "learning_rate": 0.0001927040711964106,
      "loss_nats": 1.1166276931762695,
      "step": 3700,
      "unclipped_grad_norm": 0.6093165278434753
    },
    {
      "bits_per_byte": 1.6773731072105547,
      "learning_rate": 0.00019215766697597923,
      "loss_nats": 1.1626664400100708,
      "step": 3710,
      "unclipped_grad_norm": 0.6192633509635925
    },
    {
      "bits_per_byte": 1.6614944650043442,
      "learning_rate": 0.00019161079870594978,
      "loss_nats": 1.1516602039337158,
      "step": 3720,
      "unclipped_grad_norm": 0.6275249719619751
    },
    {
      "bits_per_byte": 1.7267034069054867,
      "learning_rate": 0.0001910634757307922,
      "loss_nats": 1.19685959815979,
      "step": 3730,
      "unclipped_grad_norm": 0.6384285092353821
    },
    {
      "bits_per_byte": 1.6506055554476162,
      "learning_rate": 0.0001905157074027459,
      "loss_nats": 1.1441125869750977,
      "step": 3740,
      "unclipped_grad_norm": 0.6508066654205322
    },
    {
      "bits_per_byte": 1.6398485565841001,
      "learning_rate": 0.00018996750308166037,
      "loss_nats": 1.136656403541565,
      "step": 3750,
      "unclipped_grad_norm": 0.6484293937683105
    },
    {
      "bits_per_byte": 1.6392961483095287,
      "learning_rate": 0.00018941887213483482,
      "loss_nats": 1.1362735033035278,
      "step": 3760,
      "unclipped_grad_norm": 0.6600143909454346
    },
    {
      "bits_per_byte": 1.686776946577082,
      "learning_rate": 0.00018886982393685845,
      "loss_nats": 1.169184684753418,
      "step": 3770,
      "unclipped_grad_norm": 0.6462109684944153
    },
    {
      "bits_per_byte": 1.6300704829692951,
      "learning_rate": 0.00018832036786945008,
      "loss_nats": 1.1298787593841553,
      "step": 3780,
      "unclipped_grad_norm": 0.6114524006843567
    },
    {
      "bits_per_byte": 1.675651388892767,
      "learning_rate": 0.0001877705133212979,
      "loss_nats": 1.161473035812378,
      "step": 3790,
      "unclipped_grad_norm": 0.6200737953186035
    },
    {
      "bits_per_byte": 1.617114685914498,
      "learning_rate": 0.00018722026968789906,
      "loss_nats": 1.1208984851837158,
      "step": 3800,
      "unclipped_grad_norm": 0.6216682195663452
    },
    {
      "bits_per_byte": 1.6234193979124778,
      "learning_rate": 0.00018666964637139907,
      "loss_nats": 1.125268578529358,
      "step": 3810,
      "unclipped_grad_norm": 0.6173998117446899
    },
    {
      "bits_per_byte": 1.7138243141129743,
      "learning_rate": 0.00018611865278043115,
      "loss_nats": 1.1879324913024902,
      "step": 3820,
      "unclipped_grad_norm": 0.6088318228721619
    },
    {
      "bits_per_byte": 1.6341469877426964,
      "learning_rate": 0.00018556729832995555,
      "loss_nats": 1.1327043771743774,
      "step": 3830,
      "unclipped_grad_norm": 0.6438922882080078
    },
    {
      "bits_per_byte": 1.655027229401299,
      "learning_rate": 0.00018501559244109864,
      "loss_nats": 1.1471774578094482,
      "step": 3840,
      "unclipped_grad_norm": 0.669468879699707
    },
    {
      "bits_per_byte": 1.6818487837166085,
      "learning_rate": 0.00018446354454099178,
      "loss_nats": 1.1657687425613403,
      "step": 3850,
      "unclipped_grad_norm": 0.6244412660598755
    },
    {
      "bits_per_byte": 1.6164324307385254,
      "learning_rate": 0.00018391116406261048,
      "loss_nats": 1.1204255819320679,
      "step": 3860,
      "unclipped_grad_norm": 0.6409146785736084
    },
    {
      "bits_per_byte": 1.6824030838003392,
      "learning_rate": 0.00018335846044461305,
      "loss_nats": 1.1661529541015625,
      "step": 3870,
      "unclipped_grad_norm": 0.6731106042861938
    },
    {
      "bits_per_byte": 1.6375393455309653,
      "learning_rate": 0.0001828054431311793,
      "loss_nats": 1.1350557804107666,
      "step": 3880,
      "unclipped_grad_norm": 0.6218310594558716
    },
    {
      "bits_per_byte": 1.6796400105315774,
      "learning_rate": 0.00018225212157184934,
      "loss_nats": 1.1642377376556396,
      "step": 3890,
      "unclipped_grad_norm": 0.635161280632019
    },
    {
      "bits_per_byte": 1.666564169586478,
      "learning_rate": 0.00018169850522136192,
      "loss_nats": 1.1551742553710938,
      "step": 3900,
      "unclipped_grad_norm": 0.7091010808944702
    },
    {
      "bits_per_byte": 1.7026119051899735,
      "learning_rate": 0.000181144603539493,
      "loss_nats": 1.180160641670227,
      "step": 3910,
      "unclipped_grad_norm": 0.6815565824508667
    },
    {
      "bits_per_byte": 1.6485720325838145,
      "learning_rate": 0.00018059042599089412,
      "loss_nats": 1.1427030563354492,
      "step": 3920,
      "unclipped_grad_norm": 0.628564178943634
    },
    {
      "bits_per_byte": 1.6283515163739215,
      "learning_rate": 0.00018003598204493048,
      "loss_nats": 1.1286872625350952,
      "step": 3930,
      "unclipped_grad_norm": 0.6302286982536316
    },
    {
      "bits_per_byte": 1.6363571367889342,
      "learning_rate": 0.00017948128117551936,
      "loss_nats": 1.1342363357543945,
      "step": 3940,
      "unclipped_grad_norm": 0.607494056224823
    },
    {
      "bits_per_byte": 1.7146744243561893,
      "learning_rate": 0.00017892633286096816,
      "loss_nats": 1.1885217428207397,
      "step": 3950,
      "unclipped_grad_norm": 0.70033860206604
    },
    {
      "bits_per_byte": 1.7144123227962744,
      "learning_rate": 0.00017837114658381247,
      "loss_nats": 1.1883400678634644,
      "step": 3960,
      "unclipped_grad_norm": 0.645106315612793
    },
    {
      "bits_per_byte": 1.6837865122438798,
      "learning_rate": 0.00017781573183065386,
      "loss_nats": 1.167111873626709,
      "step": 3970,
      "unclipped_grad_norm": 0.6373703479766846
    },
    {
      "bits_per_byte": 1.6677649524548022,
      "learning_rate": 0.0001772600980919981,
      "loss_nats": 1.1560065746307373,
      "step": 3980,
      "unclipped_grad_norm": 0.6565650701522827
    },
    {
      "bits_per_byte": 1.7059107044161732,
      "learning_rate": 0.0001767042548620927,
      "loss_nats": 1.1824471950531006,
      "step": 3990,
      "unclipped_grad_norm": 0.6542640924453735
    },
    {
      "bits_per_byte": 1.7121254694877515,
      "learning_rate": 0.00017614821163876486,
      "loss_nats": 1.1867549419403076,
      "step": 4000,
      "unclipped_grad_norm": 0.6575196385383606
    },
    {
      "bits_per_byte": 1.7220670986035271,
      "learning_rate": 0.00017559197792325906,
      "loss_nats": 1.19364595413208,
      "step": 4010,
      "unclipped_grad_norm": 0.6806800961494446
    },
    {
      "bits_per_byte": 1.5958445595863893,
      "learning_rate": 0.0001750355632200748,
      "loss_nats": 1.1061551570892334,
      "step": 4020,
      "unclipped_grad_norm": 0.6171497106552124
    },
    {
      "bits_per_byte": 1.6345342926724393,
      "learning_rate": 0.00017447897703680418,
      "loss_nats": 1.1329728364944458,
      "step": 4030,
      "unclipped_grad_norm": 0.6173087954521179
    },
    {
      "bits_per_byte": 1.6363991005557448,
      "learning_rate": 0.0001739222288839694,
      "loss_nats": 1.134265422821045,
      "step": 4040,
      "unclipped_grad_norm": 0.634988009929657
    },
    {
      "bits_per_byte": 1.6319382145576653,
      "learning_rate": 0.00017336532827486026,
      "loss_nats": 1.1311733722686768,
      "step": 4050,
      "unclipped_grad_norm": 0.6698096990585327
    },
    {
      "bits_per_byte": 1.6486795217406036,
      "learning_rate": 0.00017280828472537163,
      "loss_nats": 1.1427775621414185,
      "step": 4060,
      "unclipped_grad_norm": 0.6841239929199219
    },
    {
      "bits_per_byte": 1.6129987971140494,
      "learning_rate": 0.0001722511077538408,
      "loss_nats": 1.1180455684661865,
      "step": 4070,
      "unclipped_grad_norm": 0.6276201605796814
    },
    {
      "bits_per_byte": 1.6508999897458934,
      "learning_rate": 0.00017169380688088497,
      "loss_nats": 1.1443166732788086,
      "step": 4080,
      "unclipped_grad_norm": 0.6372854113578796
    },
    {
      "bits_per_byte": 1.6061886281051838,
      "learning_rate": 0.0001711363916292384,
      "loss_nats": 1.1133251190185547,
      "step": 4090,
      "unclipped_grad_norm": 0.6243290305137634
    },
    {
      "bits_per_byte": 1.578650766049031,
      "learning_rate": 0.00017057887152358974,
      "loss_nats": 1.0942373275756836,
      "step": 4100,
      "unclipped_grad_norm": 0.5791712403297424
    },
    {
      "bits_per_byte": 1.6840439702722212,
      "learning_rate": 0.0001700212560904194,
      "loss_nats": 1.1672903299331665,
      "step": 4110,
      "unclipped_grad_norm": 0.6848294734954834
    },
    {
      "bits_per_byte": 1.6312047085517354,
      "learning_rate": 0.00016946355485783656,
      "loss_nats": 1.1306649446487427,
      "step": 4120,
      "unclipped_grad_norm": 0.6449601054191589
    },
    {
      "bits_per_byte": 1.594994965291127,
      "learning_rate": 0.0001689057773554166,
      "loss_nats": 1.1055662631988525,
      "step": 4130,
      "unclipped_grad_norm": 0.6161995530128479
    },
    {
      "bits_per_byte": 1.6232881751498696,
      "learning_rate": 0.00016834793311403806,
      "loss_nats": 1.1251776218414307,
      "step": 4140,
      "unclipped_grad_norm": 0.6562530398368835
    },
    {
      "bits_per_byte": 1.67032388231699,
      "learning_rate": 0.00016779003166571981,
      "loss_nats": 1.1577802896499634,
      "step": 4150,
      "unclipped_grad_norm": 0.672451376914978
    },
    {
      "bits_per_byte": 1.6420112384187,
      "learning_rate": 0.00016723208254345838,
      "loss_nats": 1.138155460357666,
      "step": 4160,
      "unclipped_grad_norm": 0.6645690202713013
    },
    {
      "bits_per_byte": 1.597898376402993,
      "learning_rate": 0.00016667409528106481,
      "loss_nats": 1.1075787544250488,
      "step": 4170,
      "unclipped_grad_norm": 0.6766495108604431
    },
    {
      "bits_per_byte": 1.5688259131531912,
      "learning_rate": 0.00016611607941300184,
      "loss_nats": 1.0874272584915161,
      "step": 4180,
      "unclipped_grad_norm": 0.6362206339836121
    },
    {
      "bits_per_byte": 1.7152548658028512,
      "learning_rate": 0.0001655580444742211,
      "loss_nats": 1.1889240741729736,
      "step": 4190,
      "unclipped_grad_norm": 0.6535307765007019
    },
    {
      "bits_per_byte": 1.6086450563074575,
      "learning_rate": 0.00016499999999999997,
      "loss_nats": 1.1150277853012085,
      "step": 4200,
      "unclipped_grad_norm": 0.6474415063858032
    },
    {
      "bits_per_byte": 1.6237748860518113,
      "learning_rate": 0.0001644419555257789,
      "loss_nats": 1.1255149841308594,
      "step": 4210,
      "unclipped_grad_norm": 0.6242862343788147
    },
    {
      "bits_per_byte": 1.6672617312183775,
      "learning_rate": 0.0001638839205869981,
      "loss_nats": 1.1556577682495117,
      "step": 4220,
      "unclipped_grad_norm": 0.6519601345062256
    },
    {
      "bits_per_byte": 1.635159105643024,
      "learning_rate": 0.0001633259047189352,
      "loss_nats": 1.1334059238433838,
      "step": 4230,
      "unclipped_grad_norm": 0.6224661469459534
    },
    {
      "bits_per_byte": 1.6806368219759782,
      "learning_rate": 0.0001627679174565416,
      "loss_nats": 1.164928674697876,
      "step": 4240,
      "unclipped_grad_norm": 0.6462982296943665
    },
    {
      "bits_per_byte": 1.7100040634893587,
      "learning_rate": 0.00016220996833428016,
      "loss_nats": 1.1852844953536987,
      "step": 4250,
      "unclipped_grad_norm": 0.6697248220443726
    },
    {
      "bits_per_byte": 1.6765044228410466,
      "learning_rate": 0.00016165206688596191,
      "loss_nats": 1.1620643138885498,
      "step": 4260,
      "unclipped_grad_norm": 0.6539841294288635
    },
    {
      "bits_per_byte": 1.6521546031839376,
      "learning_rate": 0.00016109422264458335,
      "loss_nats": 1.1451863050460815,
      "step": 4270,
      "unclipped_grad_norm": 0.6337559819221497
    },
    {
      "bits_per_byte": 1.688240002987972,
      "learning_rate": 0.00016053644514216336,
      "loss_nats": 1.1701987981796265,
      "step": 4280,
      "unclipped_grad_norm": 0.6939600706100464
    },
    {
      "bits_per_byte": 1.645464994013327,
      "learning_rate": 0.00015997874390958058,
      "loss_nats": 1.1405494213104248,
      "step": 4290,
      "unclipped_grad_norm": 0.6979376673698425
    },
    {
      "bits_per_byte": 1.5687516166480184,
      "learning_rate": 0.00015942112847641018,
      "loss_nats": 1.0873757600784302,
      "step": 4300,
      "unclipped_grad_norm": 0.6490140557289124
    },
    {
      "bits_per_byte": 1.720270739815265,
      "learning_rate": 0.00015886360837076156,
      "loss_nats": 1.1924008131027222,
      "step": 4310,
      "unclipped_grad_norm": 0.7267009615898132
    },
    {
      "bits_per_byte": 1.5912837516681586,
      "learning_rate": 0.00015830619311911495,
      "loss_nats": 1.1029938459396362,
      "step": 4320,
      "unclipped_grad_norm": 0.6906614899635315
    },
    {
      "bits_per_byte": 1.5899403951822693,
      "learning_rate": 0.00015774889224615915,
      "loss_nats": 1.102062702178955,
      "step": 4330,
      "unclipped_grad_norm": 0.6376214027404785
    },
    {
      "bits_per_byte": 1.5963857889886546,
      "learning_rate": 0.00015719171527462834,
      "loss_nats": 1.1065303087234497,
      "step": 4340,
      "unclipped_grad_norm": 0.6380553841590881
    },
    {
      "bits_per_byte": 1.5301035033634773,
      "learning_rate": 0.0001566346717251397,
      "loss_nats": 1.060586929321289,
      "step": 4350,
      "unclipped_grad_norm": 0.6225847601890564
    },
    {
      "bits_per_byte": 1.67761336697381,
      "learning_rate": 0.00015607777111603056,
      "loss_nats": 1.1628329753875732,
      "step": 4360,
      "unclipped_grad_norm": 0.6995339393615723
    },
    {
      "bits_per_byte": 1.6233086410853221,
      "learning_rate": 0.00015552102296319576,
      "loss_nats": 1.1251918077468872,
      "step": 4370,
      "unclipped_grad_norm": 0.6690109372138977
    },
    {
      "bits_per_byte": 1.5971554113512656,
      "learning_rate": 0.00015496443677992516,
      "loss_nats": 1.1070637702941895,
      "step": 4380,
      "unclipped_grad_norm": 0.7015417814254761
    },
    {
      "bits_per_byte": 1.6390723988807563,
      "learning_rate": 0.0001544080220767409,
      "loss_nats": 1.1361184120178223,
      "step": 4390,
      "unclipped_grad_norm": 0.6816950440406799
    },
    {
      "bits_per_byte": 1.5932933689434905,
      "learning_rate": 0.00015385178836123511,
      "loss_nats": 1.104386806488037,
      "step": 4400,
      "unclipped_grad_norm": 0.667495846748352
    },
    {
      "bits_per_byte": 1.5991400911422224,
      "learning_rate": 0.00015329574513790726,
      "loss_nats": 1.1084394454956055,
      "step": 4410,
      "unclipped_grad_norm": 0.6873564124107361
    },
    {
      "bits_per_byte": 1.58255941575519,
      "learning_rate": 0.00015273990190800186,
      "loss_nats": 1.0969465970993042,
      "step": 4420,
      "unclipped_grad_norm": 0.6864304542541504
    },
    {
      "bits_per_byte": 1.5926611607189187,
      "learning_rate": 0.0001521842681693461,
      "loss_nats": 1.1039485931396484,
      "step": 4430,
      "unclipped_grad_norm": 0.6576697826385498
    },
    {
      "bits_per_byte": 1.6095080652494869,
      "learning_rate": 0.00015162885341618752,
      "loss_nats": 1.1156259775161743,
      "step": 4440,
      "unclipped_grad_norm": 0.6515920162200928
    },
    {
      "bits_per_byte": 1.5838350110766395,
      "learning_rate": 0.00015107366713903178,
      "loss_nats": 1.0978307723999023,
      "step": 4450,
      "unclipped_grad_norm": 0.6458138227462769
    },
    {
      "bits_per_byte": 1.5629618207067222,
      "learning_rate": 0.0001505187188244806,
      "loss_nats": 1.0833625793457031,
      "step": 4460,
      "unclipped_grad_norm": 0.6665330529212952
    },
    {
      "bits_per_byte": 1.6502230660320973,
      "learning_rate": 0.00014996401795506952,
      "loss_nats": 1.1438474655151367,
      "step": 4470,
      "unclipped_grad_norm": 0.6849002838134766
    },
    {
      "bits_per_byte": 1.5654218605446637,
      "learning_rate": 0.00014940957400910585,
      "loss_nats": 1.0850677490234375,
      "step": 4480,
      "unclipped_grad_norm": 0.630775511264801
    },
    {
      "bits_per_byte": 1.533044750658533,
      "learning_rate": 0.00014885539646050699,
      "loss_nats": 1.0626256465911865,
      "step": 4490,
      "unclipped_grad_norm": 0.6286258697509766
    },
    {
      "bits_per_byte": 1.6266820807819962,
      "learning_rate": 0.00014830149477863805,
      "loss_nats": 1.1275300979614258,
      "step": 4500,
      "unclipped_grad_norm": 0.6595945954322815
    },
    {
      "bits_per_byte": 1.5885456158837719,
      "learning_rate": 0.00014774787842815066,
      "loss_nats": 1.1010959148406982,
      "step": 4510,
      "unclipped_grad_norm": 0.6231974363327026
    },
    {
      "bits_per_byte": 1.5897587815029581,
      "learning_rate": 0.00014719455686882066,
      "loss_nats": 1.1019368171691895,
      "step": 4520,
      "unclipped_grad_norm": 0.6408888697624207
    },
    {
      "bits_per_byte": 1.6075569220754484,
      "learning_rate": 0.00014664153955538695,
      "loss_nats": 1.1142735481262207,
      "step": 4530,
      "unclipped_grad_norm": 0.6721575260162354
    },
    {
      "bits_per_byte": 1.5571181941957055,
      "learning_rate": 0.00014608883593738946,
      "loss_nats": 1.0793120861053467,
      "step": 4540,
      "unclipped_grad_norm": 0.6309816837310791
    },
    {
      "bits_per_byte": 1.6337729254770696,
      "learning_rate": 0.00014553645545900817,
      "loss_nats": 1.1324450969696045,
      "step": 4550,
      "unclipped_grad_norm": 0.6641245484352112
    },
    {
      "bits_per_byte": 1.6370763682348426,
      "learning_rate": 0.0001449844075589013,
      "loss_nats": 1.134734869003296,
      "step": 4560,
      "unclipped_grad_norm": 0.6858609914779663
    },
    {
      "bits_per_byte": 1.5844336826842929,
      "learning_rate": 0.0001444327016700444,
      "loss_nats": 1.0982457399368286,
      "step": 4570,
      "unclipped_grad_norm": 0.6500639319419861
    },
    {
      "bits_per_byte": 1.5406408803818417,
      "learning_rate": 0.0001438813472195688,
      "loss_nats": 1.0678908824920654,
      "step": 4580,
      "unclipped_grad_norm": 0.636735200881958
    },
    {
      "bits_per_byte": 1.6489479866586005,
      "learning_rate": 0.00014333035362860093,
      "loss_nats": 1.1429636478424072,
      "step": 4590,
      "unclipped_grad_norm": 0.6933116316795349
    },
    {
      "bits_per_byte": 1.6320697812855753,
      "learning_rate": 0.00014277973031210088,
      "loss_nats": 1.131264567375183,
      "step": 4600,
      "unclipped_grad_norm": 0.6834596395492554
    },
    {
      "bits_per_byte": 1.589729888417613,
      "learning_rate": 0.00014222948667870207,
      "loss_nats": 1.101916790008545,
      "step": 4610,
      "unclipped_grad_norm": 0.6632579565048218
    },
    {
      "bits_per_byte": 1.5335797886853673,
      "learning_rate": 0.00014167963213054991,
      "loss_nats": 1.062996506690979,
      "step": 4620,
      "unclipped_grad_norm": 0.6395919322967529
    },
    {
      "bits_per_byte": 1.6054043872172494,
      "learning_rate": 0.00014113017606314152,
      "loss_nats": 1.1127815246582031,
      "step": 4630,
      "unclipped_grad_norm": 0.6490065455436707
    },
    {
      "bits_per_byte": 1.5840957367753474,
      "learning_rate": 0.00014058112786516518,
      "loss_nats": 1.0980114936828613,
      "step": 4640,
      "unclipped_grad_norm": 0.6492030024528503
    },
    {
      "bits_per_byte": 1.5576186637097162,
      "learning_rate": 0.0001400324969183396,
      "loss_nats": 1.0796589851379395,
      "step": 4650,
      "unclipped_grad_norm": 0.6488379836082458
    },
    {
      "bits_per_byte": 1.593877078060519,
      "learning_rate": 0.00013948429259725406,
      "loss_nats": 1.1047914028167725,
      "step": 4660,
      "unclipped_grad_norm": 0.6992486119270325
    },
    {
      "bits_per_byte": 1.6105064245377456,
      "learning_rate": 0.00013893652426920776,
      "loss_nats": 1.1163179874420166,
      "step": 4670,
      "unclipped_grad_norm": 0.6828540563583374
    },
    {
      "bits_per_byte": 1.5810454524796447,
      "learning_rate": 0.00013838920129405016,
      "loss_nats": 1.0958971977233887,
      "step": 4680,
      "unclipped_grad_norm": 0.6880136728286743
    },
    {
      "bits_per_byte": 1.5551754781715592,
      "learning_rate": 0.0001378423330240207,
      "loss_nats": 1.077965497970581,
      "step": 4690,
      "unclipped_grad_norm": 0.6890891194343567
    },
    {
      "bits_per_byte": 1.6623815515174944,
      "learning_rate": 0.00013729592880358936,
      "loss_nats": 1.1522750854492188,
      "step": 4700,
      "unclipped_grad_norm": 0.6787605285644531
    },
    {
      "bits_per_byte": 1.574140349047503,
      "learning_rate": 0.00013674999796929659,
      "loss_nats": 1.0911109447479248,
      "step": 4710,
      "unclipped_grad_norm": 0.6684292554855347
    },
    {
      "bits_per_byte": 1.5498508953008467,
      "learning_rate": 0.00013620454984959417,
      "loss_nats": 1.0742747783660889,
      "step": 4720,
      "unclipped_grad_norm": 0.6828400492668152
    },
    {
      "bits_per_byte": 1.6704473658603094,
      "learning_rate": 0.00013565959376468542,
      "loss_nats": 1.1578658819198608,
      "step": 4730,
      "unclipped_grad_norm": 0.6912305355072021
    },
    {
      "bits_per_byte": 1.6014281483293014,
      "learning_rate": 0.00013511513902636634,
      "loss_nats": 1.110025405883789,
      "step": 4740,
      "unclipped_grad_norm": 0.6723529100418091
    },
    {
      "bits_per_byte": 1.639597977861793,
      "learning_rate": 0.0001345711949378662,
      "loss_nats": 1.1364827156066895,
      "step": 4750,
      "unclipped_grad_norm": 0.7447375059127808
    },
    {
      "bits_per_byte": 1.523300557625948,
      "learning_rate": 0.00013402777079368853,
      "loss_nats": 1.0558714866638184,
      "step": 4760,
      "unclipped_grad_norm": 0.6460341811180115
    },
    {
      "bits_per_byte": 1.6314774730360038,
      "learning_rate": 0.0001334848758794528,
      "loss_nats": 1.1308540105819702,
      "step": 4770,
      "unclipped_grad_norm": 0.6882368922233582
    },
    {
      "bits_per_byte": 1.5726310293035308,
      "learning_rate": 0.00013294251947173504,
      "loss_nats": 1.0900647640228271,
      "step": 4780,
      "unclipped_grad_norm": 0.6284239888191223
    },
    {
      "bits_per_byte": 1.60157536547844,
      "learning_rate": 0.00013240071083790996,
      "loss_nats": 1.1101274490356445,
      "step": 4790,
      "unclipped_grad_norm": 0.6737696528434753
    },
    {
      "bits_per_byte": 1.577140070443852,
      "learning_rate": 0.0001318594592359921,
      "loss_nats": 1.0931901931762695,
      "step": 4800,
      "unclipped_grad_norm": 0.6438981890678406
    },
    {
      "bits_per_byte": 1.5444138357764698,
      "learning_rate": 0.0001313187739144781,
      "loss_nats": 1.0705060958862305,
      "step": 4810,
      "unclipped_grad_norm": 0.690973699092865
    },
    {
      "bits_per_byte": 1.6096637095485178,
      "learning_rate": 0.0001307786641121882,
      "loss_nats": 1.1157338619232178,
      "step": 4820,
      "unclipped_grad_norm": 0.6775097250938416
    },
    {
      "bits_per_byte": 1.6126245628657718,
      "learning_rate": 0.00013023913905810876,
      "loss_nats": 1.117786169052124,
      "step": 4830,
      "unclipped_grad_norm": 0.7250232696533203
    },
    {
      "bits_per_byte": 1.6110075819823597,
      "learning_rate": 0.00012970020797123432,
      "loss_nats": 1.1166653633117676,
      "step": 4840,
      "unclipped_grad_norm": 0.6946505308151245
    },
    {
      "bits_per_byte": 1.560937584906067,
      "learning_rate": 0.00012916188006041027,
      "loss_nats": 1.0819594860076904,
      "step": 4850,
      "unclipped_grad_norm": 0.7250073552131653
    },
    {
      "bits_per_byte": 1.5705242418304615,
      "learning_rate": 0.00012862416452417516,
      "loss_nats": 1.08860445022583,
      "step": 4860,
      "unclipped_grad_norm": 0.6968505382537842
    },
    {
      "bits_per_byte": 1.5589560008028256,
      "learning_rate": 0.00012808707055060405,
      "loss_nats": 1.0805859565734863,
      "step": 4870,
      "unclipped_grad_norm": 0.659229040145874
    },
    {
      "bits_per_byte": 1.7343542271117696,
      "learning_rate": 0.000127550607317151,
      "loss_nats": 1.202162742614746,
      "step": 4880,
      "unclipped_grad_norm": 0.7630480527877808
    },
    {
      "bits_per_byte": 1.5109135071975544,
      "learning_rate": 0.00012701478399049248,
      "loss_nats": 1.0472854375839233,
      "step": 4890,
      "unclipped_grad_norm": 0.6373679041862488
    },
    {
      "bits_per_byte": 1.6030399697331876,
      "learning_rate": 0.0001264796097263708,
      "loss_nats": 1.111142635345459,
      "step": 4900,
      "unclipped_grad_norm": 0.6882696151733398
    },
    {
      "bits_per_byte": 1.6017267102111992,
      "learning_rate": 0.0001259450936694375,
      "loss_nats": 1.1102323532104492,
      "step": 4910,
      "unclipped_grad_norm": 0.7222312688827515
    },
    {
      "bits_per_byte": 1.533604038239139,
      "learning_rate": 0.00012541124495309727,
      "loss_nats": 1.0630133152008057,
      "step": 4920,
      "unclipped_grad_norm": 0.6319286227226257
    },
    {
      "bits_per_byte": 1.5357895937663035,
      "learning_rate": 0.00012487807269935167,
      "loss_nats": 1.064528226852417,
      "step": 4930,
      "unclipped_grad_norm": 0.6290988922119141
    },
    {
      "bits_per_byte": 1.5644580697692285,
      "learning_rate": 0.00012434558601864349,
      "loss_nats": 1.084399700164795,
      "step": 4940,
      "unclipped_grad_norm": 0.6635708808898926
    },
    {
      "bits_per_byte": 1.596738869370876,
      "learning_rate": 0.0001238137940097009,
      "loss_nats": 1.1067750453948975,
      "step": 4950,
      "unclipped_grad_norm": 0.7171201109886169
    },
    {
      "bits_per_byte": 1.6163304450265636,
      "learning_rate": 0.0001232827057593821,
      "loss_nats": 1.1203548908233643,
      "step": 4960,
      "unclipped_grad_norm": 0.739273726940155
    },
    {
      "bits_per_byte": 1.566945798813959,
      "learning_rate": 0.00012275233034251992,
      "loss_nats": 1.086124062538147,
      "step": 4970,
      "unclipped_grad_norm": 0.6495019197463989
    },
    {
      "bits_per_byte": 1.623716067985216,
      "learning_rate": 0.0001222226768217669,
      "loss_nats": 1.125474214553833,
      "step": 4980,
      "unclipped_grad_norm": 0.7210032939910889
    },
    {
      "bits_per_byte": 1.574822088275523,
      "learning_rate": 0.00012169375424744034,
      "loss_nats": 1.091583490371704,
      "step": 4990,
      "unclipped_grad_norm": 0.6862797737121582
    },
    {
      "bits_per_byte": 1.5840169687212522,
      "learning_rate": 0.00012116557165736772,
      "loss_nats": 1.097956895828247,
      "step": 5000,
      "unclipped_grad_norm": 0.6914816498756409
    },
    {
      "bits_per_byte": 1.6270380848692823,
      "learning_rate": 0.00012063813807673214,
      "loss_nats": 1.127776861190796,
      "step": 5010,
      "unclipped_grad_norm": 0.7159916758537292
    },
    {
      "bits_per_byte": 1.5917599716283977,
      "learning_rate": 0.00012011146251791832,
      "loss_nats": 1.1033239364624023,
      "step": 5020,
      "unclipped_grad_norm": 0.7277734875679016
    },
    {
      "bits_per_byte": 1.5241281381418998,
      "learning_rate": 0.0001195855539803584,
      "loss_nats": 1.0564451217651367,
      "step": 5030,
      "unclipped_grad_norm": 0.6716130375862122
    },
    {
      "bits_per_byte": 1.5837488477685573,
      "learning_rate": 0.00011906042145037817,
      "loss_nats": 1.0977710485458374,
      "step": 5040,
      "unclipped_grad_norm": 0.6623762845993042
    },
    {
      "bits_per_byte": 1.554003760371231,
      "learning_rate": 0.0001185360739010438,
      "loss_nats": 1.0771533250808716,
      "step": 5050,
      "unclipped_grad_norm": 0.7006968855857849
    },
    {
      "bits_per_byte": 1.621560093474,
      "learning_rate": 0.00011801252029200808,
      "loss_nats": 1.1239798069000244,
      "step": 5060,
      "unclipped_grad_norm": 0.6823760271072388
    },
    {
      "bits_per_byte": 1.5589410383122004,
      "learning_rate": 0.00011748976956935777,
      "loss_nats": 1.0805755853652954,
      "step": 5070,
      "unclipped_grad_norm": 0.685271680355072
    },
    {
      "bits_per_byte": 1.6273230601217619,
      "learning_rate": 0.00011696783066546024,
      "loss_nats": 1.1279743909835815,
      "step": 5080,
      "unclipped_grad_norm": 0.725353479385376
    },
    {
      "bits_per_byte": 1.5689380458415538,
      "learning_rate": 0.00011644671249881147,
      "loss_nats": 1.0875049829483032,
      "step": 5090,
      "unclipped_grad_norm": 0.6860542297363281
    },
    {
      "bits_per_byte": 1.5571642855461367,
      "learning_rate": 0.00011592642397388293,
      "loss_nats": 1.0793440341949463,
      "step": 5100,
      "unclipped_grad_norm": 0.7099642753601074
    },
    {
      "bits_per_byte": 1.593192587110085,
      "learning_rate": 0.00011540697398097015,
      "loss_nats": 1.1043169498443604,
      "step": 5110,
      "unclipped_grad_norm": 0.683228075504303
    },
    {
      "bits_per_byte": 1.5025353723607726,
      "learning_rate": 0.00011488837139604016,
      "loss_nats": 1.041478157043457,
      "step": 5120,
      "unclipped_grad_norm": 0.6536243557929993
    },
    {
      "bits_per_byte": 1.6259317204762818,
      "learning_rate": 0.00011437062508058039,
      "loss_nats": 1.1270099878311157,
      "step": 5130,
      "unclipped_grad_norm": 0.7276868224143982
    },
    {
      "bits_per_byte": 1.5937286570328242,
      "learning_rate": 0.00011385374388144673,
      "loss_nats": 1.1046885251998901,
      "step": 5140,
      "unclipped_grad_norm": 0.7137117981910706
    },
    {
      "bits_per_byte": 1.5363523209999266,
      "learning_rate": 0.00011333773663071286,
      "loss_nats": 1.0649182796478271,
      "step": 5150,
      "unclipped_grad_norm": 0.6770874857902527
    },
    {
      "bits_per_byte": 1.5548322008004372,
      "learning_rate": 0.00011282261214551897,
      "loss_nats": 1.0777275562286377,
      "step": 5160,
      "unclipped_grad_norm": 0.6993823051452637
    },
    {
      "bits_per_byte": 1.6209285731800318,
      "learning_rate": 0.00011230837922792112,
      "loss_nats": 1.123542070388794,
      "step": 5170,
      "unclipped_grad_norm": 0.7498713135719299
    },
    {
      "bits_per_byte": 1.5887963665887297,
      "learning_rate": 0.0001117950466647412,
      "loss_nats": 1.1012697219848633,
      "step": 5180,
      "unclipped_grad_norm": 0.7006720304489136
    },
    {
      "bits_per_byte": 1.4814539110006604,
      "learning_rate": 0.00011128262322741627,
      "loss_nats": 1.0268656015396118,
      "step": 5190,
      "unclipped_grad_norm": 0.6951001286506653
    },
    {
      "bits_per_byte": 1.5501159205658264,
      "learning_rate": 0.00011077111767184913,
      "loss_nats": 1.0744584798812866,
      "step": 5200,
      "unclipped_grad_norm": 0.696289598941803
    },
    {
      "bits_per_byte": 1.519997458833477,
      "learning_rate": 0.00011026053873825837,
      "loss_nats": 1.053581953048706,
      "step": 5210,
      "unclipped_grad_norm": 0.6860307455062866
    },
    {
      "bits_per_byte": 1.5330389032484038,
      "learning_rate": 0.0001097508951510293,
      "loss_nats": 1.0626215934753418,
      "step": 5220,
      "unclipped_grad_norm": 0.6861689686775208
    },
    {
      "bits_per_byte": 1.6136053799236425,
      "learning_rate": 0.0001092421956185646,
      "loss_nats": 1.1184660196304321,
      "step": 5230,
      "unclipped_grad_norm": 0.6980286836624146
    },
    {
      "bits_per_byte": 1.5805682006235005,
      "learning_rate": 0.00010873444883313581,
      "loss_nats": 1.0955663919448853,
      "step": 5240,
      "unclipped_grad_norm": 0.7267829775810242
    },
    {
      "bits_per_byte": 1.5736680846882336,
      "learning_rate": 0.00010822766347073447,
      "loss_nats": 1.0907835960388184,
      "step": 5250,
      "unclipped_grad_norm": 0.6905968189239502
    },
    {
      "bits_per_byte": 1.5160989561037188,
      "learning_rate": 0.00010772184819092424,
      "loss_nats": 1.050879716873169,
      "step": 5260,
      "unclipped_grad_norm": 0.7048349380493164
    },
    {
      "bits_per_byte": 1.5706724908755052,
      "learning_rate": 0.0001072170116366926,
      "loss_nats": 1.0887072086334229,
      "step": 5270,
      "unclipped_grad_norm": 0.7007996439933777
    },
    {
      "bits_per_byte": 1.4955876172312168,
      "learning_rate": 0.00010671316243430345,
      "loss_nats": 1.0366623401641846,
      "step": 5280,
      "unclipped_grad_norm": 0.7013132572174072
    },
    {
      "bits_per_byte": 1.524417756925953,
      "learning_rate": 0.00010621030919314944,
      "loss_nats": 1.0566458702087402,
      "step": 5290,
      "unclipped_grad_norm": 0.6891999840736389
    },
    {
      "bits_per_byte": 1.4950008124264729,
      "learning_rate": 0.000105708460505605,
      "loss_nats": 1.0362555980682373,
      "step": 5300,
      "unclipped_grad_norm": 0.6865445971488953
    },
    {
      "bits_per_byte": 1.4986692023693766,
      "learning_rate": 0.00010520762494687964,
      "loss_nats": 1.0387983322143555,
      "step": 5310,
      "unclipped_grad_norm": 0.6979682445526123
    },
    {
      "bits_per_byte": 1.5140539104023092,
      "learning_rate": 0.00010470781107487111,
      "loss_nats": 1.0494621992111206,
      "step": 5320,
      "unclipped_grad_norm": 0.7112694978713989
    },
    {
      "bits_per_byte": 1.5563401446832021,
      "learning_rate": 0.00010420902743001955,
      "loss_nats": 1.078772783279419,
      "step": 5330,
      "unclipped_grad_norm": 0.7123240828514099
    },
    {
      "bits_per_byte": 1.620746271570117,
      "learning_rate": 0.00010371128253516117,
      "loss_nats": 1.1234157085418701,
      "step": 5340,
      "unclipped_grad_norm": 0.7256788611412048
    },
    {
      "bits_per_byte": 1.4811187168141287,
      "learning_rate": 0.000103214584895383,
      "loss_nats": 1.0266332626342773,
      "step": 5350,
      "unclipped_grad_norm": 0.6512540578842163
    },
    {
      "bits_per_byte": 1.5334795227999143,
      "learning_rate": 0.00010271894299787715,
      "loss_nats": 1.062927007675171,
      "step": 5360,
      "unclipped_grad_norm": 0.7001609206199646
    },
    {
      "bits_per_byte": 1.6031871868823262,
      "learning_rate": 0.00010222436531179622,
      "loss_nats": 1.1112446784973145,
      "step": 5370,
      "unclipped_grad_norm": 0.7270175814628601
    },
    {
      "bits_per_byte": 1.5079560935333174,
      "learning_rate": 0.0001017308602881082,
      "loss_nats": 1.045235514640808,
      "step": 5380,
      "unclipped_grad_norm": 0.6764644980430603
    },
    {
      "bits_per_byte": 1.5113635857948622,
      "learning_rate": 0.00010123843635945242,
      "loss_nats": 1.0475974082946777,
      "step": 5390,
      "unclipped_grad_norm": 0.7015690803527832
    },
    {
      "bits_per_byte": 1.567986121869028,
      "learning_rate": 0.00010074710193999505,
      "loss_nats": 1.0868451595306396,
      "step": 5400,
      "unclipped_grad_norm": 0.7621549963951111
    },
    {
      "bits_per_byte": 1.456367317667255,
      "learning_rate": 0.0001002568654252858,
      "loss_nats": 1.009476900100708,
      "step": 5410,
      "unclipped_grad_norm": 0.7006528377532959
    },
    {
      "bits_per_byte": 1.6203708334432836,
      "learning_rate": 9.976773519211404e-05,
      "loss_nats": 1.1231554746627808,
      "step": 5420,
      "unclipped_grad_norm": 0.7240427136421204
    },
    {
      "bits_per_byte": 1.543870886547696,
      "learning_rate": 9.92797195983659e-05,
      "loss_nats": 1.0701297521591187,
      "step": 5430,
      "unclipped_grad_norm": 0.7349725961685181
    },
    {
      "bits_per_byte": 1.5391332804643783,
      "learning_rate": 9.879282698288142e-05,
      "loss_nats": 1.0668458938598633,
      "step": 5440,
      "unclipped_grad_norm": 0.6845248937606812
    },
    {
      "bits_per_byte": 1.5639070373558641,
      "learning_rate": 9.830706566531197e-05,
      "loss_nats": 1.0840177536010742,
      "step": 5450,
      "unclipped_grad_norm": 0.7343961596488953
    },
    {
      "bits_per_byte": 1.5791133993798518,
      "learning_rate": 9.782244394597826e-05,
      "loss_nats": 1.0945580005645752,
      "step": 5460,
      "unclipped_grad_norm": 0.6990759968757629
    },
    {
      "bits_per_byte": 1.5450971228483479,
      "learning_rate": 9.733897010572829e-05,
      "loss_nats": 1.0709797143936157,
      "step": 5470,
      "unclipped_grad_norm": 0.73023521900177
    },
    {
      "bits_per_byte": 1.4896457886265577,
      "learning_rate": 9.685665240579612e-05,
      "loss_nats": 1.0325437784194946,
      "step": 5480,
      "unclipped_grad_norm": 0.7134303450584412
    },
    {
      "bits_per_byte": 1.5553481487530256,
      "learning_rate": 9.637549908766041e-05,
      "loss_nats": 1.07808518409729,
      "step": 5490,
      "unclipped_grad_norm": 0.713401734828949
    },
    {
      "bits_per_byte": 1.5796455137016214,
      "learning_rate": 9.589551837290389e-05,
      "loss_nats": 1.0949268341064453,
      "step": 5500,
      "unclipped_grad_norm": 0.7012893557548523
    },
    {
      "bits_per_byte": 1.56096613402611,
      "learning_rate": 9.541671846307257e-05,
      "loss_nats": 1.0819792747497559,
      "step": 5510,
      "unclipped_grad_norm": 0.7354693412780762
    },
    {
      "bits_per_byte": 1.558417523122974,
      "learning_rate": 9.493910753953597e-05,
      "loss_nats": 1.0802127122879028,
      "step": 5520,
      "unclipped_grad_norm": 0.8108682036399841
    },
    {
      "bits_per_byte": 1.572987033390817,
      "learning_rate": 9.446269376334689e-05,
      "loss_nats": 1.0903115272521973,
      "step": 5530,
      "unclipped_grad_norm": 0.813023567199707
    },
    {
      "bits_per_byte": 1.603627462468535,
      "learning_rate": 9.39874852751024e-05,
      "loss_nats": 1.1115498542785645,
      "step": 5540,
      "unclipped_grad_norm": 0.7556710243225098
    },
    {
      "bits_per_byte": 1.5354511319094055,
      "learning_rate": 9.351349019480433e-05,
      "loss_nats": 1.064293622970581,
      "step": 5550,
      "unclipped_grad_norm": 0.6996996998786926
    },
    {
      "bits_per_byte": 1.531031521747533,
      "learning_rate": 9.304071662172086e-05,
      "loss_nats": 1.061230182647705,
      "step": 5560,
      "unclipped_grad_norm": 0.7085531949996948
    },
    {
      "bits_per_byte": 1.5727634559446952,
      "learning_rate": 9.256917263424787e-05,
      "loss_nats": 1.0901565551757812,
      "step": 5570,
      "unclipped_grad_norm": 0.7508514523506165
    },
    {
      "bits_per_byte": 1.4695356852785186,
      "learning_rate": 9.209886628977114e-05,
      "loss_nats": 1.0186045169830322,
      "step": 5580,
      "unclipped_grad_norm": 0.6981281042098999
    },
    {
      "bits_per_byte": 1.5444360215384312,
      "learning_rate": 9.162980562452843e-05,
      "loss_nats": 1.0705214738845825,
      "step": 5590,
      "unclipped_grad_norm": 0.7099512815475464
    },
    {
      "bits_per_byte": 1.5326933901028204,
      "learning_rate": 9.116199865347238e-05,
      "loss_nats": 1.0623821020126343,
      "step": 5600,
      "unclipped_grad_norm": 0.7117766737937927
    },
    {
      "bits_per_byte": 1.5419961036706404,
      "learning_rate": 9.069545337013333e-05,
      "loss_nats": 1.0688302516937256,
      "step": 5610,
      "unclipped_grad_norm": 0.7040424942970276
    },
    {
      "bits_per_byte": 1.4775775940328635,
      "learning_rate": 9.0230177746483e-05,
      "loss_nats": 1.0241787433624268,
      "step": 5620,
      "unclipped_grad_norm": 0.7147260308265686
    },
    {
      "bits_per_byte": 1.4921469323180554,
      "learning_rate": 8.976617973279797e-05,
      "loss_nats": 1.0342774391174316,
      "step": 5630,
      "unclipped_grad_norm": 0.7185394763946533
    },
    {
      "bits_per_byte": 1.4712231790487844,
      "learning_rate": 8.930346725752419e-05,
      "loss_nats": 1.0197741985321045,
      "step": 5640,
      "unclipped_grad_norm": 0.7179238796234131
    },
    {
      "bits_per_byte": 1.5902296700010203,
      "learning_rate": 8.88420482271411e-05,
      "loss_nats": 1.1022632122039795,
      "step": 5650,
      "unclipped_grad_norm": 0.7511428594589233
    },
    {
      "bits_per_byte": 1.541337238135185,
      "learning_rate": 8.838193052602691e-05,
      "loss_nats": 1.0683735609054565,
      "step": 5660,
      "unclipped_grad_norm": 0.7174038887023926
    },
    {
      "bits_per_byte": 1.5341244577406499,
      "learning_rate": 8.792312201632352e-05,
      "loss_nats": 1.0633740425109863,
      "step": 5670,
      "unclipped_grad_norm": 0.7094559669494629
    },
    {
      "bits_per_byte": 1.5432753106277581,
      "learning_rate": 8.746563053780261e-05,
      "loss_nats": 1.0697169303894043,
      "step": 5680,
      "unclipped_grad_norm": 0.7527599334716797
    },
    {
      "bits_per_byte": 1.5847210656938846,
      "learning_rate": 8.700946390773114e-05,
      "loss_nats": 1.098444938659668,
      "step": 5690,
      "unclipped_grad_norm": 0.7476640939712524
    },
    {
      "bits_per_byte": 1.5107903676195364,
      "learning_rate": 8.655462992073833e-05,
      "loss_nats": 1.047200083732605,
      "step": 5700,
      "unclipped_grad_norm": 0.7240637540817261
    },
    {
      "bits_per_byte": 1.5529789157547396,
      "learning_rate": 8.610113634868199e-05,
      "loss_nats": 1.0764429569244385,
      "step": 5710,
      "unclipped_grad_norm": 0.7409384250640869
    },
    {
      "bits_per_byte": 1.5779322225337262,
      "learning_rate": 8.564899094051613e-05,
      "loss_nats": 1.0937392711639404,
      "step": 5720,
      "unclipped_grad_norm": 0.7523439526557922
    },
    {
      "bits_per_byte": 1.4812139952027068,
      "learning_rate": 8.519820142215816e-05,
      "loss_nats": 1.0266993045806885,
      "step": 5730,
      "unclipped_grad_norm": 0.6902204155921936
    },
    {
      "bits_per_byte": 1.5538140635073292,
      "learning_rate": 8.474877549635728e-05,
      "loss_nats": 1.077021837234497,
      "step": 5740,
      "unclipped_grad_norm": 0.7577133178710938
    },
    {
      "bits_per_byte": 1.5073047952345,
      "learning_rate": 8.430072084256247e-05,
      "loss_nats": 1.0447840690612793,
      "step": 5750,
      "unclipped_grad_norm": 0.7320472002029419
    },
    {
      "bits_per_byte": 1.5094478710469015,
      "learning_rate": 8.385404511679161e-05,
      "loss_nats": 1.0462695360183716,
      "step": 5760,
      "unclipped_grad_norm": 0.7613655924797058
    },
    {
      "bits_per_byte": 1.4626881960344158,
      "learning_rate": 8.340875595150034e-05,
      "loss_nats": 1.0138581991195679,
      "step": 5770,
      "unclipped_grad_norm": 0.7212797403335571
    },
    {
      "bits_per_byte": 1.5541401426133652,
      "learning_rate": 8.296486095545203e-05,
      "loss_nats": 1.0772478580474854,
      "step": 5780,
      "unclipped_grad_norm": 0.7401144504547119
    },
    {
      "bits_per_byte": 1.4976115090665703,
      "learning_rate": 8.252236771358729e-05,
      "loss_nats": 1.0380651950836182,
      "step": 5790,
      "unclipped_grad_norm": 0.6966454386711121
    },
    {
      "bits_per_byte": 1.508392929466509,
      "learning_rate": 8.208128378689484e-05,
      "loss_nats": 1.045538306236267,
      "step": 5800,
      "unclipped_grad_norm": 0.7519288063049316
    },
    {
      "bits_per_byte": 1.536275616737642,
      "learning_rate": 8.16416167122819e-05,
      "loss_nats": 1.0648651123046875,
      "step": 5810,
      "unclipped_grad_norm": 0.7322137355804443
    },
    {
      "bits_per_byte": 1.5318357126229676,
      "learning_rate": 8.120337400244578e-05,
      "loss_nats": 1.0617876052856445,
      "step": 5820,
      "unclipped_grad_norm": 0.7490501999855042
    },
    {
      "bits_per_byte": 1.5069907549140245,
      "learning_rate": 8.076656314574521e-05,
      "loss_nats": 1.0445663928985596,
      "step": 5830,
      "unclipped_grad_norm": 0.7249871492385864
    },
    {
      "bits_per_byte": 1.4694756633333674,
      "learning_rate": 8.03311916060725e-05,
      "loss_nats": 1.018562912940979,
      "step": 5840,
      "unclipped_grad_norm": 0.7599647641181946
    },
    {
      "bits_per_byte": 1.4762989030236986,
      "learning_rate": 7.989726682272606e-05,
      "loss_nats": 1.0232924222946167,
      "step": 5850,
      "unclipped_grad_norm": 0.7097748517990112
    },
    {
      "bits_per_byte": 1.502543627528014,
      "learning_rate": 7.946479621028318e-05,
      "loss_nats": 1.0414838790893555,
      "step": 5860,
      "unclipped_grad_norm": 0.7764433026313782
    },
    {
      "bits_per_byte": 1.5481047554466367,
      "learning_rate": 7.903378715847343e-05,
      "loss_nats": 1.0730644464492798,
      "step": 5870,
      "unclipped_grad_norm": 0.770169734954834
    },
    {
      "bits_per_byte": 1.5544658777540994,
      "learning_rate": 7.860424703205224e-05,
      "loss_nats": 1.0774736404418945,
      "step": 5880,
      "unclipped_grad_norm": 0.731796145439148
    },
    {
      "bits_per_byte": 1.528056737835559,
      "learning_rate": 7.817618317067534e-05,
      "loss_nats": 1.0591682195663452,
      "step": 5890,
      "unclipped_grad_norm": 0.7195150852203369
    },
    {
      "bits_per_byte": 1.5162458292875556,
      "learning_rate": 7.774960288877295e-05,
      "loss_nats": 1.0509815216064453,
      "step": 5900,
      "unclipped_grad_norm": 0.8138624429702759
    },
    {
      "bits_per_byte": 1.5072279189895643,
      "learning_rate": 7.732451347542522e-05,
      "loss_nats": 1.04473078250885,
      "step": 5910,
      "unclipped_grad_norm": 0.7780150771141052
    },
    {
      "bits_per_byte": 1.514708304388842,
      "learning_rate": 7.690092219423722e-05,
      "loss_nats": 1.0499157905578613,
      "step": 5920,
      "unclipped_grad_norm": 0.7354658246040344
    },
    {
      "bits_per_byte": 1.5573187259666117,
      "learning_rate": 7.647883628321536e-05,
      "loss_nats": 1.079451084136963,
      "step": 5930,
      "unclipped_grad_norm": 0.7978370785713196
    },
    {
      "bits_per_byte": 1.5533937379086207,
      "learning_rate": 7.605826295464316e-05,
      "loss_nats": 1.076730489730835,
      "step": 5940,
      "unclipped_grad_norm": 0.8458540439605713
    },
    {
      "bits_per_byte": 1.5137727187681485,
      "learning_rate": 7.563920939495851e-05,
      "loss_nats": 1.049267292022705,
      "step": 5950,
      "unclipped_grad_norm": 0.7532646059989929
    },
    {
      "bits_per_byte": 1.5086940710881698,
      "learning_rate": 7.522168276463047e-05,
      "loss_nats": 1.0457470417022705,
      "step": 5960,
      "unclipped_grad_norm": 0.7333019375801086
    },
    {
      "bits_per_byte": 1.5303000795334134,
      "learning_rate": 7.480569019803719e-05,
      "loss_nats": 1.0607231855392456,
      "step": 5970,
      "unclipped_grad_norm": 0.8049731254577637
    },
    {
      "bits_per_byte": 1.4828514420215715,
      "learning_rate": 7.439123880334388e-05,
      "loss_nats": 1.0278342962265015,
      "step": 5980,
      "unclipped_grad_norm": 0.7353474497795105
    },
    {
      "bits_per_byte": 1.4545697550004368,
      "learning_rate": 7.39783356623813e-05,
      "loss_nats": 1.0082309246063232,
      "step": 5990,
      "unclipped_grad_norm": 0.737419068813324
    },
    {
      "bits_per_byte": 1.5419821730759207,
      "learning_rate": 7.356698783052497e-05,
      "loss_nats": 1.068820595741272,
      "step": 6000,
      "unclipped_grad_norm": 0.7629180550575256
    },
    {
      "bits_per_byte": 1.5277340983825405,
      "learning_rate": 7.315720233657431e-05,
      "loss_nats": 1.058944582939148,
      "step": 6010,
      "unclipped_grad_norm": 0.7620755434036255
    },
    {
      "bits_per_byte": 1.4955477172562166,
      "learning_rate": 7.274898618263286e-05,
      "loss_nats": 1.0366346836090088,
      "step": 6020,
      "unclipped_grad_norm": 0.7292536497116089
    },
    {
      "bits_per_byte": 1.5533901262729524,
      "learning_rate": 7.234234634398831e-05,
      "loss_nats": 1.0767279863357544,
      "step": 6030,
      "unclipped_grad_norm": 0.767517626285553
    },
    {
      "bits_per_byte": 1.495873108431649,
      "learning_rate": 7.193728976899362e-05,
      "loss_nats": 1.0368602275848389,
      "step": 6040,
      "unclipped_grad_norm": 0.7536795735359192
    },
    {
      "bits_per_byte": 1.5050292927809341,
      "learning_rate": 7.153382337894797e-05,
      "loss_nats": 1.043206810951233,
      "step": 6050,
      "unclipped_grad_norm": 0.7510635256767273
    },
    {
      "bits_per_byte": 1.556982155918873,
      "learning_rate": 7.113195406797887e-05,
      "loss_nats": 1.079217791557312,
      "step": 6060,
      "unclipped_grad_norm": 0.7961422801017761
    },
    {
      "bits_per_byte": 1.5009287104364122,
      "learning_rate": 7.073168870292397e-05,
      "loss_nats": 1.0403645038604736,
      "step": 6070,
      "unclipped_grad_norm": 0.7310256958007812
    },
    {
      "bits_per_byte": 1.5023740526342633,
      "learning_rate": 7.033303412321404e-05,
      "loss_nats": 1.0413663387298584,
      "step": 6080,
      "unclipped_grad_norm": 0.7653129696846008
    },
    {
      "bits_per_byte": 1.5019286175685285,
      "learning_rate": 6.993599714075586e-05,
      "loss_nats": 1.0410575866699219,
      "step": 6090,
      "unclipped_grad_norm": 0.7687012553215027
    },
    {
      "bits_per_byte": 1.5089062976793344,
      "learning_rate": 6.954058453981609e-05,
      "loss_nats": 1.0458941459655762,
      "step": 6100,
      "unclipped_grad_norm": 0.7417535185813904
    },
    {
      "bits_per_byte": 1.475459455704837,
      "learning_rate": 6.914680307690504e-05,
      "loss_nats": 1.0227105617523193,
      "step": 6110,
      "unclipped_grad_norm": 0.7481279969215393
    },
    {
      "bits_per_byte": 1.4820735644917191,
      "learning_rate": 6.875465948066139e-05,
      "loss_nats": 1.0272951126098633,
      "step": 6120,
      "unclipped_grad_norm": 0.7211357951164246
    },
    {
      "bits_per_byte": 1.5036838725032344,
      "learning_rate": 6.836416045173733e-05,
      "loss_nats": 1.0422742366790771,
      "step": 6130,
      "unclipped_grad_norm": 0.7536711096763611
    },
    {
      "bits_per_byte": 1.5397933498783898,
      "learning_rate": 6.797531266268377e-05,
      "loss_nats": 1.0673034191131592,
      "step": 6140,
      "unclipped_grad_norm": 0.7828236818313599
    },
    {
      "bits_per_byte": 1.4767353949915882,
      "learning_rate": 6.758812275783663e-05,
      "loss_nats": 1.0235949754714966,
      "step": 6150,
      "unclipped_grad_norm": 0.7367910742759705
    },
    {
      "bits_per_byte": 1.563501158299828,
      "learning_rate": 6.720259735320298e-05,
      "loss_nats": 1.0837364196777344,
      "step": 6160,
      "unclipped_grad_norm": 0.7933995723724365
    },
    {
      "bits_per_byte": 1.433251129564785,
      "learning_rate": 6.681874303634842e-05,
      "loss_nats": 0.9934539794921875,
      "step": 6170,
      "unclipped_grad_norm": 0.728959321975708
    },
    {
      "bits_per_byte": 1.4945863342378936,
      "learning_rate": 6.643656636628402e-05,
      "loss_nats": 1.03596830368042,
      "step": 6180,
      "unclipped_grad_norm": 0.7535345554351807
    },
    {
      "bits_per_byte": 1.5074559679846085,
      "learning_rate": 6.605607387335466e-05,
      "loss_nats": 1.0448888540267944,
      "step": 6190,
      "unclipped_grad_norm": 0.7919867038726807
    },
    {
      "bits_per_byte": 1.4628087558726706,
      "learning_rate": 6.567727205912724e-05,
      "loss_nats": 1.013941764831543,
      "step": 6200,
      "unclipped_grad_norm": 0.7595178484916687
    },
    {
      "bits_per_byte": 1.4945321597028718,
      "learning_rate": 6.530016739627959e-05,
      "loss_nats": 1.0359307527542114,
      "step": 6210,
      "unclipped_grad_norm": 0.7536692023277283
    },
    {
      "bits_per_byte": 1.4821234394604692,
      "learning_rate": 6.492476632848993e-05,
      "loss_nats": 1.027329683303833,
      "step": 6220,
      "unclipped_grad_norm": 0.7611709833145142
    },
    {
      "bits_per_byte": 1.4502368240945993,
      "learning_rate": 6.455107527032678e-05,
      "loss_nats": 1.0052275657653809,
      "step": 6230,
      "unclipped_grad_norm": 0.7465225458145142
    },
    {
      "bits_per_byte": 1.4792031740488187,
      "learning_rate": 6.417910060713926e-05,
      "loss_nats": 1.0253055095672607,
      "step": 6240,
      "unclipped_grad_norm": 0.8008144497871399
    },
    {
      "bits_per_byte": 1.4709335602647315,
      "learning_rate": 6.380884869494799e-05,
      "loss_nats": 1.019573450088501,
      "step": 6250,
      "unclipped_grad_norm": 0.7485248446464539
    },
    {
      "bits_per_byte": 1.5311833824282448,
      "learning_rate": 6.34403258603367e-05,
      "loss_nats": 1.0613354444503784,
      "step": 6260,
      "unclipped_grad_norm": 0.7729804515838623
    },
    {
      "bits_per_byte": 1.494225858601685,
      "learning_rate": 6.30735384003437e-05,
      "loss_nats": 1.0357184410095215,
      "step": 6270,
      "unclipped_grad_norm": 0.7719584703445435
    },
    {
      "bits_per_byte": 1.606436799070379,
      "learning_rate": 6.270849258235484e-05,
      "loss_nats": 1.1134971380233765,
      "step": 6280,
      "unclipped_grad_norm": 0.814758837223053
    },
    {
      "bits_per_byte": 1.469587624039079,
      "learning_rate": 6.234519464399581e-05,
      "loss_nats": 1.0186405181884766,
      "step": 6290,
      "unclipped_grad_norm": 0.7629152536392212
    },
    {
      "bits_per_byte": 1.4932348945674137,
      "learning_rate": 6.198365079302614e-05,
      "loss_nats": 1.0350315570831299,
      "step": 6300,
      "unclipped_grad_norm": 0.7575296759605408
    },
    {
      "bits_per_byte": 1.5423225267419782,
      "learning_rate": 6.16238672072326e-05,
      "loss_nats": 1.069056510925293,
      "step": 6310,
      "unclipped_grad_norm": 0.7936079502105713
    },
    {
      "bits_per_byte": 1.497591903044372,
      "learning_rate": 6.126585003432406e-05,
      "loss_nats": 1.0380516052246094,
      "step": 6320,
      "unclipped_grad_norm": 0.7546678781509399
    },
    {
      "bits_per_byte": 1.438224007914483,
      "learning_rate": 6.090960539182614e-05,
      "loss_nats": 0.9969009160995483,
      "step": 6330,
      "unclipped_grad_norm": 0.7539783716201782
    },
    {
      "bits_per_byte": 1.5101449167308483,
      "learning_rate": 6.0555139366976935e-05,
      "loss_nats": 1.046752691268921,
      "step": 6340,
      "unclipped_grad_norm": 0.7760982513427734
    },
    {
      "bits_per_byte": 1.5028686747381446,
      "learning_rate": 6.020245801662274e-05,
      "loss_nats": 1.0417091846466064,
      "step": 6350,
      "unclipped_grad_norm": 0.8006681203842163
    },
    {
      "bits_per_byte": 1.4519638738745637,
      "learning_rate": 5.985156736711483e-05,
      "loss_nats": 1.0064246654510498,
      "step": 6360,
      "unclipped_grad_norm": 0.7430040240287781
    },
    {
      "bits_per_byte": 1.5063248380898837,
      "learning_rate": 5.950247341420618e-05,
      "loss_nats": 1.044104814529419,
      "step": 6370,
      "unclipped_grad_norm": 0.8066008687019348
    },
    {
      "bits_per_byte": 1.461864399136783,
      "learning_rate": 5.915518212294935e-05,
      "loss_nats": 1.0132871866226196,
      "step": 6380,
      "unclipped_grad_norm": 0.739668071269989
    },
    {
      "bits_per_byte": 1.518872692296834,
      "learning_rate": 5.880969942759425e-05,
      "loss_nats": 1.052802324295044,
      "step": 6390,
      "unclipped_grad_norm": 0.8261381983757019
    },
    {
      "bits_per_byte": 1.4734527621345699,
      "learning_rate": 5.846603123148688e-05,
      "loss_nats": 1.0213196277618408,
      "step": 6400,
      "unclipped_grad_norm": 0.7401824593544006
    },
    {
      "bits_per_byte": 1.4482483606853236,
      "learning_rate": 5.812418340696858e-05,
      "loss_nats": 1.0038492679595947,
      "step": 6410,
      "unclipped_grad_norm": 0.7752094268798828
    },
    {
      "bits_per_byte": 1.4492296936911466,
      "learning_rate": 5.778416179527541e-05,
      "loss_nats": 1.0045294761657715,
      "step": 6420,
      "unclipped_grad_norm": 0.7460422515869141
    },
    {
      "bits_per_byte": 1.4798391658917094,
      "learning_rate": 5.7445972206438635e-05,
      "loss_nats": 1.0257463455200195,
      "step": 6430,
      "unclipped_grad_norm": 0.7759443521499634
    },
    {
      "bits_per_byte": 1.4925646781770012,
      "learning_rate": 5.710962041918516e-05,
      "loss_nats": 1.0345669984817505,
      "step": 6440,
      "unclipped_grad_norm": 0.7866507768630981
    },
    {
      "bits_per_byte": 1.473837315341899,
      "learning_rate": 5.677511218083908e-05,
      "loss_nats": 1.0215861797332764,
      "step": 6450,
      "unclipped_grad_norm": 0.7885896563529968
    },
    {
      "bits_per_byte": 1.452434934355277,
      "learning_rate": 5.644245320722316e-05,
      "loss_nats": 1.0067511796951294,
      "step": 6460,
      "unclipped_grad_norm": 0.7844802737236023
    },
    {
      "bits_per_byte": 1.4767341911130323,
      "learning_rate": 5.61116491825615e-05,
      "loss_nats": 1.0235941410064697,
      "step": 6470,
      "unclipped_grad_norm": 0.7592610716819763
    },
    {
      "bits_per_byte": 1.5086992305676956,
      "learning_rate": 5.5782705759382104e-05,
      "loss_nats": 1.045750617980957,
      "step": 6480,
      "unclipped_grad_norm": 0.8035558462142944
    },
    {
      "bits_per_byte": 1.4908353926225757,
      "learning_rate": 5.5455628558420546e-05,
      "loss_nats": 1.0333683490753174,
      "step": 6490,
      "unclipped_grad_norm": 0.8066015243530273
    },
    {
      "bits_per_byte": 1.4884610001447638,
      "learning_rate": 5.513042316852368e-05,
      "loss_nats": 1.0317225456237793,
      "step": 6500,
      "unclipped_grad_norm": 0.7627172470092773
    },
    {
      "bits_per_byte": 1.4363769142442164,
      "learning_rate": 5.480709514655443e-05,
      "loss_nats": 0.995620608329773,
      "step": 6510,
      "unclipped_grad_norm": 0.7699123024940491
    },
    {
      "bits_per_byte": 1.4159479550614777,
      "learning_rate": 5.448565001729653e-05,
      "loss_nats": 0.9814603328704834,
      "step": 6520,
      "unclipped_grad_norm": 0.7812637090682983
    },
    {
      "bits_per_byte": 1.498887964301274,
      "learning_rate": 5.41660932733604e-05,
      "loss_nats": 1.038949966430664,
      "step": 6530,
      "unclipped_grad_norm": 0.8116927742958069
    },
    {
      "bits_per_byte": 1.4673561491441343,
      "learning_rate": 5.384843037508902e-05,
      "loss_nats": 1.0170937776565552,
      "step": 6540,
      "unclipped_grad_norm": 0.7371414303779602
    },
    {
      "bits_per_byte": 1.5117546743429242,
      "learning_rate": 5.353266675046498e-05,
      "loss_nats": 1.0478684902191162,
      "step": 6550,
      "unclipped_grad_norm": 0.8187429308891296
    },
    {
      "bits_per_byte": 1.4773364743563537,
      "learning_rate": 5.321880779501729e-05,
      "loss_nats": 1.0240116119384766,
      "step": 6560,
      "unclipped_grad_norm": 0.7700304388999939
    },
    {
      "bits_per_byte": 1.4043874532531306,
      "learning_rate": 5.290685887172962e-05,
      "loss_nats": 0.9734472036361694,
      "step": 6570,
      "unclipped_grad_norm": 0.7884607315063477
    },
    {
      "bits_per_byte": 1.4975829599465271,
      "learning_rate": 5.2596825310948295e-05,
      "loss_nats": 1.0380454063415527,
      "step": 6580,
      "unclipped_grad_norm": 0.778003454208374
    },
    {
      "bits_per_byte": 1.4674713775202122,
      "learning_rate": 5.228871241029154e-05,
      "loss_nats": 1.0171736478805542,
      "step": 6590,
      "unclipped_grad_norm": 0.7916907668113708
    },
    {
      "bits_per_byte": 1.3998400599816672,
      "learning_rate": 5.198252543455865e-05,
      "loss_nats": 0.9702951908111572,
      "step": 6600,
      "unclipped_grad_norm": 0.7509376406669617
    },
    {
      "bits_per_byte": 1.5187203156681697,
      "learning_rate": 5.167826961564027e-05,
      "loss_nats": 1.052696704864502,
      "step": 6610,
      "unclipped_grad_norm": 0.7783279418945312
    },
    {
      "bits_per_byte": 1.511926828976438,
      "learning_rate": 5.137595015242884e-05,
      "loss_nats": 1.0479878187179565,
      "step": 6620,
      "unclipped_grad_norm": 0.8419784903526306
    },
    {
      "bits_per_byte": 1.5028970518755371,
      "learning_rate": 5.1075572210729925e-05,
      "loss_nats": 1.0417288541793823,
      "step": 6630,
      "unclipped_grad_norm": 0.8222247958183289
    },
    {
      "bits_per_byte": 1.4773347545298452,
      "learning_rate": 5.07771409231737e-05,
      "loss_nats": 1.024010419845581,
      "step": 6640,
      "unclipped_grad_norm": 0.8660594820976257
    },
    {
      "bits_per_byte": 1.4614148364874275,
      "learning_rate": 5.048066138912756e-05,
      "loss_nats": 1.0129755735397339,
      "step": 6650,
      "unclipped_grad_norm": 0.7894479036331177
    },
    {
      "bits_per_byte": 1.5265555012761776,
      "learning_rate": 5.018613867460863e-05,
      "loss_nats": 1.0581276416778564,
      "step": 6660,
      "unclipped_grad_norm": 0.810829758644104
    },
    {
      "bits_per_byte": 1.4984096805492246,
      "learning_rate": 4.989357781219755e-05,
      "loss_nats": 1.0386184453964233,
      "step": 6670,
      "unclipped_grad_norm": 0.8341873288154602
    },
    {
      "bits_per_byte": 1.5059076081788905,
      "learning_rate": 4.960298380095222e-05,
      "loss_nats": 1.0438156127929688,
      "step": 6680,
      "unclipped_grad_norm": 0.822468638420105
    },
    {
      "bits_per_byte": 1.4780378196065724,
      "learning_rate": 4.931436160632256e-05,
      "loss_nats": 1.0244977474212646,
      "step": 6690,
      "unclipped_grad_norm": 0.810896635055542
    },
    {
      "bits_per_byte": 1.524204498438883,
      "learning_rate": 4.9027716160065485e-05,
      "loss_nats": 1.0564980506896973,
      "step": 6700,
      "unclipped_grad_norm": 0.8146765828132629
    },
    {
      "bits_per_byte": 1.451385668202363,
      "learning_rate": 4.874305236016084e-05,
      "loss_nats": 1.00602388381958,
      "step": 6710,
      "unclipped_grad_norm": 0.8321484327316284
    },
    {
      "bits_per_byte": 1.4556473982907434,
      "learning_rate": 4.846037507072752e-05,
      "loss_nats": 1.0089778900146484,
      "step": 6720,
      "unclipped_grad_norm": 0.781524658203125
    },
    {
      "bits_per_byte": 1.448410712307738,
      "learning_rate": 4.817968912194049e-05,
      "loss_nats": 1.0039618015289307,
      "step": 6730,
      "unclipped_grad_norm": 0.8034297227859497
    },
    {
      "bits_per_byte": 1.4779710903380376,
      "learning_rate": 4.790099930994813e-05,
      "loss_nats": 1.024451494216919,
      "step": 6740,
      "unclipped_grad_norm": 0.8218453526496887
    },
    {
      "bits_per_byte": 1.4435792036970492,
      "learning_rate": 4.762431039679047e-05,
      "loss_nats": 1.0006128549575806,
      "step": 6750,
      "unclipped_grad_norm": 0.7989961504936218
    },
    {
      "bits_per_byte": 1.4296412137231747,
      "learning_rate": 4.734962711031755e-05,
      "loss_nats": 0.9909517765045166,
      "step": 6760,
      "unclipped_grad_norm": 0.7599804997444153
    },
    {
      "bits_per_byte": 1.4469161830717403,
      "learning_rate": 4.707695414410892e-05,
      "loss_nats": 1.0029258728027344,
      "step": 6770,
      "unclipped_grad_norm": 0.7921444773674011
    },
    {
      "bits_per_byte": 1.438630832875099,
      "learning_rate": 4.680629615739322e-05,
      "loss_nats": 0.9971829056739807,
      "step": 6780,
      "unclipped_grad_norm": 0.7756091356277466
    },
    {
      "bits_per_byte": 1.5384264317693321,
      "learning_rate": 4.653765777496863e-05,
      "loss_nats": 1.0663559436798096,
      "step": 6790,
      "unclipped_grad_norm": 0.8340134024620056
    },
    {
      "bits_per_byte": 1.5144365718004789,
      "learning_rate": 4.6271043587123986e-05,
      "loss_nats": 1.049727439880371,
      "step": 6800,
      "unclipped_grad_norm": 0.8667078614234924
    },
    {
      "bits_per_byte": 1.4971179188585941,
      "learning_rate": 4.600645814956006e-05,
      "loss_nats": 1.0377230644226074,
      "step": 6810,
      "unclipped_grad_norm": 0.8064154386520386
    },
    {
      "bits_per_byte": 1.4405763866129846,
      "learning_rate": 4.574390598331204e-05,
      "loss_nats": 0.9985314607620239,
      "step": 6820,
      "unclipped_grad_norm": 0.7713363170623779
    },
    {
      "bits_per_byte": 1.4385394240961653,
      "learning_rate": 4.548339157467199e-05,
      "loss_nats": 0.9971195459365845,
      "step": 6830,
      "unclipped_grad_norm": 0.8126972317695618
    },
    {
      "bits_per_byte": 1.4959006256557872,
      "learning_rate": 4.52249193751124e-05,
      "loss_nats": 1.036879301071167,
      "step": 6840,
      "unclipped_grad_norm": 0.7896748185157776
    },
    {
      "bits_per_byte": 1.465242826330332,
      "learning_rate": 4.4968493801209985e-05,
      "loss_nats": 1.0156289339065552,
      "step": 6850,
      "unclipped_grad_norm": 0.8369152545928955
    },
    {
      "bits_per_byte": 1.4691382333723746,
      "learning_rate": 4.471411923457034e-05,
      "loss_nats": 1.0183290243148804,
      "step": 6860,
      "unclipped_grad_norm": 0.8024821877479553
    },
    {
      "bits_per_byte": 1.4717907217966317,
      "learning_rate": 4.446180002175291e-05,
      "loss_nats": 1.020167589187622,
      "step": 6870,
      "unclipped_grad_norm": 0.808424174785614
    },
    {
      "bits_per_byte": 1.5114406340224489,
      "learning_rate": 4.421154047419693e-05,
      "loss_nats": 1.0476508140563965,
      "step": 6880,
      "unclipped_grad_norm": 0.8209573030471802
    },
    {
      "bits_per_byte": 1.4045318326885299,
      "learning_rate": 4.396334486814753e-05,
      "loss_nats": 0.9735472798347473,
      "step": 6890,
      "unclipped_grad_norm": 0.7931460738182068
    },
    {
      "bits_per_byte": 1.436284731543354,
      "learning_rate": 4.371721744458285e-05,
      "loss_nats": 0.9955567121505737,
      "step": 6900,
      "unclipped_grad_norm": 0.7861348390579224
    },
    {
      "bits_per_byte": 1.6150507221214936,
      "learning_rate": 4.347316240914145e-05,
      "loss_nats": 1.119467854499817,
      "step": 6910,
      "unclipped_grad_norm": 0.8759272694587708
    },
    {
      "bits_per_byte": 1.4220056999728183,
      "learning_rate": 4.3231183932050546e-05,
      "loss_nats": 0.9856592416763306,
      "step": 6920,
      "unclipped_grad_norm": 0.7773904204368591
    },
    {
      "bits_per_byte": 1.4332720254568647,
      "learning_rate": 4.299128614805464e-05,
      "loss_nats": 0.9934684634208679,
      "step": 6930,
      "unclipped_grad_norm": 0.8008426427841187
    },
    {
      "bits_per_byte": 1.46887819560427,
      "learning_rate": 4.2753473156344965e-05,
      "loss_nats": 1.0181487798690796,
      "step": 6940,
      "unclipped_grad_norm": 0.8342399597167969
    },
    {
      "bits_per_byte": 1.4565233059315876,
      "learning_rate": 4.251774902048942e-05,
      "loss_nats": 1.0095850229263306,
      "step": 6950,
      "unclipped_grad_norm": 0.8141571283340454
    },
    {
      "bits_per_byte": 1.5009811651449254,
      "learning_rate": 4.228411776836306e-05,
      "loss_nats": 1.0404008626937866,
      "step": 6960,
      "unclipped_grad_norm": 0.8327651619911194
    },
    {
      "bits_per_byte": 1.4767801104808127,
      "learning_rate": 4.2052583392079427e-05,
      "loss_nats": 1.0236259698867798,
      "step": 6970,
      "unclipped_grad_norm": 0.801481306552887
    },
    {
      "bits_per_byte": 1.517935386849632,
      "learning_rate": 4.182314984792214e-05,
      "loss_nats": 1.0521526336669922,
      "step": 6980,
      "unclipped_grad_norm": 0.8559384942054749
    },
    {
      "bits_per_byte": 1.462364696668143,
      "learning_rate": 4.159582105627748e-05,
      "loss_nats": 1.0136339664459229,
      "step": 6990,
      "unclipped_grad_norm": 0.8250771164894104
    },
    {
      "bits_per_byte": 1.4113358963132898,
      "learning_rate": 4.137060090156724e-05,
      "loss_nats": 0.9782634973526001,
      "step": 7000,
      "unclipped_grad_norm": 0.8160101771354675
    },
    {
      "bits_per_byte": 1.4572850170922589,
      "learning_rate": 4.114749323218249e-05,
      "loss_nats": 1.010113000869751,
      "step": 7010,
      "unclipped_grad_norm": 0.8388007283210754
    },
    {
      "bits_per_byte": 1.4311967968002286,
      "learning_rate": 4.0926501860417706e-05,
      "loss_nats": 0.9920300245285034,
      "step": 7020,
      "unclipped_grad_norm": 0.8063672780990601
    },
    {
      "bits_per_byte": 1.4271649215247264,
      "learning_rate": 4.070763056240571e-05,
      "loss_nats": 0.9892353415489197,
      "step": 7030,
      "unclipped_grad_norm": 0.7880839109420776
    },
    {
      "bits_per_byte": 1.5230513547648479,
      "learning_rate": 4.049088307805306e-05,
      "loss_nats": 1.0556987524032593,
      "step": 7040,
      "unclipped_grad_norm": 0.8723810911178589
    },
    {
      "bits_per_byte": 1.48768088684045,
      "learning_rate": 4.027626311097629e-05,
      "loss_nats": 1.031181812286377,
      "step": 7050,
      "unclipped_grad_norm": 0.8570515513420105
    },
    {
      "bits_per_byte": 1.4927676177050193,
      "learning_rate": 4.006377432843846e-05,
      "loss_nats": 1.0347076654434204,
      "step": 7060,
      "unclipped_grad_norm": 0.8982753157615662
    },
    {
      "bits_per_byte": 1.4459658069430723,
      "learning_rate": 3.985342036128656e-05,
      "loss_nats": 1.0022671222686768,
      "step": 7070,
      "unclipped_grad_norm": 0.8039788603782654
    },
    {
      "bits_per_byte": 1.364939964607379,
      "learning_rate": 3.964520480388956e-05,
      "loss_nats": 0.9461042881011963,
      "step": 7080,
      "unclipped_grad_norm": 0.7935104966163635
    },
    {
      "bits_per_byte": 1.4353330655448047,
      "learning_rate": 3.943913121407678e-05,
      "loss_nats": 0.9948970675468445,
      "step": 7090,
      "unclipped_grad_norm": 0.810722291469574
    },
    {
      "bits_per_byte": 1.4301969756594377,
      "learning_rate": 3.923520311307735e-05,
      "loss_nats": 0.9913370013237,
      "step": 7100,
      "unclipped_grad_norm": 0.7980725169181824
    },
    {
      "bits_per_byte": 1.4812796925753362,
      "learning_rate": 3.9033423985459816e-05,
      "loss_nats": 1.0267448425292969,
      "step": 7110,
      "unclipped_grad_norm": 0.8589518666267395
    },
    {
      "bits_per_byte": 1.491397775890897,
      "learning_rate": 3.88337972790728e-05,
      "loss_nats": 1.0337581634521484,
      "step": 7120,
      "unclipped_grad_norm": 0.881410539150238
    },
    {
      "bits_per_byte": 1.4146181852050066,
      "learning_rate": 3.8636326404985845e-05,
      "loss_nats": 0.9805386066436768,
      "step": 7130,
      "unclipped_grad_norm": 0.8223039507865906
    },
    {
      "bits_per_byte": 1.522814190689308,
      "learning_rate": 3.844101473743143e-05,
      "loss_nats": 1.0555343627929688,
      "step": 7140,
      "unclipped_grad_norm": 0.8451608419418335
    },
    {
      "bits_per_byte": 1.5467134158011566,
      "learning_rate": 3.824786561374701e-05,
      "loss_nats": 1.072100043296814,
      "step": 7150,
      "unclipped_grad_norm": 0.8498587012290955
    },
    {
      "bits_per_byte": 1.4723551688567635,
      "learning_rate": 3.805688233431824e-05,
      "loss_nats": 1.0205588340759277,
      "step": 7160,
      "unclipped_grad_norm": 0.8513677716255188
    },
    {
      "bits_per_byte": 1.4676500674944588,
      "learning_rate": 3.786806816252236e-05,
      "loss_nats": 1.0172975063323975,
      "step": 7170,
      "unclipped_grad_norm": 0.8321399092674255
    },
    {
      "bits_per_byte": 1.3859237398218012,
      "learning_rate": 3.768142632467269e-05,
      "loss_nats": 0.9606491327285767,
      "step": 7180,
      "unclipped_grad_norm": 0.7659841179847717
    },
    {
      "bits_per_byte": 1.3902817801946648,
      "learning_rate": 3.7496960009963236e-05,
      "loss_nats": 0.9636698961257935,
      "step": 7190,
      "unclipped_grad_norm": 0.7895154356956482
    },
    {
      "bits_per_byte": 1.4776350362382518,
      "learning_rate": 3.7314672370414325e-05,
      "loss_nats": 1.0242185592651367,
      "step": 7200,
      "unclipped_grad_norm": 0.8478621244430542
    },
    {
      "bits_per_byte": 1.4778720283311406,
      "learning_rate": 3.713456652081882e-05,
      "loss_nats": 1.0243828296661377,
      "step": 7210,
      "unclipped_grad_norm": 0.8023228049278259
    },
    {
      "bits_per_byte": 1.4423936412933265,
      "learning_rate": 3.695664553868873e-05,
      "loss_nats": 0.9997910857200623,
      "step": 7220,
      "unclipped_grad_norm": 0.8181896805763245
    },
    {
      "bits_per_byte": 1.4464050506333759,
      "learning_rate": 3.678091246420277e-05,
      "loss_nats": 1.0025715827941895,
      "step": 7230,
      "unclipped_grad_norm": 0.8425759077072144
    },
    {
      "bits_per_byte": 1.4082518174266925,
      "learning_rate": 3.660737030015427e-05,
      "loss_nats": 0.9761257767677307,
      "step": 7240,
      "unclipped_grad_norm": 0.7972860336303711
    },
    {
      "bits_per_byte": 1.4478307868090285,
      "learning_rate": 3.6436022011900056e-05,
      "loss_nats": 1.0035598278045654,
      "step": 7250,
      "unclipped_grad_norm": 0.8181965947151184
    },
    {
      "bits_per_byte": 1.4351106059859136,
      "learning_rate": 3.626687052730954e-05,
      "loss_nats": 0.9947428703308105,
      "step": 7260,
      "unclipped_grad_norm": 0.8426879048347473
    },
    {
      "bits_per_byte": 1.5111732010003571,
      "learning_rate": 3.6099918736714955e-05,
      "loss_nats": 1.047465443611145,
      "step": 7270,
      "unclipped_grad_norm": 0.862413763999939
    },
    {
      "bits_per_byte": 1.457399041589781,
      "learning_rate": 3.5935169492861716e-05,
      "loss_nats": 1.0101920366287231,
      "step": 7280,
      "unclipped_grad_norm": 0.8488914370536804
    },
    {
      "bits_per_byte": 1.4536087159474156,
      "learning_rate": 3.5772625610859886e-05,
      "loss_nats": 1.0075647830963135,
      "step": 7290,
      "unclipped_grad_norm": 0.8383923172950745
    },
    {
      "bits_per_byte": 1.4305855704590622,
      "learning_rate": 3.5612289868135887e-05,
      "loss_nats": 0.9916063547134399,
      "step": 7300,
      "unclipped_grad_norm": 0.8480672240257263
    },
    {
      "bits_per_byte": 1.4800558640317965,
      "learning_rate": 3.545416500438522e-05,
      "loss_nats": 1.0258965492248535,
      "step": 7310,
      "unclipped_grad_norm": 0.8378974795341492
    },
    {
      "bits_per_byte": 1.4690943777964045,
      "learning_rate": 3.52982537215255e-05,
      "loss_nats": 1.018298625946045,
      "step": 7320,
      "unclipped_grad_norm": 0.8481025695800781
    },
    {
      "bits_per_byte": 1.4814797083982898,
      "learning_rate": 3.5144558683650396e-05,
      "loss_nats": 1.0268834829330444,
      "step": 7330,
      "unclipped_grad_norm": 0.9106674194335938
    },
    {
      "bits_per_byte": 1.449738418372399,
      "learning_rate": 3.499308251698404e-05,
      "loss_nats": 1.0048820972442627,
      "step": 7340,
      "unclipped_grad_norm": 0.8405259847640991
    },
    {
      "bits_per_byte": 1.486937577823421,
      "learning_rate": 3.484382780983616e-05,
      "loss_nats": 1.0306665897369385,
      "step": 7350,
      "unclipped_grad_norm": 0.8228537440299988
    },
    {
      "bits_per_byte": 1.4739011209053692,
      "learning_rate": 3.469679711255795e-05,
      "loss_nats": 1.0216304063796997,
      "step": 7360,
      "unclipped_grad_norm": 0.8316923975944519
    },
    {
      "bits_per_byte": 1.4767537971352307,
      "learning_rate": 3.4551992937498284e-05,
      "loss_nats": 1.0236077308654785,
      "step": 7370,
      "unclipped_grad_norm": 0.8346049785614014
    },
    {
      "bits_per_byte": 1.5318123229824503,
      "learning_rate": 3.440941775896104e-05,
      "loss_nats": 1.0617713928222656,
      "step": 7380,
      "unclipped_grad_norm": 0.9013696908950806
    },
    {
      "bits_per_byte": 1.5039485538029123,
      "learning_rate": 3.4269074013162596e-05,
      "loss_nats": 1.0424576997756958,
      "step": 7390,
      "unclipped_grad_norm": 0.8856703639030457
    },
    {
      "bits_per_byte": 1.4314098833046476,
      "learning_rate": 3.41309640981904e-05,
      "loss_nats": 0.9921777248382568,
      "step": 7400,
      "unclipped_grad_norm": 0.8249487280845642
    },
    {
      "bits_per_byte": 1.4394619390353935,
      "learning_rate": 3.3995090373961795e-05,
      "loss_nats": 0.9977589845657349,
      "step": 7410,
      "unclipped_grad_norm": 0.8362482786178589
    },
    {
      "bits_per_byte": 1.518494330464936,
      "learning_rate": 3.3861455162183876e-05,
      "loss_nats": 1.0525400638580322,
      "step": 7420,
      "unclipped_grad_norm": 0.8421095609664917
    },
    {
      "bits_per_byte": 1.4326227909498577,
      "learning_rate": 3.3730060746313664e-05,
      "loss_nats": 0.9930184483528137,
      "step": 7430,
      "unclipped_grad_norm": 0.8718878626823425
    },
    {
      "bits_per_byte": 1.4438445729273304,
      "learning_rate": 3.3600909371519225e-05,
      "loss_nats": 1.0007967948913574,
      "step": 7440,
      "unclipped_grad_norm": 0.8476481437683105
    },
    {
      "bits_per_byte": 1.3938656406646694,
      "learning_rate": 3.3474003244641194e-05,
      "loss_nats": 0.9661540389060974,
      "step": 7450,
      "unclipped_grad_norm": 0.8507251739501953
    },
    {
      "bits_per_byte": 1.420028243453198,
      "learning_rate": 3.334934453415514e-05,
      "loss_nats": 0.9842885732650757,
      "step": 7460,
      "unclipped_grad_norm": 0.8349717259407043
    },
    {
      "bits_per_byte": 1.4775118966602339,
      "learning_rate": 3.3226935370134486e-05,
      "loss_nats": 1.0241332054138184,
      "step": 7470,
      "unclipped_grad_norm": 0.8582044243812561
    },
    {
      "bits_per_byte": 1.3999588999934132,
      "learning_rate": 3.31067778442141e-05,
      "loss_nats": 0.9703775644302368,
      "step": 7480,
      "unclipped_grad_norm": 0.84687340259552
    },
    {
      "bits_per_byte": 1.4226986180731447,
      "learning_rate": 3.298887400955457e-05,
      "loss_nats": 0.9861395359039307,
      "step": 7490,
      "unclipped_grad_norm": 0.831066370010376
    },
    {
      "bits_per_byte": 1.4892197876003705,
      "learning_rate": 3.287322588080714e-05,
      "loss_nats": 1.0322484970092773,
      "step": 7500,
      "unclipped_grad_norm": 0.8232710361480713
    },
    {
      "bits_per_byte": 1.461985990870943,
      "learning_rate": 3.275983543407924e-05,
      "loss_nats": 1.013371467590332,
      "step": 7510,
      "unclipped_grad_norm": 0.8441055417060852
    },
    {
      "bits_per_byte": 1.4446456681150495,
      "learning_rate": 3.2648704606900735e-05,
      "loss_nats": 1.001352071762085,
      "step": 7520,
      "unclipped_grad_norm": 0.8653393387794495
    },
    {
      "bits_per_byte": 1.4775328785436392,
      "learning_rate": 3.253983529819086e-05,
      "loss_nats": 1.0241477489471436,
      "step": 7530,
      "unclipped_grad_norm": 0.8701198697090149
    },
    {
      "bits_per_byte": 1.5027517265355579,
      "learning_rate": 3.243322936822571e-05,
      "loss_nats": 1.041628122329712,
      "step": 7540,
      "unclipped_grad_norm": 0.8485988974571228
    },
    {
      "bits_per_byte": 1.4225563884208812,
      "learning_rate": 3.232888863860648e-05,
      "loss_nats": 0.9860409498214722,
      "step": 7550,
      "unclipped_grad_norm": 0.8343487977981567
    },
    {
      "bits_per_byte": 1.4745732291049412,
      "learning_rate": 3.222681489222838e-05,
      "loss_nats": 1.0220962762832642,
      "step": 7560,
      "unclipped_grad_norm": 0.8472090363502502
    },
    {
      "bits_per_byte": 1.480253472097638,
      "learning_rate": 3.212700987325009e-05,
      "loss_nats": 1.0260335206985474,
      "step": 7570,
      "unclipped_grad_norm": 0.8703269958496094
    },
    {
      "bits_per_byte": 1.4382653697420156,
      "learning_rate": 3.202947528706403e-05,
      "loss_nats": 0.9969295859336853,
      "step": 7580,
      "unclipped_grad_norm": 0.847757875919342
    },
    {
      "bits_per_byte": 1.4346089325933469,
      "learning_rate": 3.1934212800267116e-05,
      "loss_nats": 0.9943951368331909,
      "step": 7590,
      "unclipped_grad_norm": 0.845406711101532
    },
    {
      "bits_per_byte": 1.414465206637064,
      "learning_rate": 3.1841224040632484e-05,
      "loss_nats": 0.9804325699806213,
      "step": 7600,
      "unclipped_grad_norm": 0.8379301428794861
    },
    {
      "bits_per_byte": 1.3641118681434745,
      "learning_rate": 3.175051059708143e-05,
      "loss_nats": 0.9455302953720093,
      "step": 7610,
      "unclipped_grad_norm": 0.7969970107078552
    },
    {
      "bits_per_byte": 1.4648639485504813,
      "learning_rate": 3.1662074019656414e-05,
      "loss_nats": 1.0153663158416748,
      "step": 7620,
      "unclipped_grad_norm": 0.8826617002487183
    },
    {
      "bits_per_byte": 1.4355323934371547,
      "learning_rate": 3.157591581949455e-05,
      "loss_nats": 0.9950352311134338,
      "step": 7630,
      "unclipped_grad_norm": 0.8636281490325928
    },
    {
      "bits_per_byte": 1.4410591419139565,
      "learning_rate": 3.149203746880175e-05,
      "loss_nats": 0.998866081237793,
      "step": 7640,
      "unclipped_grad_norm": 0.8592743277549744
    },
    {
      "bits_per_byte": 1.5176450801349755,
      "learning_rate": 3.141044040082763e-05,
      "loss_nats": 1.0519514083862305,
      "step": 7650,
      "unclipped_grad_norm": 0.9256166219711304
    },
    {
      "bits_per_byte": 1.4474489853241133,
      "learning_rate": 3.133112600984091e-05,
      "loss_nats": 1.0032951831817627,
      "step": 7660,
      "unclipped_grad_norm": 0.8606430292129517
    },
    {
      "bits_per_byte": 1.4195236463555665,
      "learning_rate": 3.125409565110571e-05,
      "loss_nats": 0.9839388132095337,
      "step": 7670,
      "unclipped_grad_norm": 0.8635953664779663
    },
    {
      "bits_per_byte": 1.3989710316468573,
      "learning_rate": 3.117935064085835e-05,
      "loss_nats": 0.9696928262710571,
      "step": 7680,
      "unclipped_grad_norm": 0.8459085822105408
    },
    {
      "bits_per_byte": 1.446349844202449,
      "learning_rate": 3.110689225628479e-05,
      "loss_nats": 1.0025333166122437,
      "step": 7690,
      "unclipped_grad_norm": 0.8500027656555176
    },
    {
      "bits_per_byte": 1.456846289349908,
      "learning_rate": 3.1036721735498926e-05,
      "loss_nats": 1.009808897972107,
      "step": 7700,
      "unclipped_grad_norm": 0.896350085735321
    },
    {
      "bits_per_byte": 1.3794107568336267,
      "learning_rate": 3.096884027752132e-05,
      "loss_nats": 0.9561346769332886,
      "step": 7710,
      "unclipped_grad_norm": 0.8459534049034119
    },
    {
      "bits_per_byte": 1.3797531742914946,
      "learning_rate": 3.090324904225885e-05,
      "loss_nats": 0.9563720226287842,
      "step": 7720,
      "unclipped_grad_norm": 0.8360687494277954
    },
    {
      "bits_per_byte": 1.4376540574095236,
      "learning_rate": 3.083994915048472e-05,
      "loss_nats": 0.996505856513977,
      "step": 7730,
      "unclipped_grad_norm": 0.82816481590271
    },
    {
      "bits_per_byte": 1.4181697129366488,
      "learning_rate": 3.077894168381942e-05,
      "loss_nats": 0.9830003380775452,
      "step": 7740,
      "unclipped_grad_norm": 0.83390212059021
    },
    {
      "bits_per_byte": 1.4407892151434272,
      "learning_rate": 3.0720227684712254e-05,
      "loss_nats": 0.998678982257843,
      "step": 7750,
      "unclipped_grad_norm": 0.8495392203330994
    },
    {
      "bits_per_byte": 1.4190407190719436,
      "learning_rate": 3.0663808156423456e-05,
      "loss_nats": 0.9836040735244751,
      "step": 7760,
      "unclipped_grad_norm": 0.850292444229126
    },
    {
      "bits_per_byte": 1.4212888762840223,
      "learning_rate": 3.060968406300711e-05,
      "loss_nats": 0.9851623773574829,
      "step": 7770,
      "unclipped_grad_norm": 0.888531506061554
    },
    {
      "bits_per_byte": 1.5506865590013892,
      "learning_rate": 3.055785632929463e-05,
      "loss_nats": 1.0748540163040161,
      "step": 7780,
      "unclipped_grad_norm": 0.9023171067237854
    },
    {
      "bits_per_byte": 1.4346873566821403,
      "learning_rate": 3.0508325840878984e-05,
      "loss_nats": 0.9944494962692261,
      "step": 7790,
      "unclipped_grad_norm": 0.8470334410667419
    },
    {
      "bits_per_byte": 1.4727918328073042,
      "learning_rate": 3.046109344409957e-05,
      "loss_nats": 1.0208615064620972,
      "step": 7800,
      "unclipped_grad_norm": 0.9052314758300781
    },
    {
      "bits_per_byte": 1.3950921349392975,
      "learning_rate": 3.0416159946027727e-05,
      "loss_nats": 0.9670041799545288,
      "step": 7810,
      "unclipped_grad_norm": 0.841830849647522
    },
    {
      "bits_per_byte": 1.4118079886899082,
      "learning_rate": 3.0373526114452946e-05,
      "loss_nats": 0.978590726852417,
      "step": 7820,
      "unclipped_grad_norm": 0.8733639121055603
    },
    {
      "bits_per_byte": 1.4649716096899212,
      "learning_rate": 3.0333192677869803e-05,
      "loss_nats": 1.0154409408569336,
      "step": 7830,
      "unclipped_grad_norm": 0.851719081401825
    },
    {
      "bits_per_byte": 1.4172854641372379,
      "learning_rate": 3.029516032546544e-05,
      "loss_nats": 0.9823874235153198,
      "step": 7840,
      "unclipped_grad_norm": 0.8457031846046448
    },
    {
      "bits_per_byte": 1.3998510668713222,
      "learning_rate": 3.02594297071078e-05,
      "loss_nats": 0.9703028202056885,
      "step": 7850,
      "unclipped_grad_norm": 0.8359724283218384
    },
    {
      "bits_per_byte": 1.413704183406996,
      "learning_rate": 3.0226001433334603e-05,
      "loss_nats": 0.9799050688743591,
      "step": 7860,
      "unclipped_grad_norm": 0.8466342091560364
    },
    {
      "bits_per_byte": 1.4588555626599382,
      "learning_rate": 3.019487607534278e-05,
      "loss_nats": 1.0112016201019287,
      "step": 7870,
      "unclipped_grad_norm": 0.8837466239929199
    },
    {
      "bits_per_byte": 1.4292317230314702,
      "learning_rate": 3.016605416497886e-05,
      "loss_nats": 0.9906679391860962,
      "step": 7880,
      "unclipped_grad_norm": 0.8551597595214844
    },
    {
      "bits_per_byte": 1.5078522160121963,
      "learning_rate": 3.0139536194729734e-05,
      "loss_nats": 1.0451635122299194,
      "step": 7890,
      "unclipped_grad_norm": 0.8927572965621948
    },
    {
      "bits_per_byte": 1.4757206973514978,
      "learning_rate": 3.0115322617714362e-05,
      "loss_nats": 1.022891640663147,
      "step": 7900,
      "unclipped_grad_norm": 0.8807728290557861
    },
    {
      "bits_per_byte": 1.4780766876856672,
      "learning_rate": 3.0093413847675965e-05,
      "loss_nats": 1.0245246887207031,
      "step": 7910,
      "unclipped_grad_norm": 0.9293215870857239
    },
    {
      "bits_per_byte": 1.3870213330996077,
      "learning_rate": 3.0073810258974985e-05,
      "loss_nats": 0.9614099264144897,
      "step": 7920,
      "unclipped_grad_norm": 0.8720042109489441
    },
    {
      "bits_per_byte": 1.3890644869918578,
      "learning_rate": 3.0056512186582658e-05,
      "loss_nats": 0.962826132774353,
      "step": 7930,
      "unclipped_grad_norm": 0.8701853156089783
    },
    {
      "bits_per_byte": 1.4495222361802644,
      "learning_rate": 3.0041519926075316e-05,
      "loss_nats": 1.0047322511672974,
      "step": 7940,
      "unclipped_grad_norm": 0.8989008069038391
    },
    {
      "bits_per_byte": 1.4133007980993975,
      "learning_rate": 3.0028833733629313e-05,
      "loss_nats": 0.9796254634857178,
      "step": 7950,
      "unclipped_grad_norm": 0.839148759841919
    },
    {
      "bits_per_byte": 1.4208300265715204,
      "learning_rate": 3.0018453826016686e-05,
      "loss_nats": 0.9848443269729614,
      "step": 7960,
      "unclipped_grad_norm": 0.8453875184059143
    },
    {
      "bits_per_byte": 1.41087549875693,
      "learning_rate": 3.001038038060143e-05,
      "loss_nats": 0.9779443740844727,
      "step": 7970,
      "unclipped_grad_norm": 0.8587523102760315
    },
    {
      "bits_per_byte": 1.446222061092858,
      "learning_rate": 3.0004613535336433e-05,
      "loss_nats": 1.0024447441101074,
      "step": 7980,
      "unclipped_grad_norm": 0.8559814691543579
    },
    {
      "bits_per_byte": 1.4566966644436572,
      "learning_rate": 3.000115338876117e-05,
      "loss_nats": 1.0097051858901978,
      "step": 7990,
      "unclipped_grad_norm": 0.860873818397522
    },
    {
      "bits_per_byte": 1.3805011268400968,
      "learning_rate": 3e-05,
      "loss_nats": 0.9568904638290405,
      "step": 8000,
      "unclipped_grad_norm": 0.8578172326087952
    }
  ]
}