{
  "architecture": "block",
  "batch_size": 32,
  "canonical_sha256_without_self": "b4262e697e2269457fdebf31f75008383c6e8024ee1bbf96e5962fb2ba1dec18",
  "depth": 32,
  "diagnostics": [
    {
      "activation_grad_rms_by_block": [
        0.00040311424527317286,
        0.00035970969474874437,
        0.00032544726855121553,
        0.00029308689408935606,
        0.00026978139067068696,
        0.00025443214690312743,
        0.00023969457834027708,
        0.0002233884297311306,
        0.0002095247182296589,
        0.0001984660339076072,
        0.0001901928335428238,
        0.00018121300672646612,
        0.0001734284160193056,
        0.0001677417167229578,
        0.00016109299031086266,
        0.00015624657680746168,
        0.0001518603676231578,
        0.00014820833166595548,
        0.00014440594532061368,
        0.0001414585713064298,
        0.00013923499500378966,
        0.00013687640603166074,
        0.00013471563579514623,
        0.00013265138841234148,
        0.00013098257477395236,
        0.0001293915556743741,
        0.00012844328011851758,
        0.0001269879430765286,
        0.0001256384130101651,
        0.00012456141121219844,
        0.0001231025526067242,
        0.00012163266364950687
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00029608183103846386,
        "first_to_last_ratio": 2.343484699021421,
        "imbalance_abs_log_ratio": 0.8516390092453431,
        "last_quartile_mean": 0.0001263425492652459,
        "mean": 0.0001858347804954974,
        "normalized": [
          2.16920774571012,
          1.9356424765570717,
          1.751272112160408,
          1.5771369240348272,
          1.451727119925344,
          1.369130935687747,
          1.289826251583108,
          1.2020808437231323,
          1.127478492836466,
          1.067970341065492,
          1.0234512239081746,
          0.9751296621832141,
          0.9332398141881068,
          0.9026389800429313,
          0.8668613586828855,
          0.8407822065969367,
          0.8171794710239251,
          0.7975274126338607,
          0.777066300159633,
          0.7612061150730458,
          0.7492407752334779,
          0.7365489154759011,
          0.7249215428669998,
          0.7138135717041164,
          0.7048334785593375,
          0.6962720074755282,
          0.6911692191098191,
          0.683337870004401,
          0.6760758813563923,
          0.6702804011180051,
          0.662430102042749,
          0.6545204472768429
        ],
        "population_cv": 0.39438035124793563
      },
      "activation_output_rms_by_block": [
        0.0036361459642648697,
        0.005152894649654627,
        0.006386642809957266,
        0.007284422870725393,
        0.0037505170330405235,
        0.0054016136564314365,
        0.006585344672203064,
        0.007922882214188576,
        0.0044892593286931515,
        0.006135277915745974,
        0.007316684816032648,
        0.0082859443500638,
        0.0044554853811860085,
        0.006464940495789051,
        0.007893112488090992,
        0.009196000173687935,
        0.004961193539202213,
        0.006907128728926182,
        0.008217990398406982,
        0.009676996618509293,
        0.005381465889513493,
        0.007502692751586437,
        0.0091153085231781,
        0.010543142445385456,
        0.006136673036962748,
        0.008469858206808567,
        0.010564134456217289,
        0.012301022186875343,
        0.005773921962827444,
        0.008156762458384037,
        0.010457510128617287,
        0.012218860909342766
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.005765057983808219,
        "first_to_last_ratio": 0.622587017371887,
        "imbalance_abs_log_ratio": 0.4738718734697682,
        "last_quartile_mean": 0.009259842918254435,
        "mean": 0.007398182220640592,
        "normalized": [
          0.49149180917985336,
          0.6965082091758006,
          0.8632718983507628,
          0.9846233376628902,
          0.5069511565390686,
          0.730127144119427,
          0.8901300989627224,
          1.0709228264321597,
          0.6068057253534959,
          0.8292953231982888,
          0.9889841312125875,
          1.119997332175246,
          0.6022405569783624,
          0.8738552664669654,
          1.0668989020126547,
          1.2430080659586231,
          0.6705962885532487,
          0.9336251153230055,
          1.1108121094231989,
          1.3080235563150786,
          0.7274038039370601,
          1.01412651484229,
          1.232101109614035,
          1.4250990487866828,
          0.8294838994154143,
          1.1448566626512724,
          1.427936504016329,
          1.6627087330393202,
          0.7804514393709395,
          1.1025360304896303,
          1.4135242707919937,
          1.6516031296515918
        ],
        "population_cv": 0.30590027610070303
      },
      "bits_per_byte": 8.038491115106986,
      "branch_output_rms_by_sublayer": [
        0.0023513585329055786,
        0.0027412420604377985,
        0.0023208872880786657,
        0.002683233004063368,
        0.002778682392090559,
        0.002711885841563344,
        0.002497476991266012,
        0.0027249811682850122,
        0.002628475893288851,
        0.0027012599166482687,
        0.002920303260907531,
        0.002700963756069541,
        0.002932395786046982,
        0.0026683746837079525,
        0.0032228652853518724,
        0.002738022478297353,
        0.003561305580660701,
        0.0027370047755539417,
        0.003354643704369664,
        0.002654933836311102,
        0.0034162327647209167,
        0.002663377905264497,
        0.0036349669098854065,
        0.002684312639757991,
        0.0035210708156228065,
        0.0027356327045708895,
        0.0038089773152023554,
        0.002652956871315837,
        0.003915763925760984,
        0.002679797587916255,
        0.0037244565319269896,
        0.002698024269193411,
        0.004162859171628952,
        0.0026484597474336624,
        0.004167158622294664,
        0.0026680538430809975,
        0.004440344404429197,
        0.002654345240443945,
        0.0041700867004692554,
        0.002591830212622881,
        0.004737259820103645,
        0.002620960585772991,
        0.004552462603896856,
        0.002651234855875373,
        0.004706811159849167,
        0.0026270023081451654,
        0.005186605267226696,
        0.002615687670186162,
        0.005580419674515724,
        0.00258000404573977,
        0.004905648995190859,
        0.002626793459057808,
        0.005227713380008936,
        0.002573782578110695,
        0.005349088925868273,
        0.002592637902125716,
        0.005235649179667234,
        0.002581534907221794,
        0.004925078712403774,
        0.002551236655563116,
        0.005810525268316269,
        0.002563327783718705,
        0.005564783234149218,
        0.002589041367173195
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.010012756197717748,
        0.008904953530422711,
        0.008980942528705073,
        0.007787912846174446,
        0.007907551845183929,
        0.008289745083748344,
        0.007663833460373458,
        0.007350080301903749,
        0.00709564946584838,
        0.006310668475383808,
        0.006285078227083486,
        0.006362421438950662,
        0.005604026016423943,
        0.005959901601525332,
        0.006167718108848569,
        0.005523196426059091,
        0.0054641843609823,
        0.00557989785603791,
        0.005260407000687673,
        0.004763357766419076,
        0.004970214176820314,
        0.004845025235106639,
        0.004921304265254701,
        0.004820106373346748,
        0.0054804096177630225,
        0.0042302744374162005,
        0.004588429583035324,
        0.004469681656036737,
        0.004349111730454971,
        0.003928502929307311,
        0.004468266803351117,
        0.003904642007649288
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.008362221974278682,
        "first_to_last_ratio": 1.8887369414995314,
        "imbalance_abs_log_ratio": 0.6359083207487641,
        "last_quartile_mean": 0.004427414845626747,
        "mean": 0.00600782035481319,
        "normalized": [
          1.6666204390908572,
          1.4822269982304765,
          1.4948753455169401,
          1.2962958920592769,
          1.3162097696294732,
          1.3798257261648947,
          1.2756429133626719,
          1.2234187888150154,
          1.1810688480662828,
          1.0504089840715678,
          1.0461494944748422,
          1.0590232502297414,
          0.9327885465041001,
          0.9920239370590588,
          1.0266149359654666,
          0.9193344840336578,
          0.9095119424808776,
          0.9287724210274617,
          0.875593258455752,
          0.7928595538984271,
          0.8272907449435313,
          0.8064530809788657,
          0.8191497039874,
          0.8023053434820334,
          0.9122126318860999,
          0.7041279844573081,
          0.7637428072161453,
          0.7439772483303089,
          0.7239084182952745,
          0.6538982022256998,
          0.7437417465006834,
          0.6499265585598059
        ],
        "population_cv": 0.2648157248292805
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.027978340163826942,
        0.014031016267836094,
        0.0140878576785326,
        0.014151837676763535,
        0.014187770895659924,
        0.014265215955674648,
        0.014337530359625816,
        0.014401593245565891,
        0.014477464370429516,
        0.009722241200506687,
        0.009757626801729202,
        0.009786579757928848,
        0.009825476445257664,
        0.009839320555329323,
        0.00987118761986494,
        0.00996283907443285,
        0.009999874979257584,
        0.007562503684312105,
        0.007595349568873644,
        0.007635751739144325,
        0.007658981718122959,
        0.007660485804080963,
        0.007689184043556452,
        0.007726665120571852,
        0.007755559403449297,
        0.006238248664885759,
        0.00625597033649683,
        0.006290202960371971,
        0.006315375678241253,
        0.006327399518340826,
        0.00635211868211627,
        0.006382675375789404,
        0.006408242508769035,
        0.005390914157032967,
        0.005408597644418478,
        0.0054517327807843685,
        0.005475745070725679,
        0.005509276874363422,
        0.005517482291907072,
        0.0055727302096784115,
        0.005585856735706329,
        0.0048178439028561115,
        0.004830927588045597,
        0.004884926602244377,
        0.004899685736745596,
        0.004938251338899136,
        0.004948681220412254,
        0.004975184798240662,
        0.004989070352166891,
        0.004412420094013214,
        0.004419817589223385,
        0.004455919377505779,
        0.004463871940970421,
        0.004530393984168768,
        0.004542885348200798,
        0.0045819394290447235,
        0.004592607729136944,
        0.004127359017729759,
        0.004137600306421518,
        0.004185030702501535,
        0.004193917382508516,
        0.004221425391733646,
        0.004230021964758635,
        0.004281661007553339
      ],
      "loss_nats": 5.571857452392578,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 2.1972246170043945,
        "mean_weights": [
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519
        ],
        "sources": 9
      },
      "step": 0,
      "stream_state_rms_by_sublayer": [
        0.0023513585329055786,
        0.0036361459642648697,
        0.004408807028084993,
        0.005152894649654627,
        0.005783648230135441,
        0.006386642809957266,
        0.006751363631337881,
        0.007284422870725393,
        0.002628475893288851,
        0.0037505170330405235,
        0.004723479505628347,
        0.0054016136564314365,
        0.006024772301316261,
        0.006585344672203064,
        0.007445912808179855,
        0.007922882214188576,
        0.003561305580660701,
        0.0044892593286931515,
        0.005543973762542009,
        0.006135277915745974,
        0.006855612155050039,
        0.007316684816032648,
        0.007876631803810596,
        0.0082859443500638,
        0.0035210708156228065,
        0.0044554853811860085,
        0.0058924350887537,
        0.006464940495789051,
        0.007390674669295549,
        0.007893112488090992,
        0.008790239691734314,
        0.009196000173687935,
        0.004162859171628952,
        0.004961193539202213,
        0.006355749908834696,
        0.006907128728926182,
        0.007784162648022175,
        0.008217990398406982,
        0.00937180407345295,
        0.009676996618509293,
        0.004737259820103645,
        0.005381465889513493,
        0.006999304983764887,
        0.007502692751586437,
        0.008728929795324802,
        0.0091153085231781,
        0.010217384435236454,
        0.010543142445385456,
        0.005580419674515724,
        0.006136673036962748,
        0.008008211851119995,
        0.008469858206808567,
        0.010209310799837112,
        0.010564134456217289,
        0.012040933594107628,
        0.012301022186875343,
        0.005235649179667234,
        0.005773921962827444,
        0.007745372597128153,
        0.008156762458384037,
        0.010180563665926456,
        0.010457510128617287,
        0.011906592175364494,
        0.012218860909342766
      ]
    },
    {
      "activation_grad_rms_by_block": [
        6.95402777637355e-05,
        6.603818474104628e-05,
        6.243807729333639e-05,
        5.902352859266102e-05,
        5.3786254284204915e-05,
        5.022022014600225e-05,
        4.6831908548483625e-05,
        4.3414886022219434e-05,
        3.877186099998653e-05,
        3.6122120945947245e-05,
        3.3541862649144605e-05,
        3.1072035199031234e-05,
        2.835266786860302e-05,
        2.6703228286351077e-05,
        2.5144798200926743e-05,
        2.3762264390825294e-05,
        2.2584137695957907e-05,
        2.1571571778622456e-05,
        2.0759547624038532e-05,
        1.9997522031189874e-05,
        1.9395642084418796e-05,
        1.8848633771995082e-05,
        1.8380347682978027e-05,
        1.797073673515115e-05,
        1.755586163199041e-05,
        1.7233234757441096e-05,
        1.6949628843576647e-05,
        1.6709716874174774e-05,
        1.630509359529242e-05,
        1.611644620425068e-05,
        1.594743662280962e-05,
        1.580910975462757e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 5.641166717396118e-05,
        "first_to_last_ratio": 3.40273807382454,
        "imbalance_abs_log_ratio": 1.2245804233600555,
        "last_quartile_mean": 1.6578316035520402e-05,
        "mean": 3.084058886315688e-05,
        "normalized": [
          2.254829765809383,
          2.1412750915381364,
          2.0245423189025695,
          1.913826251974467,
          1.7440086673720954,
          1.6283807150646488,
          1.518515381022133,
          1.407719100788124,
          1.2571699318719751,
          1.1712526341901288,
          1.0875882687575642,
          1.0075046017085245,
          0.9193296533476374,
          0.8658469008110146,
          0.8153151132261776,
          0.770486727612793,
          0.7322862023214354,
          0.6994539525278819,
          0.6731242297658762,
          0.648415700488765,
          0.6288998621420432,
          0.6111632256967778,
          0.5959791417905044,
          0.5826975877435189,
          0.5692453444999938,
          0.5587842318415797,
          0.5495883661230995,
          0.5418092679201958,
          0.5286894380532074,
          0.5225725836741036,
          0.517092481391006,
          0.5126072600226392
        ],
        "population_cv": 0.5330470009230729
      },
      "activation_output_rms_by_block": [
        0.005864492617547512,
        0.009892112575471401,
        0.014509912580251694,
        0.019168438389897346,
        0.006793031003326178,
        0.01223041582852602,
        0.017751524224877357,
        0.023574935272336006,
        0.008267601951956749,
        0.014755276031792164,
        0.02151178941130638,
        0.02830074541270733,
        0.009282662533223629,
        0.01676790602505207,
        0.02516039088368416,
        0.0331411249935627,
        0.010437680408358574,
        0.01991143450140953,
        0.029664477333426476,
        0.04036801680922508,
        0.01357664167881012,
        0.025029508396983147,
        0.03699437528848648,
        0.05095698684453964,
        0.015438608825206757,
        0.029965175315737724,
        0.04443887993693352,
        0.060807984322309494,
        0.019359532743692398,
        0.037317242473363876,
        0.05564062297344208,
        0.07935244590044022
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.01372310781152919,
        "first_to_last_ratio": 0.3207078305283738,
        "imbalance_abs_log_ratio": 1.137224755592113,
        "last_quartile_mean": 0.04279006156139076,
        "mean": 0.02613224917149637,
        "normalized": [
          0.224415915333617,
          0.37854041994445614,
          0.5552492816453899,
          0.7335165933901026,
          0.2599481949963819,
          0.4680200218611986,
          0.6792956897196492,
          0.9021395409795135,
          0.31637544467372514,
          0.5646385787522038,
          0.8231893577216564,
          1.0829816150527234,
          0.35521865998999624,
          0.6416556766702496,
          0.9628100022529916,
          1.268207905720998,
          0.3994176061868959,
          0.7619487465750876,
          1.1351673994362057,
          1.5447586062839287,
          0.519535903309034,
          0.9578015360531602,
          1.4156598249812316,
          1.9499655965365867,
          0.5907876020884704,
          1.1466741771474498,
          1.700537892674199,
          2.3269326694097012,
          0.7408291807047638,
          1.4280149492094805,
          2.1291938021979306,
          3.0365716085010215
        ],
        "population_cv": 0.6534988385002395
      },
      "bits_per_byte": 5.118322357706281,
      "branch_output_rms_by_sublayer": [
        0.003087003482505679,
        0.0049620093777775764,
        0.0026560353580862284,
        0.004914142191410065,
        0.0032712947577238083,
        0.00555405905470252,
        0.002734958194196224,
        0.005823355168104172,
        0.003094236133620143,
        0.006141432095319033,
        0.0030397549271583557,
        0.006136075593531132,
        0.0035086015705019236,
        0.006357426289469004,
        0.0038121570833027363,
        0.0064498125575482845,
        0.0047028325498104095,
        0.006525229662656784,
        0.004038033075630665,
        0.007023691199719906,
        0.004860103130340576,
        0.006682637147605419,
        0.005031581036746502,
        0.006716069765388966,
        0.005196469835937023,
        0.006977807730436325,
        0.005135253071784973,
        0.006669311318546534,
        0.006614624988287687,
        0.006923797074705362,
        0.005866772960871458,
        0.007155857048928738,
        0.006203304044902325,
        0.007240671198815107,
        0.007167549338191748,
        0.007543859537690878,
        0.007609309628605843,
        0.007281317841261625,
        0.008147990331053734,
        0.007711030542850494,
        0.00901782512664795,
        0.00788753479719162,
        0.008936969563364983,
        0.007908415980637074,
        0.009206709451973438,
        0.008078642189502716,
        0.01152731105685234,
        0.008181036449968815,
        0.010577243752777576,
        0.007894126698374748,
        0.011347576975822449,
        0.008452145382761955,
        0.011137762106955051,
        0.008351319469511509,
        0.012936098501086235,
        0.008498509414494038,
        0.013407763093709946,
        0.009082214906811714,
        0.013788831420242786,
        0.009139914996922016,
        0.015265416353940964,
        0.008129997178912163,
        0.0202538650482893,
        0.00888112559914589
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0002932670153511829,
        0.0002289733734981176,
        0.00023382191596673618,
        0.00020236837455671756,
        0.0002080571281121399,
        0.00019123637931049836,
        0.00021124835961838567,
        0.00020444780259018514,
        0.00020817260281626697,
        0.00017921191649714578,
        0.0001893830879389061,
        0.0001891854298096901,
        0.00018418310100442068,
        0.00017033524110907422,
        0.00020556753369166975,
        0.00017823357128584385,
        0.0001749817385153215,
        0.00018220334134876856,
        0.00017289789528954973,
        0.00017141475273833558,
        0.00017603112434287998,
        0.00016603952795395127,
        0.0001609386975915838,
        0.00016666055126527177,
        0.00016172141650657888,
        0.00015460675258205218,
        0.00014513294616478987,
        0.00015851211030580876,
        0.00015100080713109572,
        0.00014721653732759288,
        0.00014725738129728224,
        0.00017222312685716471
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.0002216775436254954,
        "first_to_last_ratio": 1.4328688617518024,
        "imbalance_abs_log_ratio": 0.35967863157527663,
        "last_quartile_mean": 0.00015470888477154567,
        "mean": 0.000183954110636719,
        "normalized": [
          1.5942400761586677,
          1.2447309424376207,
          1.2710882902122171,
          1.1001024867357483,
          1.1310273382420937,
          1.0395874201917712,
          1.1483753142954691,
          1.1114065452657267,
          1.1316550747124958,
          0.974220776458003,
          1.02951267184732,
          1.0284381749060347,
          1.0012448233252798,
          0.9259659407419281,
          1.1174935584757544,
          0.9689023565112261,
          0.9512249435995668,
          0.9904825758886794,
          0.9398968834732723,
          0.9318343153355962,
          0.956929550166532,
          0.9026138496130359,
          0.8748850299377811,
          0.9059898182672343,
          0.879139998268121,
          0.8404637006855296,
          0.7889627780669988,
          0.8616937656742324,
          0.8208612822428254,
          0.8002894679441159,
          0.8005115014151157,
          0.936228748904007
        ],
        "population_cv": 0.1635293006829388
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.692298173904419,
          "mean_weights": [
            0.48010045289993286,
            0.5198994874954224
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6927915811538696,
          "mean_weights": [
            0.5130036473274231,
            0.4869963526725769
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6928644776344299,
          "mean_weights": [
            0.5081723928451538,
            0.4918276071548462
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.692646861076355,
          "mean_weights": [
            0.5149973034858704,
            0.48500269651412964
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6925361156463623,
          "mean_weights": [
            0.5161343812942505,
            0.4838656485080719
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6928029656410217,
          "mean_weights": [
            0.5107344388961792,
            0.4892655611038208
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6924350261688232,
          "mean_weights": [
            0.5171386003494263,
            0.48286139965057373
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6929639577865601,
          "mean_weights": [
            0.5072796940803528,
            0.49272027611732483
          ],
          "sources": 2
        },
        {
          "entropy_mean": 1.0978727340698242,
          "mean_weights": [
            0.34929555654525757,
            0.32294148206710815,
            0.3277629613876343
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.098320722579956,
          "mean_weights": [
            0.3432672619819641,
            0.3268624544143677,
            0.329870343208313
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.098150610923767,
          "mean_weights": [
            0.3442404866218567,
            0.33018797636032104,
            0.32557153701782227
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0984500646591187,
          "mean_weights": [
            0.34007447957992554,
            0.32975536584854126,
            0.3301700949668884
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0982636213302612,
          "mean_weights": [
            0.3434537351131439,
            0.33091628551483154,
            0.3256300091743469
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0984963178634644,
          "mean_weights": [
            0.33822712302207947,
            0.33379626274108887,
            0.32797661423683167
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0980933904647827,
          "mean_weights": [
            0.34460341930389404,
            0.33179372549057007,
            0.3236028254032135
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0985530614852905,
          "mean_weights": [
            0.3358767628669739,
            0.3345070779323578,
            0.32961612939834595
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.3860312700271606,
          "mean_weights": [
            0.25749218463897705,
            0.24992284178733826,
            0.24571210145950317,
            0.24687285721302032
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862019777297974,
          "mean_weights": [
            0.25314557552337646,
            0.25236955285072327,
            0.24678170680999756,
            0.2477031946182251
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.385892391204834,
          "mean_weights": [
            0.25815317034721375,
            0.25234687328338623,
            0.2448991984128952,
            0.24460072815418243
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3861613273620605,
          "mean_weights": [
            0.25378289818763733,
            0.25339171290397644,
            0.24767157435417175,
            0.24515381455421448
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3855845928192139,
          "mean_weights": [
            0.2592235505580902,
            0.2543467879295349,
            0.2462996244430542,
            0.24013003706932068
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3861030340194702,
          "mean_weights": [
            0.25138330459594727,
            0.25586679577827454,
            0.24899962544441223,
            0.24375030398368835
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3856689929962158,
          "mean_weights": [
            0.2574368119239807,
            0.25204822421073914,
            0.2460038810968399,
            0.24451106786727905
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.386252999305725,
          "mean_weights": [
            0.25002047419548035,
            0.25257009267807007,
            0.2502796947956085,
            0.24712975323200226
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.608945608139038,
          "mean_weights": [
            0.20495271682739258,
            0.2066430151462555,
            0.2000001072883606,
            0.19580814242362976,
            0.19259600341320038
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6093038320541382,
          "mean_weights": [
            0.20154938101768494,
            0.20420850813388824,
            0.2010108083486557,
            0.19671520590782166,
            0.19651609659194946
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6085759401321411,
          "mean_weights": [
            0.20549985766410828,
            0.209174245595932,
            0.20157773792743683,
            0.19417354464530945,
            0.18957462906837463
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6093413829803467,
          "mean_weights": [
            0.20104944705963135,
            0.20404505729675293,
            0.2002137303352356,
            0.19850106537342072,
            0.1961907148361206
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6080443859100342,
          "mean_weights": [
            0.20538949966430664,
            0.21283748745918274,
            0.20341980457305908,
            0.19302619993686676,
            0.18532699346542358
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6093816757202148,
          "mean_weights": [
            0.2010989934206009,
            0.20253092050552368,
            0.19950580596923828,
            0.19837552309036255,
            0.1984887421131134
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6076874732971191,
          "mean_weights": [
            0.20581099390983582,
            0.21552260220050812,
            0.20295079052448273,
            0.19259145855903625,
            0.18312416970729828
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.609355092048645,
          "mean_weights": [
            0.20059309899806976,
            0.203348308801651,
            0.2011984884738922,
            0.1984248012304306,
            0.19643528759479523
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.7901467084884644,
          "mean_weights": [
            0.17125868797302246,
            0.18124164640903473,
            0.17160004377365112,
            0.1620369851589203,
            0.15427924692630768,
            0.15958338975906372
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7916979789733887,
          "mean_weights": [
            0.16689208149909973,
            0.16927623748779297,
            0.1683787703514099,
            0.16569086909294128,
            0.1651553511619568,
            0.1646067053079605
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.789665699005127,
          "mean_weights": [
            0.1720278263092041,
            0.18269017338752747,
            0.17356479167938232,
            0.16241812705993652,
            0.15660038590431213,
            0.15269872546195984
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7917176485061646,
          "mean_weights": [
            0.16644492745399475,
            0.16854418814182281,
            0.16665583848953247,
            0.16677430272102356,
            0.16461026668548584,
            0.16697049140930176
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7893871068954468,
          "mean_weights": [
            0.17303556203842163,
            0.18359911441802979,
            0.17276406288146973,
            0.16385790705680847,
            0.1554735004901886,
            0.15126986801624298
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7917413711547852,
          "mean_weights": [
            0.16618916392326355,
            0.16740113496780396,
            0.16675519943237305,
            0.1668717861175537,
            0.1653555929660797,
            0.16742712259292603
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7882882356643677,
          "mean_weights": [
            0.17288538813591003,
            0.18687990307807922,
            0.17607632279396057,
            0.162863090634346,
            0.15389539301395416,
            0.14739990234375
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7917335033416748,
          "mean_weights": [
            0.16660982370376587,
            0.16799160838127136,
            0.16742658615112305,
            0.1664734035730362,
            0.16542598605155945,
            0.1660725623369217
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.941649079322815,
          "mean_weights": [
            0.15037035942077637,
            0.16432975232601166,
            0.15334898233413696,
            0.1418340504169464,
            0.13212506473064423,
            0.12697139382362366,
            0.13102038204669952
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9458887577056885,
          "mean_weights": [
            0.1431310474872589,
            0.14324922859668732,
            0.14309079945087433,
            0.142366424202919,
            0.14222830533981323,
            0.14392223954200745,
            0.14201194047927856
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9409761428833008,
          "mean_weights": [
            0.1505419909954071,
            0.16567251086235046,
            0.1550290882587433,
            0.14243587851524353,
            0.13249853253364563,
            0.12776729464530945,
            0.12605470418930054
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9458999633789062,
          "mean_weights": [
            0.14249610900878906,
            0.14256571233272552,
            0.1435922086238861,
            0.14210085570812225,
            0.14355877041816711,
            0.1427185833454132,
            0.14296773076057434
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.941575527191162,
          "mean_weights": [
            0.15046924352645874,
            0.1640411913394928,
            0.15395450592041016,
            0.14198558032512665,
            0.13400861620903015,
            0.1288735270500183,
            0.126667320728302
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9458911418914795,
          "mean_weights": [
            0.14260169863700867,
            0.14355596899986267,
            0.1428615152835846,
            0.14244115352630615,
            0.14214617013931274,
            0.14205437898635864,
            0.14433911442756653
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9400205612182617,
          "mean_weights": [
            0.1507154405117035,
            0.16689133644104004,
            0.15648439526557922,
            0.1427520215511322,
            0.13353285193443298,
            0.12712189555168152,
            0.12250205874443054
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9458441734313965,
          "mean_weights": [
            0.1422245353460312,
            0.14142528176307678,
            0.14198407530784607,
            0.14165860414505005,
            0.14288201928138733,
            0.14370128512382507,
            0.14612418413162231
          ],
          "sources": 7
        },
        {
          "entropy_mean": 2.0741422176361084,
          "mean_weights": [
            0.13300861418247223,
            0.14654949307441711,
            0.1379268318414688,
            0.12711459398269653,
            0.1185426190495491,
            0.1134917140007019,
            0.10956315696239471,
            0.11380299925804138
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0794055461883545,
          "mean_weights": [
            0.12428364902734756,
            0.1234029084444046,
            0.12436957657337189,
            0.12428068369626999,
            0.12555459141731262,
            0.12575605511665344,
            0.1263071894645691,
            0.1260453760623932
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.071772813796997,
          "mean_weights": [
            0.134201318025589,
            0.1495765745639801,
            0.14145472645759583,
            0.12840476632118225,
            0.11872729659080505,
            0.11272042989730835,
            0.10793697834014893,
            0.1069779098033905
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079404592514038,
          "mean_weights": [
            0.12455718219280243,
            0.12374450266361237,
            0.12391386926174164,
            0.12458803504705429,
            0.1251734495162964,
            0.1251622885465622,
            0.1266230344772339,
            0.1262376308441162
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0732176303863525,
          "mean_weights": [
            0.13281872868537903,
            0.1473538726568222,
            0.13968363404273987,
            0.12848341464996338,
            0.12016644328832626,
            0.11505335569381714,
            0.10907343029975891,
            0.1073671206831932
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0794081687927246,
          "mean_weights": [
            0.12450858950614929,
            0.12424902617931366,
            0.12409761548042297,
            0.12468484044075012,
            0.12504076957702637,
            0.1255234032869339,
            0.12684062123298645,
            0.12505511939525604
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.072478771209717,
          "mean_weights": [
            0.13302236795425415,
            0.14877822995185852,
            0.14031866192817688,
            0.12882456183433533,
            0.11896270513534546,
            0.11445862054824829,
            0.11000129580497742,
            0.10563355684280396
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079291820526123,
          "mean_weights": [
            0.12426379323005676,
            0.12249694019556046,
            0.12262053787708282,
            0.12315947562456131,
            0.1255876123905182,
            0.126260906457901,
            0.1273583620786667,
            0.12825237214565277
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.1905503273010254,
          "mean_weights": [
            0.11892962455749512,
            0.13242119550704956,
            0.12570281326770782,
            0.11684690415859222,
            0.10851435363292694,
            0.10355254262685776,
            0.09824737906455994,
            0.0950399786233902,
            0.10074522346258163
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1971964836120605,
          "mean_weights": [
            0.11044953763484955,
            0.11010265350341797,
            0.1108412891626358,
            0.11044170707464218,
            0.11131919175386429,
            0.11133328080177307,
            0.11097346246242523,
            0.11225177347660065,
            0.11228710412979126
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1902995109558105,
          "mean_weights": [
            0.11926670372486115,
            0.13237422704696655,
            0.12644407153129578,
            0.11611802130937576,
            0.11051423847675323,
            0.10466914623975754,
            0.09992008656263351,
            0.09535142779350281,
            0.09534208476543427
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.197113037109375,
          "mean_weights": [
            0.11037027835845947,
            0.10917964577674866,
            0.10961420089006424,
            0.10947252064943314,
            0.11109665036201477,
            0.11174711585044861,
            0.1118166521191597,
            0.11308293044567108,
            0.11361998319625854
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1894307136535645,
          "mean_weights": [
            0.1198490560054779,
            0.13425537943840027,
            0.12682178616523743,
            0.11624304950237274,
            0.10967148095369339,
            0.10377678275108337,
            0.09994889795780182,
            0.0946216955780983,
            0.09481187164783478
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.196977138519287,
          "mean_weights": [
            0.10978136211633682,
            0.1075383648276329,
            0.10839901119470596,
            0.11003260314464569,
            0.11053797602653503,
            0.11152469366788864,
            0.11391061544418335,
            0.11424341052770615,
            0.11403194814920425
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1905651092529297,
          "mean_weights": [
            0.11943811178207397,
            0.13202990591526031,
            0.12506362795829773,
            0.11625386774539948,
            0.11043960601091385,
            0.1047937273979187,
            0.1010652631521225,
            0.09624719619750977,
            0.0946686640381813
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.02856997773051262,
        0.013703026808798313,
        0.015031684190034866,
        0.014947476796805859,
        0.015670204535126686,
        0.015775443986058235,
        0.016530683264136314,
        0.016744175925850868,
        0.01773575134575367,
        0.012103146873414516,
        0.01292925700545311,
        0.013081531040370464,
        0.014021075330674648,
        0.01416850183159113,
        0.015247474424540997,
        0.015494133345782757,
        0.01669454388320446,
        0.012773395515978336,
        0.01373021025210619,
        0.013903867453336716,
        0.01499613095074892,
        0.015176740474998951,
        0.01623532734811306,
        0.016410861164331436,
        0.01758282259106636,
        0.014334342442452908,
        0.015234442427754402,
        0.015468464232981205,
        0.01637510396540165,
        0.01670125499367714,
        0.01772587187588215,
        0.017849057912826538,
        0.01901337131857872,
        0.01606663130223751,
        0.017032917588949203,
        0.0173187293112278,
        0.01834850199520588,
        0.018571525812149048,
        0.019697511568665504,
        0.019770435988903046,
        0.021141810342669487,
        0.018272221088409424,
        0.019469235092401505,
        0.01950262486934662,
        0.02084389142692089,
        0.02077464945614338,
        0.022254284471273422,
        0.0220524650067091,
        0.02407064661383629,
        0.020764723420143127,
        0.022380243986845016,
        0.02203267626464367,
        0.023947052657604218,
        0.023422541096806526,
        0.025450291112065315,
        0.024811791256070137,
        0.027382872998714447,
        0.023658527061343193,
        0.025761734694242477,
        0.025185640901327133,
        0.02756396308541298,
        0.026624467223882675,
        0.02946169301867485,
        0.02854793891310692
      ],
      "loss_nats": 3.54775071144104,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 2.1935477256774902,
        "mean_weights": [
          0.10072024166584015,
          0.09480113536119461,
          0.10205776989459991,
          0.11059839278459549,
          0.1157325804233551,
          0.11904929578304291,
          0.11986880004405975,
          0.11937490850687027,
          0.11779685318470001
        ],
        "sources": 9
      },
      "step": 100,
      "stream_state_rms_by_sublayer": [
        0.003087003482505679,
        0.005864492617547512,
        0.006683253683149815,
        0.009892112575471401,
        0.010629965923726559,
        0.014509912580251694,
        0.01484723947942257,
        0.019168438389897346,
        0.003094236133620143,
        0.006793031003326178,
        0.007696541491895914,
        0.01223041582852602,
        0.012912816368043423,
        0.017751524224877357,
        0.018475037068128586,
        0.023574935272336006,
        0.0047028325498104095,
        0.008267601951956749,
        0.0095678074285388,
        0.014755276031792164,
        0.01631263643503189,
        0.02151178941130638,
        0.023062339052557945,
        0.02830074541270733,
        0.005196469835937023,
        0.009282662533223629,
        0.011978921480476856,
        0.01676790602505207,
        0.01994149386882782,
        0.02516039088368416,
        0.027569252997636795,
        0.0331411249935627,
        0.006203304044902325,
        0.010437680408358574,
        0.014378104358911514,
        0.01991143450140953,
        0.024205319583415985,
        0.029664477333426476,
        0.03444278985261917,
        0.04036801680922508,
        0.00901782512664795,
        0.01357664167881012,
        0.01929495297372341,
        0.025029508396983147,
        0.030904190614819527,
        0.03699437528848648,
        0.04446368291974068,
        0.05095698684453964,
        0.010577243752777576,
        0.015438608825206757,
        0.023803669959306717,
        0.029965175315737724,
        0.03807948902249336,
        0.04443887993693352,
        0.05412018671631813,
        0.060807984322309494,
        0.013407763093709946,
        0.019359532743692398,
        0.030261604115366936,
        0.037317242473363876,
        0.04958754777908325,
        0.05564062297344208,
        0.07224015146493912,
        0.07935244590044022
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.00014133188233245164,
        0.0001488620473537594,
        0.00014702342741657048,
        0.0001400294277118519,
        0.00013315585965756327,
        0.00011939816613448784,
        0.0001078194472938776,
        9.923135075950995e-05,
        8.953637006925419e-05,
        8.09328121249564e-05,
        7.468360854545608e-05,
        6.987994856899604e-05,
        7.115858170436695e-05,
        6.467441562563181e-05,
        5.974166560918093e-05,
        5.531677743420005e-05,
        5.705624425900169e-05,
        5.295562368701212e-05,
        4.8012458137236536e-05,
        4.413425631355494e-05,
        5.2346804295666516e-05,
        4.681985228671692e-05,
        4.172014814685099e-05,
        3.8159589166752994e-05,
        3.1379553547594696e-05,
        3.02761654893402e-05,
        2.9319184250198305e-05,
        2.8667578590102494e-05,
        2.2355405235430226e-05,
        2.2064739823690616e-05,
        2.201247843913734e-05,
        2.2052488930057734e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.000129606451082509,
        "first_to_last_ratio": 4.9818074922726145,
        "imbalance_abs_log_ratio": 1.6057927754415957,
        "last_quartile_mean": 2.601594928819395e-05,
        "mean": 6.850338621688934e-05,
        "normalized": [
          2.0631371693799077,
          2.173061151786652,
          2.146221311616304,
          2.04412417319785,
          1.9437850741564366,
          1.7429527608527258,
          1.5739287245233284,
          1.448561249882341,
          1.3070356812110293,
          1.181442503713818,
          1.0902177639657034,
          1.0200948073975256,
          1.038760061861336,
          0.9441053821903842,
          0.8720979926456799,
          0.8075042780959905,
          0.832896699126015,
          0.7730365841967101,
          0.7008771505867426,
          0.6442638641806842,
          0.7641491492103867,
          0.6834677068149604,
          0.6090231512754765,
          0.5570467574542299,
          0.45807302792658366,
          0.4419659692950432,
          0.4279961308389998,
          0.4184841096663127,
          0.3263401485679923,
          0.32209706763737483,
          0.3213341654300619,
          0.32191823131541414
        ],
        "population_cv": 0.5858213540607353
      },
      "activation_output_rms_by_block": [
        0.021054552868008614,
        0.038058556616306305,
        0.04883526638150215,
        0.05791458487510681,
        0.016990283504128456,
        0.027475843206048012,
        0.03445403650403023,
        0.040619369596242905,
        0.019055280834436417,
        0.029903952032327652,
        0.037492990493774414,
        0.04475866258144379,
        0.0285621490329504,
        0.03507128730416298,
        0.04228900745511055,
        0.05219251662492752,
        0.027010982856154442,
        0.04389137029647827,
        0.054182786494493484,
        0.06662839651107788,
        0.03837457299232483,
        0.041143495589494705,
        0.05716610327363014,
        0.07329611480236053,
        0.028997479006648064,
        0.053451184183359146,
        0.08026835322380066,
        0.10854664444923401,
        0.027061885222792625,
        0.05247335880994797,
        0.08323372155427933,
        0.11460508406162262
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.035675311693921685,
        "first_to_last_ratio": 0.5202021080271609,
        "imbalance_abs_log_ratio": 0.6535378736360024,
        "last_quartile_mean": 0.06857971381396055,
        "mean": 0.047658121038693935,
        "normalized": [
          0.4417831087153916,
          0.798574425236075,
          1.0246997849926245,
          1.2152091524566315,
          0.3565034276180115,
          0.5765196488493577,
          0.7229415628043072,
          0.8523074076559539,
          0.3998328179779751,
          0.6274681288431082,
          0.7867072741565622,
          0.9391612931005807,
          0.5993133680146686,
          0.7358932022453922,
          0.8873410561187638,
          1.095144237485823,
          0.5667655851187262,
          0.92096308750491,
          1.136905637771622,
          1.398049168933493,
          0.8052053281993276,
          0.8633050295056751,
          1.1995039256209161,
          1.5379564532737442,
          0.6084478022770731,
          1.121554585418124,
          1.6842534173479116,
          2.277610658655726,
          0.567833658419326,
          1.1010370880409766,
          1.746475096798326,
          2.404733580842896
        ],
        "population_cv": 0.48875915937608194
      },
      "bits_per_byte": 3.059021014366605,
      "branch_output_rms_by_sublayer": [
        0.01139580737799406,
        0.01652366667985916,
        0.014748631045222282,
        0.013563322834670544,
        0.012339585460722446,
        0.012220539152622223,
        0.0120800556614995,
        0.0097816726192832,
        0.01439172774553299,
        0.008202088065445423,
        0.0171960536390543,
        0.008006753399968147,
        0.01820177212357521,
        0.008192610926926136,
        0.01519495714455843,
        0.008106541819870472,
        0.016362585127353668,
        0.008609342388808727,
        0.015944689512252808,
        0.008666405454277992,
        0.017555685713887215,
        0.009109876118600368,
        0.01772458106279373,
        0.008975221775472164,
        0.025544000789523125,
        0.009615846909582615,
        0.019797103479504585,
        0.009856111370027065,
        0.02002105489373207,
        0.009463186375796795,
        0.017730148509144783,
        0.009926749393343925,
        0.022795414552092552,
        0.010939708910882473,
        0.02134518511593342,
        0.010758070275187492,
        0.021650495007634163,
        0.011536303907632828,
        0.03255617991089821,
        0.012654256075620651,
        0.033973678946495056,
        0.014695252291858196,
        0.031387828290462494,
        0.016267452389001846,
        0.019825322553515434,
        0.01879013702273369,
        0.013387269340455532,
        0.02227928675711155,
        0.015395586378872395,
        0.021389417350292206,
        0.011897368356585503,
        0.023454682901501656,
        0.014087310060858727,
        0.024781694635748863,
        0.014466723427176476,
        0.026336129754781723,
        0.006911881268024445,
        0.026170482859015465,
        0.005224450025707483,
        0.026235800236463547,
        0.00758614856749773,
        0.030007388442754745,
        0.005234629847109318,
        0.031163375824689865
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00048079443816738296,
        0.0004352800282264352,
        0.00039416580024316875,
        0.0003828745093102654,
        0.0003643213588911513,
        0.0002989310042525185,
        0.0002673721517191021,
        0.00023887606182770654,
        0.0002716851528017292,
        0.00026407625379019375,
        0.00023115090959535868,
        0.00019271062698547964,
        0.00023181255047936123,
        0.00019176941572124543,
        0.00020689480536159778,
        0.0001935784151883457,
        0.00017612030874433946,
        0.00017372524780256995,
        0.0001906158264031438,
        0.00020699034417808569,
        0.0002469606938485226,
        0.00020806313134476826,
        0.00017307229934302834,
        0.00014129096626273445,
        0.00011634997010006574,
        0.00010725656087507846,
        0.00011122939555071074,
        0.00011004876520390145,
        7.494181919924042e-05,
        7.493373834181984e-05,
        7.712439386045853e-05,
        7.424519704760952e-05
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00035782691907971636,
        "first_to_last_ratio": 3.8366182378544447,
        "imbalance_abs_log_ratio": 1.3445913113590247,
        "last_quartile_mean": 9.326623002236059e-05,
        "mean": 0.0002159144418958475,
        "normalized": [
          2.226782210331757,
          2.015983851772199,
          1.8255647782620577,
          1.7732695689478515,
          1.6873413176636518,
          1.384488233523149,
          1.2383245389767708,
          1.1063459198478962,
          1.258300048927641,
          1.2230597058328248,
          1.070567154127008,
          0.892532362788586,
          1.0736315201703026,
          0.8881731765481019,
          0.9582258766247762,
          0.896551492751577,
          0.8156948983954307,
          0.8046022594744787,
          0.8828303689620391,
          0.9586683612295531,
          1.1437896033265396,
          0.9636369365469933,
          0.801578152083568,
          0.6543840468572741,
          0.5388707169305075,
          0.49675491798189547,
          0.5151549594091785,
          0.5096869122677149,
          0.34709034996089216,
          0.3470529237593392,
          0.3571988662882573,
          0.34386396943018666
        ],
        "population_cv": 0.48652454891086055
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.6572662591934204,
          "mean_weights": [
            0.3911391496658325,
            0.6088608503341675
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6751130819320679,
          "mean_weights": [
            0.44147175550460815,
            0.5585282444953918
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6885842084884644,
          "mean_weights": [
            0.4896422326564789,
            0.5103577375411987
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.668085515499115,
          "mean_weights": [
            0.4332469701766968,
            0.5667530298233032
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6886032223701477,
          "mean_weights": [
            0.51185142993927,
            0.4881485104560852
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6789880990982056,
          "mean_weights": [
            0.4699174761772156,
            0.5300825834274292
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6772009134292603,
          "mean_weights": [
            0.4395230710506439,
            0.5604768991470337
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6781555414199829,
          "mean_weights": [
            0.4541339576244354,
            0.5458660125732422
          ],
          "sources": 2
        },
        {
          "entropy_mean": 1.0917184352874756,
          "mean_weights": [
            0.30861830711364746,
            0.355538010597229,
            0.33584368228912354
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0860013961791992,
          "mean_weights": [
            0.30104559659957886,
            0.3248867988586426,
            0.37406760454177856
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0827364921569824,
          "mean_weights": [
            0.2735002934932709,
            0.34776103496551514,
            0.3787387013435364
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0724949836730957,
          "mean_weights": [
            0.2886848449707031,
            0.3024986684322357,
            0.40881651639938354
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0897927284240723,
          "mean_weights": [
            0.2973601222038269,
            0.3298739790916443,
            0.3727658987045288
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0697213411331177,
          "mean_weights": [
            0.30285951495170593,
            0.28946101665496826,
            0.4076794683933258
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0826778411865234,
          "mean_weights": [
            0.2862996757030487,
            0.3235614597797394,
            0.3901388645172119
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0579445362091064,
          "mean_weights": [
            0.28999701142311096,
            0.2830814719200134,
            0.4269214868545532
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.3809051513671875,
          "mean_weights": [
            0.23795948922634125,
            0.2348281741142273,
            0.26517200469970703,
            0.26204031705856323
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3605632781982422,
          "mean_weights": [
            0.24186556041240692,
            0.22099661827087402,
            0.2607640027999878,
            0.27637380361557007
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3829561471939087,
          "mean_weights": [
            0.23731295764446259,
            0.2382979840040207,
            0.2565648853778839,
            0.2678241431713104
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.35977041721344,
          "mean_weights": [
            0.23858636617660522,
            0.21978360414505005,
            0.2459542453289032,
            0.29567578434944153
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3790044784545898,
          "mean_weights": [
            0.24452988803386688,
            0.2358427345752716,
            0.24750955402851105,
            0.27211782336235046
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3556556701660156,
          "mean_weights": [
            0.2410905957221985,
            0.22088873386383057,
            0.23317404091358185,
            0.3048466145992279
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3804312944412231,
          "mean_weights": [
            0.26568183302879333,
            0.22685569524765015,
            0.2384241223335266,
            0.2690383493900299
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3533532619476318,
          "mean_weights": [
            0.24537262320518494,
            0.2171444594860077,
            0.23335666954517365,
            0.30412623286247253
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.6053518056869507,
          "mean_weights": [
            0.20177139341831207,
            0.18913303315639496,
            0.18395328521728516,
            0.1997503787279129,
            0.22539189457893372
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.585533618927002,
          "mean_weights": [
            0.19244730472564697,
            0.1780712604522705,
            0.17078758776187897,
            0.20516318082809448,
            0.25353068113327026
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6022908687591553,
          "mean_weights": [
            0.1985350400209427,
            0.1897844821214676,
            0.17792725563049316,
            0.1998080015182495,
            0.23394520580768585
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5833771228790283,
          "mean_weights": [
            0.1926397681236267,
            0.1801614761352539,
            0.17439623177051544,
            0.19375719130039215,
            0.2590453326702118
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.601109504699707,
          "mean_weights": [
            0.19747966527938843,
            0.19028550386428833,
            0.18151578307151794,
            0.19478440284729004,
            0.23593464493751526
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5755603313446045,
          "mean_weights": [
            0.1969345360994339,
            0.17712506651878357,
            0.16844677925109863,
            0.18347150087356567,
            0.27402210235595703
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5980476140975952,
          "mean_weights": [
            0.19043894112110138,
            0.1923028826713562,
            0.1810823678970337,
            0.19125500321388245,
            0.24492081999778748
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5842485427856445,
          "mean_weights": [
            0.1964084804058075,
            0.17743000388145447,
            0.17621047794818878,
            0.18783687055110931,
            0.26211416721343994
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.7856926918029785,
          "mean_weights": [
            0.15997162461280823,
            0.16431675851345062,
            0.15234553813934326,
            0.15752777457237244,
            0.18046650290489197,
            0.18537181615829468
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7751184701919556,
          "mean_weights": [
            0.16064736247062683,
            0.15155571699142456,
            0.14907033741474152,
            0.14832919836044312,
            0.18695974349975586,
            0.2034376710653305
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7829411029815674,
          "mean_weights": [
            0.14932186901569366,
            0.16724151372909546,
            0.15592646598815918,
            0.1555134654045105,
            0.17433607578277588,
            0.19766061007976532
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7661566734313965,
          "mean_weights": [
            0.165928453207016,
            0.143815815448761,
            0.1375015527009964,
            0.1444939523935318,
            0.17902101576328278,
            0.22923919558525085
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7807259559631348,
          "mean_weights": [
            0.14958028495311737,
            0.16952179372310638,
            0.15695258975028992,
            0.1499582827091217,
            0.1711740493774414,
            0.20281299948692322
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.781292200088501,
          "mean_weights": [
            0.1682473123073578,
            0.15929323434829712,
            0.14874719083309174,
            0.15058940649032593,
            0.1693388819694519,
            0.20378397405147552
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7818934917449951,
          "mean_weights": [
            0.14812925457954407,
            0.18386004865169525,
            0.16644306480884552,
            0.15062814950942993,
            0.15809310972690582,
            0.19284634292125702
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.77958083152771,
          "mean_weights": [
            0.19125157594680786,
            0.15093471109867096,
            0.14643779397010803,
            0.1490234136581421,
            0.16375312209129333,
            0.19859938323497772
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.9289655685424805,
          "mean_weights": [
            0.12320417165756226,
            0.15015369653701782,
            0.13329030573368073,
            0.12274879962205887,
            0.13076716661453247,
            0.1441647708415985,
            0.19567111134529114
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9340521097183228,
          "mean_weights": [
            0.14550700783729553,
            0.14042077958583832,
            0.13418063521385193,
            0.1260203719139099,
            0.12906627357006073,
            0.1355920135974884,
            0.18921294808387756
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9230798482894897,
          "mean_weights": [
            0.12067099660634995,
            0.15888366103172302,
            0.14002522826194763,
            0.12021138519048691,
            0.12209220230579376,
            0.13571104407310486,
            0.20240548253059387
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9308109283447266,
          "mean_weights": [
            0.1450628787279129,
            0.14527732133865356,
            0.13644321262836456,
            0.12497064471244812,
            0.12634633481502533,
            0.1283716857433319,
            0.19352790713310242
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9273775815963745,
          "mean_weights": [
            0.1260491907596588,
            0.16657930612564087,
            0.14038512110710144,
            0.12195964902639389,
            0.12248437106609344,
            0.13317090272903442,
            0.18937145173549652
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.918284296989441,
          "mean_weights": [
            0.13994047045707703,
            0.15779416263103485,
            0.13759920001029968,
            0.11949826776981354,
            0.11786683648824692,
            0.12190048396587372,
            0.20540057122707367
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9060933589935303,
          "mean_weights": [
            0.11515206843614578,
            0.18610316514968872,
            0.15217435359954834,
            0.11993157863616943,
            0.11246243864297867,
            0.12337306141853333,
            0.19080331921577454
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9199097156524658,
          "mean_weights": [
            0.13630829751491547,
            0.1662534922361374,
            0.14684012532234192,
            0.12322510778903961,
            0.12947364151477814,
            0.1381947100162506,
            0.15970461070537567
          ],
          "sources": 7
        },
        {
          "entropy_mean": 2.0490760803222656,
          "mean_weights": [
            0.10742150247097015,
            0.1722080111503601,
            0.14153704047203064,
            0.1159135103225708,
            0.10868977755308151,
            0.11180940270423889,
            0.1477845162153244,
            0.09463626146316528
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.051286220550537,
          "mean_weights": [
            0.1131708174943924,
            0.16089829802513123,
            0.14448881149291992,
            0.11956015229225159,
            0.10795272141695023,
            0.11352235078811646,
            0.13960374891757965,
            0.10080308467149734
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0247201919555664,
          "mean_weights": [
            0.1011081114411354,
            0.1894242763519287,
            0.1531166434288025,
            0.11718504130840302,
            0.10284577310085297,
            0.10429200530052185,
            0.14699342846870422,
            0.08503472805023193
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.047537326812744,
          "mean_weights": [
            0.1111423671245575,
            0.16533835232257843,
            0.14553973078727722,
            0.12043843418359756,
            0.10876712203025818,
            0.11067016422748566,
            0.13975468277931213,
            0.09834913164377213
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0282247066497803,
          "mean_weights": [
            0.100978784263134,
            0.18681633472442627,
            0.15267831087112427,
            0.11643697321414948,
            0.10204557329416275,
            0.10493149608373642,
            0.14740025997161865,
            0.08871228992938995
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.053830623626709,
          "mean_weights": [
            0.11781682819128036,
            0.1550440490245819,
            0.1398259550333023,
            0.11913557350635529,
            0.10805961489677429,
            0.11164884269237518,
            0.13677509129047394,
            0.11169403791427612
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.017721176147461,
          "mean_weights": [
            0.09720280021429062,
            0.19308976829051971,
            0.16317345201969147,
            0.11856384575366974,
            0.10023905336856842,
            0.10130266845226288,
            0.14337101578712463,
            0.08305738866329193
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0608584880828857,
          "mean_weights": [
            0.12434182316064835,
            0.14232873916625977,
            0.13082580268383026,
            0.11684279143810272,
            0.10329097509384155,
            0.10883669555187225,
            0.1425117552280426,
            0.13102145493030548
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.142296314239502,
          "mean_weights": [
            0.08636107295751572,
            0.16583400964736938,
            0.14278221130371094,
            0.1055997684597969,
            0.08843953907489777,
            0.08854378759860992,
            0.12657995522022247,
            0.07546606659889221,
            0.12039357423782349
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.191580057144165,
          "mean_weights": [
            0.1060912162065506,
            0.12397187203168869,
            0.11601399630308151,
            0.1091354638338089,
            0.10782144963741302,
            0.10360677540302277,
            0.12169313430786133,
            0.10359254479408264,
            0.10807354003190994
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1432788372039795,
          "mean_weights": [
            0.08927536010742188,
            0.16873395442962646,
            0.14930182695388794,
            0.112123504281044,
            0.09766747802495956,
            0.0965762585401535,
            0.1252625584602356,
            0.07491002976894379,
            0.08614903688430786
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1724743843078613,
          "mean_weights": [
            0.1105542778968811,
            0.12893787026405334,
            0.11523330211639404,
            0.10665617883205414,
            0.10366716235876083,
            0.10828493535518646,
            0.11819782853126526,
            0.10312190651893616,
            0.10534654557704926
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1452555656433105,
          "mean_weights": [
            0.09202822297811508,
            0.1701684445142746,
            0.14628148078918457,
            0.11057832837104797,
            0.09616327285766602,
            0.09830240160226822,
            0.12624149024486542,
            0.08024715632200241,
            0.07998920977115631
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1762189865112305,
          "mean_weights": [
            0.09908370673656464,
            0.13803134858608246,
            0.13144394755363464,
            0.11672185361385345,
            0.1074555367231369,
            0.10124591737985611,
            0.12084682285785675,
            0.08855879306793213,
            0.09661206603050232
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1389729976654053,
          "mean_weights": [
            0.09089216589927673,
            0.17082780599594116,
            0.14892268180847168,
            0.11308471858501434,
            0.10078474879264832,
            0.09951519966125488,
            0.12574949860572815,
            0.07681785523891449,
            0.07340531051158905
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.04405451565980911,
        0.018189694732427597,
        0.0240375604480505,
        0.02713710628449917,
        0.030349314212799072,
        0.03182194381952286,
        0.034919846802949905,
        0.03608781099319458,
        0.038470037281513214,
        0.026185201480984688,
        0.02659580484032631,
        0.02762730047106743,
        0.028048591688275337,
        0.028690988197922707,
        0.029997555539011955,
        0.03034786321222782,
        0.031361404806375504,
        0.02331089787185192,
        0.0246393084526062,
        0.024178674444556236,
        0.025546060875058174,
        0.024928271770477295,
        0.02669469080865383,
        0.0256717037409544,
        0.027990806847810745,
        0.020905915647745132,
        0.02195541374385357,
        0.021578654646873474,
        0.023044798523187637,
        0.022400982677936554,
        0.02394026517868042,
        0.023724498227238655,
        0.025360919535160065,
        0.020535636693239212,
        0.02148214913904667,
        0.021988606080412865,
        0.02322523109614849,
        0.023164646700024605,
        0.023834573104977608,
        0.024238524958491325,
        0.02502058818936348,
        0.019457563757896423,
        0.01980534754693508,
        0.019637931138277054,
        0.02057415433228016,
        0.02042427472770214,
        0.02201046608388424,
        0.021937908604741096,
        0.02379545010626316,
        0.021518733352422714,
        0.022529911249876022,
        0.02267260104417801,
        0.023956971243023872,
        0.024102909490466118,
        0.02642984129488468,
        0.025153208523988724,
        0.030454812571406364,
        0.023361310362815857,
        0.02776513062417507,
        0.025513608008623123,
        0.030121317133307457,
        0.027568820863962173,
        0.03041756898164749,
        0.028944427147507668
      ],
      "loss_nats": 2.120351791381836,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 2.1635239124298096,
        "mean_weights": [
          0.1242082342505455,
          0.0880277156829834,
          0.0938715785741806,
          0.09339353442192078,
          0.09749531745910645,
          0.10569863021373749,
          0.13438579440116882,
          0.14857196807861328,
          0.11434720456600189
        ],
        "sources": 9
      },
      "step": 500,
      "stream_state_rms_by_sublayer": [
        0.01139580737799406,
        0.021054552868008614,
        0.02950318343937397,
        0.038058556616306305,
        0.04104945436120033,
        0.04883526638150215,
        0.05214058980345726,
        0.05791458487510681,
        0.01439172774553299,
        0.016990283504128456,
        0.024067245423793793,
        0.027475843206048012,
        0.030465533956885338,
        0.03445403650403023,
        0.036618731915950775,
        0.040619369596242905,
        0.016362585127353668,
        0.019055280834436417,
        0.02618422359228134,
        0.029903952032327652,
        0.033149950206279755,
        0.037492990493774414,
        0.04017872363328934,
        0.04475866258144379,
        0.025544000789523125,
        0.0285621490329504,
        0.030907167121767998,
        0.03507128730416298,
        0.03762345016002655,
        0.04228900745511055,
        0.04704070836305618,
        0.05219251662492752,
        0.022795414552092552,
        0.027010982856154442,
        0.03890290856361389,
        0.04389137029647827,
        0.04891306161880493,
        0.054182786494493484,
        0.060658931732177734,
        0.06662839651107788,
        0.033973678946495056,
        0.03837457299232483,
        0.03364448994398117,
        0.041143495589494705,
        0.04609670862555504,
        0.05716610327363014,
        0.05855504423379898,
        0.07329611480236053,
        0.015395586378872395,
        0.028997479006648064,
        0.03421308845281601,
        0.053451184183359146,
        0.058928847312927246,
        0.08026835322380066,
        0.08527950197458267,
        0.10854664444923401,
        0.006911881268024445,
        0.027061885222792625,
        0.028406694531440735,
        0.05247335880994797,
        0.05527278408408165,
        0.08323372155427933,
        0.0854114517569542,
        0.11460508406162262
      ]
    },
    {
      "activation_grad_rms_by_block": [
        5.134936509421095e-05,
        5.603071258519776e-05,
        4.972576425643638e-05,
        4.7571254981448874e-05,
        4.3214979086769745e-05,
        3.9845475839683786e-05,
        3.5890228900825605e-05,
        3.406162795727141e-05,
        3.5466175177134573e-05,
        3.383069270057604e-05,
        3.1626921554561704e-05,
        3.0198520107660443e-05,
        2.883204069803469e-05,
        2.77392227872042e-05,
        2.7800235329777934e-05,
        3.112062404397875e-05,
        6.127587403170764e-05,
        6.331028998829424e-05,
        5.8612553402781487e-05,
        5.708334720111452e-05,
        0.00015888184134382755,
        0.00012872362276539207,
        0.00010434311116114259,
        8.657111175125465e-05,
        5.805831096949987e-05,
        5.078948743175715e-05,
        4.287513002054766e-05,
        3.959577952628024e-05,
        2.2001073375577107e-05,
        1.997212712012697e-05,
        1.8872458895202726e-05,
        1.86532452062238e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 4.4711176087730564e-05,
        "first_to_last_ratio": 1.3207760209544162,
        "imbalance_abs_log_ratio": 0.27821945851834845,
        "last_quartile_mean": 3.385220156815194e-05,
        "mean": 4.981010016535947e-05,
        "normalized": [
          1.0309026668033479,
          1.1248865671658381,
          0.9983068512481783,
          0.9550523854303026,
          0.8675947035501784,
          0.799947715571839,
          0.7205411910772573,
          0.68382974224492,
          0.7120277827066004,
          0.6791934284063869,
          0.6349499689734955,
          0.6062730250974693,
          0.5788392434931497,
          0.5568995584252106,
          0.5581244614543389,
          0.6247854138149606,
          1.2301897371875206,
          1.271033179578376,
          1.1767202476646232,
          1.146019522378179,
          3.189751492495938,
          2.584287570955602,
          2.0948183363362958,
          1.7380232415485222,
          1.165593138274326,
          1.019662423146035,
          0.8607718088943986,
          0.7949347500773989,
          0.4416990390008851,
          0.4009654076948961,
          0.3788881940118565,
          0.3744872052916737
        ],
        "population_cv": 0.6175097196118516
      },
      "activation_output_rms_by_block": [
        0.0955304205417633,
        0.19925470650196075,
        0.2359117865562439,
        0.25103330612182617,
        0.09304249286651611,
        0.11085781455039978,
        0.12910984456539154,
        0.14378803968429565,
        0.04010729491710663,
        0.06472750008106232,
        0.09987020492553711,
        0.14899125695228577,
        0.05702618882060051,
        0.10170809924602509,
        0.15923865139484406,
        0.38537684082984924,
        0.05919196084141731,
        0.08540526777505875,
        0.1145314872264862,
        0.15640178322792053,
        0.06262174993753433,
        0.08427268266677856,
        0.11030438542366028,
        0.13170360028743744,
        0.05311359465122223,
        0.08614693582057953,
        0.12623678147792816,
        0.17503507435321808,
        0.07505694776773453,
        0.13522638380527496,
        0.2187158316373825,
        0.3292153775691986
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.15731605142354965,
        "first_to_last_ratio": 1.0498699791884785,
        "imbalance_abs_log_ratio": 0.04866632715763954,
        "last_quartile_mean": 0.14984336588531733,
        "mean": 0.13496107165701687,
        "normalized": [
          0.7078368552417796,
          1.4763865169086419,
          1.7479987648273716,
          1.86004232953773,
          0.6894024456398029,
          0.8214058556983611,
          0.9566450753555414,
          1.06540380806779,
          0.2971767667867461,
          0.479601260469815,
          0.7399926786247083,
          1.1039572754054923,
          0.4225380557552472,
          0.7536106374770113,
          1.1798857955094264,
          2.8554666623367226,
          0.4385854388578413,
          0.632814090214868,
          0.8486260950679959,
          1.158865895977708,
          0.4639986121085199,
          0.624422151010662,
          0.8173051982276947,
          0.9758636225277033,
          0.39354751706627217,
          0.6383095122385285,
          0.935356987967166,
          1.2969300866107762,
          0.5561377576971261,
          1.0019658420387498,
          1.6205845800722127,
          2.439335828673987
        ],
        "population_cv": 0.5772709429538598
      },
      "bits_per_byte": 2.064953725142933,
      "branch_output_rms_by_sublayer": [
        0.051234737038612366,
        0.07594770938158035,
        0.11323792487382889,
        0.04037529230117798,
        0.05108218640089035,
        0.04642988368868828,
        0.03771067038178444,
        0.02321692928671837,
        0.08815877884626389,
        0.015561277978122234,
        0.03706873208284378,
        0.01583549566566944,
        0.0438041090965271,
        0.020819898694753647,
        0.030363978818058968,
        0.027020027860999107,
        0.03348636254668236,
        0.0216005090624094,
        0.027010800316929817,
        0.024715786799788475,
        0.032826121896505356,
        0.03509491682052612,
        0.031702958047389984,
        0.05490833893418312,
        0.038218315690755844,
        0.037195492535829544,
        0.03341353312134743,
        0.042240556329488754,
        0.04180334508419037,
        0.04918703809380531,
        0.2476748526096344,
        0.04376247525215149,
        0.05078785493969917,
        0.02373618632555008,
        0.0355411171913147,
        0.01774558797478676,
        0.052597854286432266,
        0.022557877004146576,
        0.06600882858037949,
        0.03756217285990715,
        0.050489071756601334,
        0.03271714597940445,
        0.022040290758013725,
        0.03932206705212593,
        0.028325051069259644,
        0.04476166144013405,
        0.02171224355697632,
        0.049177221953868866,
        0.024430278688669205,
        0.0453680194914341,
        0.015834001824259758,
        0.0437605082988739,
        0.018936052918434143,
        0.052028607577085495,
        0.02182060107588768,
        0.060354240238666534,
        0.030554499477148056,
        0.06733348965644836,
        0.016542606055736542,
        0.07210637629032135,
        0.015565099194645882,
        0.09804103523492813,
        0.013472299091517925,
        0.1243291050195694
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00020065673277085515,
        0.0001817740596446822,
        0.0001465045892684325,
        9.755666249346124e-05,
        0.00012624719322665892,
        7.565538032235173e-05,
        8.200655559893429e-05,
        7.97391790009874e-05,
        8.09162329894288e-05,
        7.508564486922528e-05,
        9.13364471873345e-05,
        0.00011039800739152168,
        8.766456722124108e-05,
        8.422239419368561e-05,
        9.398125423258295e-05,
        0.00017161898601766447,
        0.0001497631172902175,
        0.00014237628599353527,
        0.00022365597941099996,
        0.00025566927173363617,
        0.0008281082302532628,
        0.0006115617552883831,
        0.0005788343529396405,
        0.00045039612289279527,
        0.0002758540055807746,
        0.00022394859978287228,
        0.00021964130134669458,
        0.00019854079025805258,
        0.00011545923673689007,
        0.00010570174900205617,
        9.906174415914337e-05,
        0.0001053958635417218
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00012376754404079543,
        "first_to_last_ratio": 0.7369290916409895,
        "imbalance_abs_log_ratio": 0.30526360357753907,
        "last_quartile_mean": 0.00016795041130102567,
        "mean": 0.00019904163414499135,
        "normalized": [
          1.0081143758329842,
          0.9132464191500225,
          0.7360499721465892,
          0.49013194105106844,
          0.6342753051087516,
          0.3800982676179234,
          0.4120070454164219,
          0.40061557645221907,
          0.4065291833892681,
          0.3772358742519635,
          0.4588811159016144,
          0.5546478145929135,
          0.4404333173700838,
          0.42313958361261256,
          0.4721688235543854,
          0.8622265726207266,
          0.7524210597121753,
          0.7153090689047581,
          1.123664304564935,
          1.2845014704179658,
          4.16047744890413,
          3.072531824386513,
          2.908106916555911,
          2.2628236792142964,
          1.385911077176092,
          1.1251344511155768,
          1.1034942628470255,
          0.9974837229954918,
          0.5800758079225982,
          0.5310534625386868,
          0.49769358347903286,
          0.5295166711952659
        ],
        "population_cv": 0.8756839263617585
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.4580664336681366,
          "mean_weights": [
            0.81428062915802,
            0.18571940064430237
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6350213885307312,
          "mean_weights": [
            0.5189062356948853,
            0.48109376430511475
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.4277062714099884,
          "mean_weights": [
            0.8402144908905029,
            0.15978553891181946
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6868079900741577,
          "mean_weights": [
            0.493515282869339,
            0.5064846873283386
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.38686758279800415,
          "mean_weights": [
            0.8638634085655212,
            0.13613657653331757
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6772255897521973,
          "mean_weights": [
            0.5679300427436829,
            0.43206995725631714
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.5690979361534119,
          "mean_weights": [
            0.7181376218795776,
            0.28186243772506714
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6592018008232117,
          "mean_weights": [
            0.5172787308692932,
            0.4827212691307068
          ],
          "sources": 2
        },
        {
          "entropy_mean": 1.038456916809082,
          "mean_weights": [
            0.4540913999080658,
            0.23118522763252258,
            0.3147233724594116
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0903832912445068,
          "mean_weights": [
            0.3152569830417633,
            0.30281925201416016,
            0.38192376494407654
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0411981344223022,
          "mean_weights": [
            0.4020236134529114,
            0.2803056836128235,
            0.31767070293426514
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0905810594558716,
          "mean_weights": [
            0.3278552293777466,
            0.2872077226638794,
            0.384937047958374
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0150185823440552,
          "mean_weights": [
            0.48287349939346313,
            0.22136211395263672,
            0.29576438665390015
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0816482305526733,
          "mean_weights": [
            0.3306047320365906,
            0.2639211118221283,
            0.40547415614128113
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0114538669586182,
          "mean_weights": [
            0.4296638071537018,
            0.22373110055923462,
            0.346605122089386
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0918333530426025,
          "mean_weights": [
            0.36806726455688477,
            0.3041593134403229,
            0.32777342200279236
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.3290088176727295,
          "mean_weights": [
            0.35130923986434937,
            0.17179939150810242,
            0.21658103168010712,
            0.2603103518486023
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.379033088684082,
          "mean_weights": [
            0.29040855169296265,
            0.23445811867713928,
            0.2306380271911621,
            0.24449528753757477
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3394957780838013,
          "mean_weights": [
            0.3262045383453369,
            0.18669262528419495,
            0.22525924444198608,
            0.26184359192848206
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3677098751068115,
          "mean_weights": [
            0.29889944195747375,
            0.22409284114837646,
            0.1924189329147339,
            0.2845887541770935
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.2910308837890625,
          "mean_weights": [
            0.3990757167339325,
            0.18164023756980896,
            0.16051796078681946,
            0.2587660551071167
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3623619079589844,
          "mean_weights": [
            0.3024129569530487,
            0.21113446354866028,
            0.18768057227134705,
            0.29877203702926636
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.2281851768493652,
          "mean_weights": [
            0.46696996688842773,
            0.16045212745666504,
            0.13728894293308258,
            0.23528897762298584
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3772211074829102,
          "mean_weights": [
            0.28748467564582825,
            0.24638661742210388,
            0.22340166568756104,
            0.24272704124450684
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.5519046783447266,
          "mean_weights": [
            0.30400484800338745,
            0.1470816731452942,
            0.13316048681735992,
            0.1807912439107895,
            0.23496176302433014
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.572178602218628,
          "mean_weights": [
            0.21885336935520172,
            0.2029750645160675,
            0.14561443030834198,
            0.14960341155529022,
            0.28295373916625977
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5391805171966553,
          "mean_weights": [
            0.2898171544075012,
            0.15984714031219482,
            0.1228363886475563,
            0.16408921778202057,
            0.26341012120246887
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5521924495697021,
          "mean_weights": [
            0.210292249917984,
            0.185291588306427,
            0.14173434674739838,
            0.14211969077587128,
            0.32056212425231934
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5468778610229492,
          "mean_weights": [
            0.2756267488002777,
            0.16966477036476135,
            0.1333748996257782,
            0.1590600460767746,
            0.26227355003356934
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5309922695159912,
          "mean_weights": [
            0.21606209874153137,
            0.27386677265167236,
            0.12764990329742432,
            0.10704417526721954,
            0.2753770351409912
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5832164287567139,
          "mean_weights": [
            0.22763225436210632,
            0.24636119604110718,
            0.17577087879180908,
            0.1817018985748291,
            0.1685338020324707
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5793015956878662,
          "mean_weights": [
            0.21581211686134338,
            0.21387425065040588,
            0.1464191973209381,
            0.17280855774879456,
            0.25108590722084045
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.7719972133636475,
          "mean_weights": [
            0.16962555050849915,
            0.20328447222709656,
            0.1659279465675354,
            0.14008209109306335,
            0.12920910120010376,
            0.1918708235025406
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.689546823501587,
          "mean_weights": [
            0.16601097583770752,
            0.1523149460554123,
            0.0875597670674324,
            0.101267509162426,
            0.1655988097190857,
            0.3272479772567749
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.5251271724700928,
          "mean_weights": [
            0.13745036721229553,
            0.11702707409858704,
            0.07592719793319702,
            0.08887732028961182,
            0.11522199213504791,
            0.4654960632324219
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7189464569091797,
          "mean_weights": [
            0.1744517982006073,
            0.15982553362846375,
            0.09849169105291367,
            0.10780535638332367,
            0.16236206889152527,
            0.29706352949142456
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.5877950191497803,
          "mean_weights": [
            0.14238224923610687,
            0.1503014713525772,
            0.10275131464004517,
            0.10159259289503098,
            0.08505517989397049,
            0.4179171919822693
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7369492053985596,
          "mean_weights": [
            0.17980654537677765,
            0.1706826537847519,
            0.10907262563705444,
            0.1106589287519455,
            0.15735921263694763,
            0.2724200189113617
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.5987625122070312,
          "mean_weights": [
            0.15493065118789673,
            0.15333226323127747,
            0.10736657679080963,
            0.1186048835515976,
            0.07128532975912094,
            0.39448028802871704
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.75795578956604,
          "mean_weights": [
            0.1946164071559906,
            0.17554126679897308,
            0.11572675406932831,
            0.1155121773481369,
            0.1708213835954666,
            0.2277819961309433
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.5344648361206055,
          "mean_weights": [
            0.0905577763915062,
            0.0786280632019043,
            0.06272733211517334,
            0.07359343767166138,
            0.050107162445783615,
            0.13821640610694885,
            0.5061697959899902
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.7138500213623047,
          "mean_weights": [
            0.13302072882652283,
            0.10984362661838531,
            0.07243296504020691,
            0.07083481550216675,
            0.08885611593723297,
            0.11790697276592255,
            0.4071047604084015
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.4565383195877075,
          "mean_weights": [
            0.09418222308158875,
            0.07457256317138672,
            0.057202234864234924,
            0.06210128590464592,
            0.04410415142774582,
            0.11846546828746796,
            0.5493720769882202
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.7785513401031494,
          "mean_weights": [
            0.1471671462059021,
            0.12308044731616974,
            0.07605892419815063,
            0.07563325762748718,
            0.10904586315155029,
            0.11226382851600647,
            0.3567505478858948
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.5135828256607056,
          "mean_weights": [
            0.10121430456638336,
            0.07933001220226288,
            0.061728835105895996,
            0.06817515194416046,
            0.048567138612270355,
            0.12093988060951233,
            0.5200446844100952
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.7546210289001465,
          "mean_weights": [
            0.14427465200424194,
            0.12224892526865005,
            0.08193168044090271,
            0.07876674830913544,
            0.09007281810045242,
            0.10526382178068161,
            0.3774413466453552
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.5284571647644043,
          "mean_weights": [
            0.10831618309020996,
            0.08754566311836243,
            0.06345342099666595,
            0.06688661873340607,
            0.05242554843425751,
            0.11085730791091919,
            0.5105152726173401
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.8309729099273682,
          "mean_weights": [
            0.1468513160943985,
            0.13658970594406128,
            0.09501184523105621,
            0.0928751677274704,
            0.10057065635919571,
            0.11721575260162354,
            0.31088554859161377
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.6937754154205322,
          "mean_weights": [
            0.09603364765644073,
            0.06767555326223373,
            0.045112669467926025,
            0.047895483672618866,
            0.03816477954387665,
            0.07518067210912704,
            0.3669250011444092,
            0.26301223039627075
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.9070472717285156,
          "mean_weights": [
            0.10919073224067688,
            0.11743452399969101,
            0.07601121068000793,
            0.0681401938199997,
            0.0658918172121048,
            0.07979628443717957,
            0.24526360630989075,
            0.2382715940475464
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.776512861251831,
          "mean_weights": [
            0.08731270581483841,
            0.08430957794189453,
            0.056895699352025986,
            0.06295455247163773,
            0.04594648256897926,
            0.09297705441713333,
            0.33455103635787964,
            0.23505286872386932
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.9573358297348022,
          "mean_weights": [
            0.11900999397039413,
            0.11752337962388992,
            0.07979564368724823,
            0.07266241312026978,
            0.07412992417812347,
            0.08811671286821365,
            0.22605092823505402,
            0.2227109968662262
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.7231189012527466,
          "mean_weights": [
            0.09091638028621674,
            0.07795695960521698,
            0.05235600098967552,
            0.053361356258392334,
            0.04162060469388962,
            0.07854442298412323,
            0.3417898118495941,
            0.2634544372558594
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.8967883586883545,
          "mean_weights": [
            0.1089041531085968,
            0.1000017300248146,
            0.07706406712532043,
            0.07702124863862991,
            0.06017591059207916,
            0.07969298958778381,
            0.261149525642395,
            0.23599036037921906
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.801592230796814,
          "mean_weights": [
            0.09432460367679596,
            0.09541776776313782,
            0.06706254929304123,
            0.07247316837310791,
            0.048089414834976196,
            0.08817503601312637,
            0.3473600149154663,
            0.1870974600315094
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.919801950454712,
          "mean_weights": [
            0.10887958109378815,
            0.10445678234100342,
            0.0838727131485939,
            0.0852852314710617,
            0.05865198373794556,
            0.08557502925395966,
            0.27338963747024536,
            0.19988907873630524
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.9332969188690186,
          "mean_weights": [
            0.07704077661037445,
            0.08837630599737167,
            0.05746292322874069,
            0.06001049652695656,
            0.04015304520726204,
            0.06584024429321289,
            0.2511133551597595,
            0.10944834351539612,
            0.25055450201034546
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.0115504264831543,
          "mean_weights": [
            0.08485299348831177,
            0.08635596185922623,
            0.06204982101917267,
            0.06870189309120178,
            0.0476498119533062,
            0.08640170097351074,
            0.2653427720069885,
            0.13704031705856323,
            0.16160473227500916
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.9781036376953125,
          "mean_weights": [
            0.08389372378587723,
            0.08841055631637573,
            0.060679104179143906,
            0.0658775195479393,
            0.045975662767887115,
            0.08423998951911926,
            0.29481440782546997,
            0.12572166323661804,
            0.15038736164569855
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.0586161613464355,
          "mean_weights": [
            0.1038442850112915,
            0.09918913245201111,
            0.08148118108510971,
            0.0698266476392746,
            0.059573929756879807,
            0.08160553872585297,
            0.22791330516338348,
            0.13585451245307922,
            0.1407114714384079
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.022735118865967,
          "mean_weights": [
            0.09707076847553253,
            0.09551473706960678,
            0.06701318919658661,
            0.0673459842801094,
            0.04480960965156555,
            0.08329527080059052,
            0.24251294136047363,
            0.15929152071475983,
            0.14314597845077515
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.9317231178283691,
          "mean_weights": [
            0.07746735960245132,
            0.08971680700778961,
            0.06909166276454926,
            0.08098623156547546,
            0.04421798139810562,
            0.0916394367814064,
            0.3483121693134308,
            0.10497690737247467,
            0.09359145164489746
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.053220272064209,
          "mean_weights": [
            0.09178322553634644,
            0.10647953301668167,
            0.07523074746131897,
            0.08394691348075867,
            0.04770101606845856,
            0.09626083076000214,
            0.26201331615448,
            0.13223087787628174,
            0.10435353964567184
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.0558863990008831,
        0.04748110845685005,
        0.061753273010253906,
        0.06085919216275215,
        0.11003358662128448,
        0.06251973658800125,
        0.1138310506939888,
        0.08963073790073395,
        0.13082429766654968,
        0.08407715708017349,
        0.10403942316770554,
        0.1004372090101242,
        0.1062953919172287,
        0.09022688120603561,
        0.10766621679067612,
        0.09703134745359421,
        0.11238158494234085,
        0.0712415874004364,
        0.08696430176496506,
        0.07720176130533218,
        0.08355452120304108,
        0.07461326569318771,
        0.08609789609909058,
        0.0729745551943779,
        0.100697360932827,
        0.06531906127929688,
        0.07805952429771423,
        0.06964895129203796,
        0.08197034895420074,
        0.08001933246850967,
        0.10499614477157593,
        0.11749708652496338,
        0.13756641745567322,
        0.09965827316045761,
        0.09882861375808716,
        0.0854005292057991,
        0.10110001266002655,
        0.08426972478628159,
        0.10269922018051147,
        0.0886540412902832,
        0.10988812148571014,
        0.04304825887084007,
        0.0575832724571228,
        0.0459144227206707,
        0.06622003018856049,
        0.05584317073225975,
        0.06460635364055634,
        0.06311307847499847,
        0.07165247201919556,
        0.05157631263136864,
        0.055589381605386734,
        0.05562245845794678,
        0.05917651578783989,
        0.059894196689128876,
        0.06219335272908211,
        0.06387592852115631,
        0.06750614196062088,
        0.050557736307382584,
        0.06024058535695076,
        0.06106090545654297,
        0.06803743541240692,
        0.06874485313892365,
        0.06952574104070663,
        0.07015988230705261
      ],
      "loss_nats": 1.43131685256958,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 1.843801498413086,
        "mean_weights": [
          0.11151862144470215,
          0.061874739825725555,
          0.05308101326227188,
          0.040848225355148315,
          0.042743273079395294,
          0.053944505751132965,
          0.1199670359492302,
          0.3250349462032318,
          0.19098761677742004
        ],
        "sources": 9
      },
      "step": 2000,
      "stream_state_rms_by_sublayer": [
        0.051234737038612366,
        0.0955304205417633,
        0.1796528398990631,
        0.19925470650196075,
        0.21105031669139862,
        0.2359117865562439,
        0.23972436785697937,
        0.25103330612182617,
        0.08815877884626389,
        0.09304249286651611,
        0.1068187803030014,
        0.11085781455039978,
        0.12298524379730225,
        0.12910984456539154,
        0.13432784378528595,
        0.14378803968429565,
        0.03348636254668236,
        0.04010729491710663,
        0.051035381853580475,
        0.06472750008106232,
        0.07822930067777634,
        0.09987020492553711,
        0.11166217923164368,
        0.14899125695228577,
        0.038218315690755844,
        0.05702618882060051,
        0.07459890842437744,
        0.10170809924602509,
        0.12532564997673035,
        0.15923865139484406,
        0.36271119117736816,
        0.38537684082984924,
        0.05078785493969917,
        0.05919196084141731,
        0.0794382095336914,
        0.08540526777505875,
        0.10824274271726608,
        0.1145314872264862,
        0.14269457757472992,
        0.15640178322792053,
        0.050489071756601334,
        0.06262174993753433,
        0.06721649318933487,
        0.08427268266677856,
        0.09199884533882141,
        0.11030438542366028,
        0.11444488167762756,
        0.13170360028743744,
        0.024430278688669205,
        0.05311359465122223,
        0.05695623904466629,
        0.08614693582057953,
        0.09002252668142319,
        0.12623678147792816,
        0.130380317568779,
        0.17503507435321808,
        0.030554499477148056,
        0.07505694776773453,
        0.07779927551746368,
        0.13522638380527496,
        0.13734298944473267,
        0.2187158316373825,
        0.22044414281845093,
        0.3292153775691986
      ]
    },
    {
      "activation_grad_rms_by_block": [
        3.57800890924409e-05,
        3.942943658330478e-05,
        3.388081677258015e-05,
        3.245583502575755e-05,
        3.3783475373638794e-05,
        2.915231198130641e-05,
        2.5950168492272496e-05,
        2.450849387969356e-05,
        2.7269536076346412e-05,
        2.568827403592877e-05,
        2.400098310317844e-05,
        2.2796592020313255e-05,
        2.010442767641507e-05,
        1.8936527339974418e-05,
        1.9039980543311685e-05,
        2.3072669137036428e-05,
        4.836148946196772e-05,
        4.9383146688342094e-05,
        4.5993543608346954e-05,
        4.280430221115239e-05,
        0.00014426946290768683,
        0.00011744296352844685,
        9.347966988570988e-05,
        7.698025729041547e-05,
        6.582303467439488e-05,
        5.6237244280055165e-05,
        4.4205091398907825e-05,
        3.852568261208944e-05,
        2.7434407456894405e-05,
        2.285257687617559e-05,
        2.028334711212665e-05,
        1.9826158677460626e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 3.186757840012433e-05,
        "first_to_last_ratio": 0.8636564556008468,
        "imbalance_abs_log_ratio": 0.146580210082559,
        "last_quartile_mean": 3.689844288601307e-05,
        "mean": 4.2179749868864747e-05,
        "normalized": [
          0.8482764645044092,
          0.9347954102594115,
          0.8032484042203742,
          0.7694648528419803,
          0.8009406285876599,
          0.6911447334785021,
          0.6152281265998635,
          0.5810488197746441,
          0.646507771172811,
          0.6090191173677569,
          0.569016724323795,
          0.5404629494292167,
          0.4766369582304063,
          0.4489483081063157,
          0.4514009827584627,
          0.5470082020108817,
          1.146557047216301,
          1.170778557053383,
          1.090417647125437,
          1.014806923801809,
          3.420348943656972,
          2.7843447126541196,
          2.216221532283493,
          1.825052484420704,
          1.5605363919661974,
          1.3332759073938238,
          1.048016916561615,
          0.913369157756135,
          0.6504165515961319,
          0.5417902416971032,
          0.48087879077488155,
          0.470039740375403
        ],
        "population_cv": 0.6827037752697015
      },
      "activation_output_rms_by_block": [
        0.15143299102783203,
        0.30262914299964905,
        0.3626292943954468,
        0.38562148809432983,
        0.13662196695804596,
        0.15442071855068207,
        0.18618692457675934,
        0.2074802666902542,
        0.04909220710396767,
        0.0881529152393341,
        0.14954733848571777,
        0.23951733112335205,
        0.08217982947826385,
        0.14409078657627106,
        0.22605396807193756,
        0.6592875719070435,
        0.06389845162630081,
        0.09458807110786438,
        0.14630787074565887,
        0.21908092498779297,
        0.07773751765489578,
        0.10736925154924393,
        0.1424422264099121,
        0.17246152460575104,
        0.06572399288415909,
        0.10123347491025925,
        0.14551502466201782,
        0.19983991980552673,
        0.09023923426866531,
        0.15600618720054626,
        0.24885882437229156,
        0.3860291540622711
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.2358778491616249,
        "first_to_last_ratio": 1.3542132581102162,
        "imbalance_abs_log_ratio": 0.30322066438495643,
        "last_quartile_mean": 0.17418072652071714,
        "mean": 0.18569613725412637,
        "normalized": [
          0.8154881047449846,
          1.6297007976288655,
          1.9528101109566227,
          2.076626330501449,
          0.7357286423846173,
          0.8315774406193291,
          1.0026429592445498,
          1.1173106223869165,
          0.26436848838048077,
          0.4747159340137284,
          0.8053335987331889,
          1.2898347518967022,
          0.442550021198342,
          0.7759492938675503,
          1.2173326349948854,
          3.5503569522547695,
          0.3441022122008675,
          0.5093701598026241,
          0.7878886061342009,
          1.1797818103667892,
          0.41862754284711634,
          0.5781986267291517,
          0.7670715638795383,
          0.9287297364173834,
          0.3539330104331431,
          0.5451566004936365,
          0.7836190176798323,
          1.0761662722797756,
          0.4859510574803844,
          0.8401154138551131,
          1.3401400161152877,
          2.078821669478174
        ],
        "population_cv": 0.6621635595354842
      },
      "bits_per_byte": 1.9288425276664853,
      "branch_output_rms_by_sublayer": [
        0.08832799643278122,
        0.11756844818592072,
        0.17041416466236115,
        0.05185626819729805,
        0.08745641261339188,
        0.0881219431757927,
        0.05030680447816849,
        0.03964918479323387,
        0.12814420461654663,
        0.022397184744477272,
        0.05527547001838684,
        0.022468235343694687,
        0.06746833771467209,
        0.031668391078710556,
        0.04974694177508354,
        0.045926205813884735,
        0.03680809214711189,
        0.03180550038814545,
        0.03558321297168732,
        0.03912416473031044,
        0.051462169736623764,
        0.06056958809494972,
        0.0526844747364521,
        0.10351218283176422,
        0.05210389941930771,
        0.05620818957686424,
        0.03620988130569458,
        0.0675361230969429,
        0.04957791045308113,
        0.0823286920785904,
        0.4858623445034027,
        0.07062793523073196,
        0.04707946628332138,
        0.0371064618229866,
        0.04491531476378441,
        0.023919809609651566,
        0.07858023792505264,
        0.035764746367931366,
        0.09729185700416565,
        0.07103991508483887,
        0.05887215584516525,
        0.045838613063097,
        0.026784580200910568,
        0.05455324053764343,
        0.041033755987882614,
        0.06383209675550461,
        0.032232560217380524,
        0.07182251662015915,
        0.02874751202762127,
        0.05687069892883301,
        0.022572210058569908,
        0.052836012095212936,
        0.02659173309803009,
        0.06559387594461441,
        0.025225715711712837,
        0.07689779996871948,
        0.041177187114953995,
        0.07740812748670578,
        0.027006076648831367,
        0.08842137455940247,
        0.023083338513970375,
        0.11899886280298233,
        0.016772178933024406,
        0.16321057081222534
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00017320581296885245,
        0.0001375362640366496,
        0.00014375813569971144,
        8.981648567965897e-05,
        0.00010459593200616139,
        6.156395218407552e-05,
        7.281931709149581e-05,
        7.667720183186952e-05,
        7.236475672264944e-05,
        7.890209128468536e-05,
        9.134838485444255e-05,
        0.00011250740948005864,
        7.70610699389613e-05,
        7.611216582142322e-05,
        8.146874247135892e-05,
        0.00017928549432252573,
        0.00013712162717798398,
        0.00013184310037104538,
        0.00022367504753246386,
        0.0002691717754589954,
        0.0008753648193242955,
        0.0006614972326966637,
        0.0006052862860379005,
        0.0005306195765578692,
        0.00037252876038501256,
        0.00029795485316373564,
        0.0002868510641817651,
        0.00023542884481748448,
        0.00018331496216173493,
        0.00013991664250432274,
        0.00012258431160994327,
        0.00012212563876750547
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00010749663768730933,
        "first_to_last_ratio": 0.4884254111851857,
        "imbalance_abs_log_ratio": 0.7165685086453754,
        "last_quartile_mean": 0.00022008813469893802,
        "mean": 0.0002132596174732282,
        "normalized": [
          0.812182892481255,
          0.6449240867362775,
          0.6740991914128248,
          0.42116030565859097,
          0.4904629073494984,
          0.2886807775119639,
          0.3414585375059921,
          0.3595486231306557,
          0.33932704925597945,
          0.3699813974138373,
          0.428343565166106,
          0.5275607769210333,
          0.36134862686150726,
          0.35689910130771973,
          0.3820167335786683,
          0.840691249692561,
          0.6429798046280267,
          0.6182281574597519,
          1.0488392044524941,
          1.262178834644052,
          4.104690938190328,
          3.101840097690814,
          2.838260207016858,
          2.4881390243724,
          1.746832170097925,
          1.3971461484082413,
          1.3450791461621905,
          1.1039541738230836,
          0.8595859091079332,
          0.6560859677143862,
          0.5748125831902143,
          0.5726618110568292
        ],
        "population_cv": 0.9042592730039112
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.2508508265018463,
          "mean_weights": [
            0.9251874089241028,
            0.07481260597705841
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.5796181559562683,
          "mean_weights": [
            0.6163991689682007,
            0.38360080122947693
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.3817020058631897,
          "mean_weights": [
            0.8651303648948669,
            0.13486966490745544
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6884468793869019,
          "mean_weights": [
            0.4932539463043213,
            0.5067461133003235
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.2797870337963104,
          "mean_weights": [
            0.9142746329307556,
            0.08572537451982498
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6642253994941711,
          "mean_weights": [
            0.605044960975647,
            0.39495497941970825
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.5038842558860779,
          "mean_weights": [
            0.7756115198135376,
            0.2243884801864624
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6434851884841919,
          "mean_weights": [
            0.5374104976654053,
            0.4625895023345947
          ],
          "sources": 2
        },
        {
          "entropy_mean": 1.0016276836395264,
          "mean_weights": [
            0.4353652000427246,
            0.18586981296539307,
            0.3787649869918823
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0655895471572876,
          "mean_weights": [
            0.2726097106933594,
            0.2790643572807312,
            0.4483259320259094
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0300546884536743,
          "mean_weights": [
            0.40803641080856323,
            0.2817816734313965,
            0.3101818859577179
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0874333381652832,
          "mean_weights": [
            0.36608952283859253,
            0.27405625581741333,
            0.35985422134399414
          ],
          "sources": 3
        },
        {
          "entropy_mean": 0.9672750234603882,
          "mean_weights": [
            0.5343947410583496,
            0.1846013069152832,
            0.2810039520263672
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0731852054595947,
          "mean_weights": [
            0.3801807165145874,
            0.24373164772987366,
            0.37608763575553894
          ],
          "sources": 3
        },
        {
          "entropy_mean": 0.9721636772155762,
          "mean_weights": [
            0.49137914180755615,
            0.2109004259109497,
            0.29772043228149414
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0864832401275635,
          "mean_weights": [
            0.39626431465148926,
            0.29088544845581055,
            0.3128502070903778
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.3038380146026611,
          "mean_weights": [
            0.35696184635162354,
            0.1472446322441101,
            0.19980239868164062,
            0.29599112272262573
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3611116409301758,
          "mean_weights": [
            0.3264957070350647,
            0.22676822543144226,
            0.20138126611709595,
            0.2453548014163971
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3161137104034424,
          "mean_weights": [
            0.3259783387184143,
            0.17862647771835327,
            0.20532535016536713,
            0.2900698482990265
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3568220138549805,
          "mean_weights": [
            0.3223523497581482,
            0.21851874887943268,
            0.1804703176021576,
            0.27865859866142273
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.2413711547851562,
          "mean_weights": [
            0.4272037148475647,
            0.16667884588241577,
            0.1365978717803955,
            0.269519567489624
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3540384769439697,
          "mean_weights": [
            0.32447606325149536,
            0.2030642032623291,
            0.18181107938289642,
            0.2906486392021179
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.1544246673583984,
          "mean_weights": [
            0.5128031969070435,
            0.13654224574565887,
            0.12138728052377701,
            0.22926729917526245
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3712997436523438,
          "mean_weights": [
            0.3103735148906708,
            0.24219870567321777,
            0.2247106432914734,
            0.22271710634231567
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.5205519199371338,
          "mean_weights": [
            0.3307538628578186,
            0.1283297836780548,
            0.12636876106262207,
            0.17177441716194153,
            0.2427731454372406
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.518624186515808,
          "mean_weights": [
            0.2150207757949829,
            0.17106646299362183,
            0.13557285070419312,
            0.11898039281368256,
            0.3593595027923584
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.4772696495056152,
          "mean_weights": [
            0.30324018001556396,
            0.13238273561000824,
            0.09796822816133499,
            0.15785877406597137,
            0.30855005979537964
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.4979676008224487,
          "mean_weights": [
            0.20932656526565552,
            0.16542977094650269,
            0.1272144764661789,
            0.1160244345664978,
            0.3820047676563263
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.49334716796875,
          "mean_weights": [
            0.28671571612358093,
            0.13947691023349762,
            0.11353637278079987,
            0.14176231622695923,
            0.31850868463516235
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5120060443878174,
          "mean_weights": [
            0.21571135520935059,
            0.28788846731185913,
            0.131511390209198,
            0.08343321830034256,
            0.2814555764198303
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5766459703445435,
          "mean_weights": [
            0.2565065026283264,
            0.22934341430664062,
            0.1593727320432663,
            0.1524350941181183,
            0.20234227180480957
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5796699523925781,
          "mean_weights": [
            0.2356114238500595,
            0.21223093569278717,
            0.1439642310142517,
            0.1658918261528015,
            0.2423015832901001
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.7626371383666992,
          "mean_weights": [
            0.16389364004135132,
            0.20886825025081635,
            0.18027648329734802,
            0.1462823748588562,
            0.11240866780281067,
            0.18827059864997864
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.6546597480773926,
          "mean_weights": [
            0.17577320337295532,
            0.14207448065280914,
            0.08111412078142166,
            0.0931762307882309,
            0.15011057257652283,
            0.35775139927864075
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.430190086364746,
          "mean_weights": [
            0.1354856789112091,
            0.09479984641075134,
            0.07003438472747803,
            0.0834486186504364,
            0.09165525436401367,
            0.5245761871337891
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7008471488952637,
          "mean_weights": [
            0.18653985857963562,
            0.149653822183609,
            0.09563066065311432,
            0.09960515052080154,
            0.15559062361717224,
            0.31297987699508667
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.514115333557129,
          "mean_weights": [
            0.14724519848823547,
            0.13274209201335907,
            0.096603162586689,
            0.11347334086894989,
            0.0527515783905983,
            0.4571846127510071
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7213332653045654,
          "mean_weights": [
            0.19527462124824524,
            0.16114139556884766,
            0.10406571626663208,
            0.10423287749290466,
            0.14977452158927917,
            0.2855108380317688
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.503009557723999,
          "mean_weights": [
            0.1677132397890091,
            0.12867671251296997,
            0.09363879263401031,
            0.12237933278083801,
            0.04204780235886574,
            0.44554415345191956
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7531253099441528,
          "mean_weights": [
            0.21246549487113953,
            0.17559614777565002,
            0.11484666168689728,
            0.11058571934700012,
            0.1656094789505005,
            0.22089649736881256
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.3981997966766357,
          "mean_weights": [
            0.08501995354890823,
            0.06276540458202362,
            0.0543542318046093,
            0.06455785036087036,
            0.03336101397871971,
            0.12979380786418915,
            0.5701477527618408
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.6064693927764893,
          "mean_weights": [
            0.12975823879241943,
            0.09198254346847534,
            0.06163205951452255,
            0.06021754443645477,
            0.078399658203125,
            0.10503693670034409,
            0.4729730188846588
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.3297276496887207,
          "mean_weights": [
            0.09028719365596771,
            0.057085148990154266,
            0.04790058359503746,
            0.05685827136039734,
            0.031231366097927094,
            0.11244906485080719,
            0.6041883826255798
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.6772459745407104,
          "mean_weights": [
            0.1455874890089035,
            0.10527922213077545,
            0.06669703871011734,
            0.06069611385464668,
            0.09951996803283691,
            0.09685218334197998,
            0.4253680109977722
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.3742375373840332,
          "mean_weights": [
            0.09774194657802582,
            0.06367622315883636,
            0.05300067365169525,
            0.05981069803237915,
            0.03241024166345596,
            0.10956928879022598,
            0.5837908983230591
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.672610878944397,
          "mean_weights": [
            0.14811497926712036,
            0.10026681423187256,
            0.0711982250213623,
            0.06714795529842377,
            0.08556141704320908,
            0.09809904545545578,
            0.42961156368255615
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.4325350522994995,
          "mean_weights": [
            0.11123399436473846,
            0.07182648032903671,
            0.05620439350605011,
            0.06410455703735352,
            0.03741488978266716,
            0.10516737401485443,
            0.5540482997894287
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.7746202945709229,
          "mean_weights": [
            0.15383490920066833,
            0.11435382068157196,
            0.09083531051874161,
            0.08329388499259949,
            0.08837151527404785,
            0.11093884706497192,
            0.3583717346191406
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.5557539463043213,
          "mean_weights": [
            0.09889799356460571,
            0.05227670818567276,
            0.03886467218399048,
            0.039664674550294876,
            0.02741250954568386,
            0.06309205293655396,
            0.40426212549209595,
            0.27552926540374756
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.7942585945129395,
          "mean_weights": [
            0.10892648994922638,
            0.08669497072696686,
            0.060229551047086716,
            0.05406671017408371,
            0.05429235100746155,
            0.07220236957073212,
            0.24074554443359375,
            0.322842001914978
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.6406831741333008,
          "mean_weights": [
            0.08106359094381332,
            0.06122073531150818,
            0.04382575303316116,
            0.0495278537273407,
            0.031025834381580353,
            0.08371539413928986,
            0.3212447166442871,
            0.3283761143684387
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.8771746158599854,
          "mean_weights": [
            0.11823929846286774,
            0.09187044948339462,
            0.06966835260391235,
            0.05918115749955177,
            0.06588791310787201,
            0.07802121341228485,
            0.22871823608875275,
            0.2884133756160736
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.612527847290039,
          "mean_weights": [
            0.09160974621772766,
            0.05898914486169815,
            0.039636120200157166,
            0.04581688344478607,
            0.032106831669807434,
            0.06671778112649918,
            0.3060194253921509,
            0.35910406708717346
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.8253357410430908,
          "mean_weights": [
            0.10355446487665176,
            0.08380813896656036,
            0.06609268486499786,
            0.07030215114355087,
            0.04850628599524498,
            0.0775601714849472,
            0.25825831294059753,
            0.2919177711009979
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.714170217514038,
          "mean_weights": [
            0.09696917235851288,
            0.07828262448310852,
            0.055068351328372955,
            0.0616651251912117,
            0.03598042577505112,
            0.07802210748195648,
            0.3351396918296814,
            0.25887250900268555
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.875918984413147,
          "mean_weights": [
            0.11156761646270752,
            0.0880264937877655,
            0.07785218954086304,
            0.07930342108011246,
            0.04990004748106003,
            0.08684437721967697,
            0.2776731252670288,
            0.22883272171020508
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.731968641281128,
          "mean_weights": [
            0.06617744266986847,
            0.05556279793381691,
            0.04319432005286217,
            0.045391663908958435,
            0.026374589651823044,
            0.054588086903095245,
            0.21148523688316345,
            0.10446193069219589,
            0.3927639424800873
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.9649744033813477,
          "mean_weights": [
            0.07813189923763275,
            0.08751581609249115,
            0.05837181955575943,
            0.07608447968959808,
            0.03110998496413231,
            0.09630586206912994,
            0.27242881059646606,
            0.1154000535607338,
            0.18465125560760498
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.8779311180114746,
          "mean_weights": [
            0.07922039180994034,
            0.064256951212883,
            0.04481552541255951,
            0.05368471518158913,
            0.03291679173707962,
            0.07845190167427063,
            0.26414477825164795,
            0.13897250592708588,
            0.24353644251823425
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.9959204196929932,
          "mean_weights": [
            0.10073552280664444,
            0.0823616087436676,
            0.0686727911233902,
            0.0658436119556427,
            0.048376426100730896,
            0.07448098808526993,
            0.23861894011497498,
            0.14355987310409546,
            0.177350252866745
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.9298079013824463,
          "mean_weights": [
            0.09570661187171936,
            0.07176163792610168,
            0.05499213933944702,
            0.057467732578516006,
            0.03367581591010094,
            0.06890923529863358,
            0.21427617967128754,
            0.18804281949996948,
            0.2151678204536438
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.8635971546173096,
          "mean_weights": [
            0.07685244828462601,
            0.07133444398641586,
            0.05962139368057251,
            0.08040933310985565,
            0.033340416848659515,
            0.10067727416753769,
            0.36328762769699097,
            0.11175162345170975,
            0.10272547602653503
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.0009379386901855,
          "mean_weights": [
            0.09139549732208252,
            0.08807491511106491,
            0.06457877159118652,
            0.07007958739995956,
            0.03462113067507744,
            0.09155915677547455,
            0.2692604660987854,
            0.15421991050243378,
            0.13621056079864502
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.059407927095890045,
        0.05594591051340103,
        0.07886742800474167,
        0.07073435187339783,
        0.16177938878536224,
        0.06750936061143875,
        0.15622827410697937,
        0.10916222631931305,
        0.1895449161529541,
        0.11270010471343994,
        0.15217198431491852,
        0.14727257192134857,
        0.14916078746318817,
        0.11801524460315704,
        0.14915435016155243,
        0.13372685015201569,
        0.16036193072795868,
        0.09399056434631348,
        0.12247956544160843,
        0.10827244818210602,
        0.12010690569877625,
        0.10214843600988388,
        0.12369577586650848,
        0.09725081920623779,
        0.14881008863449097,
        0.0902416780591011,
        0.10321364551782608,
        0.09460548311471939,
        0.1168464794754982,
        0.11164536327123642,
        0.16267083585262299,
        0.19675607979297638,
        0.21974295377731323,
        0.15372900664806366,
        0.1438056081533432,
        0.10524243861436844,
        0.14797262847423553,
        0.10666082799434662,
        0.15148812532424927,
        0.11902136355638504,
        0.1692686378955841,
        0.05537491664290428,
        0.07994630187749863,
        0.06347332894802094,
        0.09600856900215149,
        0.07961282879114151,
        0.09550122171640396,
        0.09120825678110123,
        0.10489360988140106,
        0.07538540661334991,
        0.07298607379198074,
        0.07336290925741196,
        0.08083610236644745,
        0.07823792845010757,
        0.0843365490436554,
        0.08682125806808472,
        0.09205732494592667,
        0.057621169835329056,
        0.07988084107637405,
        0.0771951824426651,
        0.08835897594690323,
        0.09140052646398544,
        0.09466564655303955,
        0.09293081611394882
      ],
      "loss_nats": 1.3369717597961426,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 1.7290174961090088,
        "mean_weights": [
          0.10267993807792664,
          0.05008202791213989,
          0.044380687177181244,
          0.0326685830950737,
          0.033161722123622894,
          0.043591417372226715,
          0.1060870811343193,
          0.3520195186138153,
          0.2353290319442749
        ],
        "sources": 9
      },
      "step": 4000,
      "stream_state_rms_by_sublayer": [
        0.08832799643278122,
        0.15143299102783203,
        0.2790028154850006,
        0.30262914299964905,
        0.315830796957016,
        0.3626292943954468,
        0.3664044439792633,
        0.38562148809432983,
        0.12814420461654663,
        0.13662196695804596,
        0.14859555661678314,
        0.15442071855068207,
        0.17815031111240387,
        0.18618692457675934,
        0.19290222227573395,
        0.2074802666902542,
        0.03680809214711189,
        0.04909220710396767,
        0.06554166972637177,
        0.0881529152393341,
        0.11251214146614075,
        0.14954733848571777,
        0.16864995658397675,
        0.23951733112335205,
        0.05210389941930771,
        0.08217982947826385,
        0.09766767174005508,
        0.14409078657627106,
        0.16688434779644012,
        0.22605396807193756,
        0.6339711546897888,
        0.6592875719070435,
        0.04707946628332138,
        0.06389845162630081,
        0.084575355052948,
        0.09458807110786438,
        0.13482175767421722,
        0.14630787074565887,
        0.19051826000213623,
        0.21908092498779297,
        0.05887215584516525,
        0.07773751765489578,
        0.08281512558460236,
        0.10736925154924393,
        0.11754488199949265,
        0.1424422264099121,
        0.14818532764911652,
        0.17246152460575104,
        0.02874751202762127,
        0.06572399288415909,
        0.07124026119709015,
        0.10123347491025925,
        0.10666517913341522,
        0.14551502466201782,
        0.14956052601337433,
        0.19983991980552673,
        0.041177187114953995,
        0.09023923426866531,
        0.09537366032600403,
        0.15600618720054626,
        0.15850447118282318,
        0.24885882437229156,
        0.25068944692611694,
        0.3860291540622711
      ]
    },
    {
      "activation_grad_rms_by_block": [
        4.789723243447952e-05,
        5.257939483271912e-05,
        4.447350511327386e-05,
        4.274819002603181e-05,
        4.96310785820242e-05,
        4.17958217440173e-05,
        3.729292075149715e-05,
        3.545484287315048e-05,
        4.479578274185769e-05,
        4.205213917884976e-05,
        3.91026733268518e-05,
        3.680578447529115e-05,
        3.1189843866741285e-05,
        2.9126334993634373e-05,
        2.9360509870457463e-05,
        3.65210507879965e-05,
        8.83217144291848e-05,
        8.915644866647199e-05,
        8.140744466800243e-05,
        7.45641955290921e-05,
        0.0002545347670093179,
        0.0001978082291316241,
        0.000154864537762478,
        0.00012389873154461384,
        0.00012786213483195752,
        0.00010459556506248191,
        7.644320430699736e-05,
        6.138104072306305e-05,
        6.131517875473946e-05,
        4.588167939800769e-05,
        3.516174911055714e-05,
        3.2676212867954746e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 4.398412329464918e-05,
        "first_to_last_ratio": 0.6452634668607965,
        "imbalance_abs_log_ratio": 0.43809656983670076,
        "last_quartile_mean": 6.816459563196986e-05,
        "mean": 7.03343731061068e-05,
        "normalized": [
          0.6809932372926891,
          0.7475632825133391,
          0.6323153694166137,
          0.6077851857944574,
          0.7056447138179579,
          0.5942446047107566,
          0.530223262177042,
          0.504089839823597,
          0.6368974480554273,
          0.5978888745536934,
          0.5559539610577222,
          0.5232972563751404,
          0.4434509399790644,
          0.41411238498840724,
          0.4174418363858031,
          0.5192489699581264,
          1.2557404086895352,
          1.2676084925356499,
          1.1574347089891701,
          1.060138766241709,
          3.6189242296270336,
          2.81239769967395,
          2.2018329150221976,
          1.7615672973682381,
          1.8179181698124194,
          1.4871187506667403,
          1.086854135909831,
          0.872703317202576,
          0.8717669049561169,
          0.6523365087620864,
          0.4999226915339381,
          0.46458383610897114
        ],
        "population_cv": 0.7299683279122143
      },
      "activation_output_rms_by_block": [
        0.15345558524131775,
        0.3249167501926422,
        0.3818626403808594,
        0.40600064396858215,
        0.1351035088300705,
        0.15324510633945465,
        0.18001793324947357,
        0.20391276478767395,
        0.04374876990914345,
        0.08211024105548859,
        0.1380448192358017,
        0.2295975536108017,
        0.08199761807918549,
        0.13892096281051636,
        0.21687114238739014,
        0.7360114455223083,
        0.057135771960020065,
        0.08251863718032837,
        0.12661415338516235,
        0.19619159400463104,
        0.06753352284431458,
        0.095451220870018,
        0.1274663656949997,
        0.15628865361213684,
        0.058436937630176544,
        0.08794140070676804,
        0.12247690558433533,
        0.1640285700559616,
        0.07537907361984253,
        0.12290703505277634,
        0.19157102704048157,
        0.29249706864356995
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.24231436662375927,
        "first_to_last_ratio": 1.7382073612286646,
        "imbalance_abs_log_ratio": 0.5528543299844475,
        "last_quartile_mean": 0.139404752291739,
        "mean": 0.17594548198394477,
        "normalized": [
          0.872176901111452,
          1.8466899321818973,
          2.1703463827261262,
          2.3075366266331874,
          0.767871429869389,
          0.870980627700565,
          1.0231460974138593,
          1.1589542538312023,
          0.24864957835709345,
          0.4666800200245053,
          0.7845885991457198,
          1.3049357734104974,
          0.46603991847126747,
          0.789568230136157,
          1.2326042132027004,
          4.183179000808161,
          0.32473565854469494,
          0.4690011715552792,
          0.7196215097851506,
          1.1150703717560293,
          0.38383209436703286,
          0.5425045292082467,
          0.7244651255474189,
          0.8882788682598759,
          0.3321309360788757,
          0.49982187502145003,
          0.6961071361611404,
          0.9322692927740504,
          0.42842289992260757,
          0.6985518108614562,
          1.0888090156129309,
          1.6624301195199809
        ],
        "population_cv": 0.7619463065887785
      },
      "bits_per_byte": 1.877417995231997,
      "branch_output_rms_by_sublayer": [
        0.09253507107496262,
        0.116500623524189,
        0.19514523446559906,
        0.0468219518661499,
        0.07917427271604538,
        0.09375449270009995,
        0.04215213656425476,
        0.041188232600688934,
        0.12552575767040253,
        0.024237047880887985,
        0.05027730390429497,
        0.022429833188652992,
        0.054851483553647995,
        0.03027583472430706,
        0.036243390291929245,
        0.04546776041388512,
        0.031880609691143036,
        0.030174262821674347,
        0.035054463893175125,
        0.03663193807005882,
        0.050445348024368286,
        0.05748353153467178,
        0.05600574240088463,
        0.10596779733896255,
        0.053015924990177155,
        0.05394246056675911,
        0.032114963978528976,
        0.06537146866321564,
        0.048427484929561615,
        0.08060523867607117,
        0.5742782950401306,
        0.06690707802772522,
        0.04075472429394722,
        0.033449217677116394,
        0.03505101054906845,
        0.022565271705389023,
        0.06577442586421967,
        0.036697570234537125,
        0.08553697913885117,
        0.07367803901433945,
        0.0493597537279129,
        0.042231619358062744,
        0.023477228358387947,
        0.052135858684778214,
        0.03512806445360184,
        0.06269875913858414,
        0.030729109421372414,
        0.0728035643696785,
        0.023035654798150063,
        0.051585447043180466,
        0.019568851217627525,
        0.04862172156572342,
        0.02376176416873932,
        0.05972515419125557,
        0.02176014520227909,
        0.06787826865911484,
        0.03424545377492905,
        0.06439156830310822,
        0.021422436460852623,
        0.07232685387134552,
        0.019477492198348045,
        0.09627991914749146,
        0.017491821199655533,
        0.1271906942129135
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00028365814152869,
        0.0002107335686261075,
        0.00020559059302825965,
        0.00013034830149957224,
        0.00015811506316286034,
        8.078727112532468e-05,
        9.745502139202278e-05,
        0.00010158534732968689,
        0.00010687310625858127,
        0.00011249947044062199,
        0.00014466041390513986,
        0.00018570148639523412,
        0.0001198924920321974,
        0.00012288623237884872,
        0.00013187077826681118,
        0.000321118955435273,
        0.0002173403860671073,
        0.00022706794561596516,
        0.0003846029635623724,
        0.00045310883243459936,
        0.001499135691470673,
        0.0011394398675334944,
        0.0010162826769661553,
        0.0008650696174683238,
        0.0006964490714203023,
        0.0005559793522399823,
        0.0005128319422021078,
        0.0003792784055626199,
        0.00037680605737877213,
        0.0002696585960614849,
        0.00021598580145167561,
        0.00019464643295656736
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00015853416346156552,
        "first_to_last_ratio": 0.39613292787359505,
        "imbalance_abs_log_ratio": 0.9260054476059185,
        "last_quartile_mean": 0.000400204457409189,
        "mean": 0.0003599206213499198,
        "normalized": [
          0.7881130579981791,
          0.5855001245433764,
          0.5712109305023165,
          0.36215846986118033,
          0.439305373973387,
          0.2244585787341764,
          0.27076809610549,
          0.28224375404965807,
          0.29693521270813145,
          0.3125674489521634,
          0.4019231056074973,
          0.5159512275199496,
          0.3331081491872516,
          0.34142592863379456,
          0.36638850469921974,
          0.892193823823939,
          0.6038564427121444,
          0.6308834008018855,
          1.0685771827128963,
          1.258913231298522,
          4.165184217142125,
          3.165808792116179,
          2.8236300358520197,
          2.4035011226191765,
          1.9350074158246267,
          1.544727696220138,
          1.4248473462806275,
          1.0537834818691318,
          1.0469143334036313,
          0.7492168552335295,
          0.6000928778173319,
          0.540803781196325
        ],
        "population_cv": 0.9317377718256953
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.2200722098350525,
          "mean_weights": [
            0.9384047985076904,
            0.06159520894289017
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.5651201605796814,
          "mean_weights": [
            0.6557347774505615,
            0.3442652225494385
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.4018177092075348,
          "mean_weights": [
            0.856421947479248,
            0.14357808232307434
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.687358021736145,
          "mean_weights": [
            0.46872246265411377,
            0.5312775373458862
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.2678314447402954,
          "mean_weights": [
            0.920531153678894,
            0.07946885377168655
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.657238245010376,
          "mean_weights": [
            0.6222448348999023,
            0.37775513529777527
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.5044373869895935,
          "mean_weights": [
            0.7792346477508545,
            0.2207653522491455
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.650418758392334,
          "mean_weights": [
            0.5397791862487793,
            0.4602207541465759
          ],
          "sources": 2
        },
        {
          "entropy_mean": 1.0032464265823364,
          "mean_weights": [
            0.4337552487850189,
            0.18956376612186432,
            0.37668097019195557
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0385558605194092,
          "mean_weights": [
            0.24478834867477417,
            0.2602521479129791,
            0.4949595034122467
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.031773328781128,
          "mean_weights": [
            0.40370821952819824,
            0.2651250958442688,
            0.33116668462753296
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0838522911071777,
          "mean_weights": [
            0.35612422227859497,
            0.2625962495803833,
            0.38127952814102173
          ],
          "sources": 3
        },
        {
          "entropy_mean": 0.96053147315979,
          "mean_weights": [
            0.5271762013435364,
            0.1581989824771881,
            0.3146248161792755
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0704245567321777,
          "mean_weights": [
            0.3788670599460602,
            0.23852786421775818,
            0.38260507583618164
          ],
          "sources": 3
        },
        {
          "entropy_mean": 0.9674104452133179,
          "mean_weights": [
            0.48829180002212524,
            0.18356263637542725,
            0.3281455934047699
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0832805633544922,
          "mean_weights": [
            0.40388280153274536,
            0.2811351418495178,
            0.3149820566177368
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.2917464971542358,
          "mean_weights": [
            0.3349481225013733,
            0.1244347095489502,
            0.20291922986507416,
            0.33769792318344116
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3533791303634644,
          "mean_weights": [
            0.32115638256073,
            0.20602071285247803,
            0.18808895349502563,
            0.28473395109176636
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.308367371559143,
          "mean_weights": [
            0.31139707565307617,
            0.16179001331329346,
            0.2296154499053955,
            0.29719749093055725
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3437637090682983,
          "mean_weights": [
            0.3217787742614746,
            0.1920534074306488,
            0.1763250231742859,
            0.3098427951335907
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.2280659675598145,
          "mean_weights": [
            0.425346314907074,
            0.14848899841308594,
            0.13232898712158203,
            0.29383569955825806
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3462648391723633,
          "mean_weights": [
            0.3221249580383301,
            0.18557843565940857,
            0.18214309215545654,
            0.3101535141468048
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.147221326828003,
          "mean_weights": [
            0.505380392074585,
            0.12089012563228607,
            0.11514648050069809,
            0.2585829496383667
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3729398250579834,
          "mean_weights": [
            0.30785465240478516,
            0.23079943656921387,
            0.23862706124782562,
            0.22271886467933655
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.5232064723968506,
          "mean_weights": [
            0.3136816620826721,
            0.11285122483968735,
            0.13196846842765808,
            0.19688263535499573,
            0.24461600184440613
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.4876129627227783,
          "mean_weights": [
            0.2035508155822754,
            0.15372204780578613,
            0.13167491555213928,
            0.11437506973743439,
            0.396677166223526
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.4587602615356445,
          "mean_weights": [
            0.28519001603126526,
            0.10532154142856598,
            0.09675675630569458,
            0.17811742424964905,
            0.33461427688598633
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.4565696716308594,
          "mean_weights": [
            0.20232918858528137,
            0.14928825199604034,
            0.12029382586479187,
            0.1059027910232544,
            0.4221859574317932
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.4788098335266113,
          "mean_weights": [
            0.2697942852973938,
            0.11948063224554062,
            0.11090938746929169,
            0.15228068828582764,
            0.34753501415252686
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5018762350082397,
          "mean_weights": [
            0.208971306681633,
            0.27331358194351196,
            0.12766750156879425,
            0.07916607707738876,
            0.31088152527809143
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5627843141555786,
          "mean_weights": [
            0.27452588081359863,
            0.21448183059692383,
            0.143874391913414,
            0.14059773087501526,
            0.22652018070220947
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.580754280090332,
          "mean_weights": [
            0.2518634498119354,
            0.18758198618888855,
            0.13903988897800446,
            0.18044015765190125,
            0.24107450246810913
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.7564777135849,
          "mean_weights": [
            0.15440437197685242,
            0.19411900639533997,
            0.17806550860404968,
            0.14696577191352844,
            0.10594475269317627,
            0.2205006331205368
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.5899518728256226,
          "mean_weights": [
            0.1636897623538971,
            0.11789832264184952,
            0.07030131667852402,
            0.09238424897193909,
            0.1399868130683899,
            0.4157395362854004
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.2894535064697266,
          "mean_weights": [
            0.1212274506688118,
            0.06944876164197922,
            0.05726535990834236,
            0.09096578508615494,
            0.06335186958312988,
            0.5977407693862915
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.6581614017486572,
          "mean_weights": [
            0.18132120370864868,
            0.12711825966835022,
            0.08560528606176376,
            0.09748884290456772,
            0.15072739124298096,
            0.35773903131484985
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.3699336051940918,
          "mean_weights": [
            0.13604651391506195,
            0.10151475667953491,
            0.07715043425559998,
            0.11300846189260483,
            0.033717285841703415,
            0.538562536239624
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.6926933526992798,
          "mean_weights": [
            0.19425281882286072,
            0.14001017808914185,
            0.09434761106967926,
            0.10331346094608307,
            0.14820793271064758,
            0.3198680281639099
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.4148362874984741,
          "mean_weights": [
            0.16652247309684753,
            0.10329679399728775,
            0.08063912391662598,
            0.1165856122970581,
            0.033275946974754333,
            0.4996800422668457
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.740808129310608,
          "mean_weights": [
            0.21459883451461792,
            0.15640267729759216,
            0.10570268332958221,
            0.11097012460231781,
            0.17001523077487946,
            0.24231043457984924
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.3179612159729004,
          "mean_weights": [
            0.08068332821130753,
            0.049232255667448044,
            0.045073021203279495,
            0.06111945956945419,
            0.028317857533693314,
            0.13192236423492432,
            0.6036517024040222
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.5319744348526,
          "mean_weights": [
            0.12406684458255768,
            0.07462044060230255,
            0.05378449708223343,
            0.05575074627995491,
            0.07357081025838852,
            0.10496053099632263,
            0.5132461786270142
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.2461028099060059,
          "mean_weights": [
            0.08556011319160461,
            0.0446893684566021,
            0.04084867238998413,
            0.05386578291654587,
            0.025119207799434662,
            0.11240880191326141,
            0.6375080347061157
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.6323832273483276,
          "mean_weights": [
            0.1418324112892151,
            0.08991365134716034,
            0.06188817322254181,
            0.05739111453294754,
            0.09848876297473907,
            0.0972195416688919,
            0.45326635241508484
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.301480770111084,
          "mean_weights": [
            0.09513165056705475,
            0.05029121786355972,
            0.045042067766189575,
            0.05902485549449921,
            0.02615261822938919,
            0.1109299436211586,
            0.6134276390075684
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.629331111907959,
          "mean_weights": [
            0.14790503680706024,
            0.08658000826835632,
            0.06608416140079498,
            0.06427294760942459,
            0.0819322019815445,
            0.097825787961483,
            0.45539984107017517
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.338958501815796,
          "mean_weights": [
            0.10803166776895523,
            0.0559723861515522,
            0.047874726355075836,
            0.060293328016996384,
            0.029215168207883835,
            0.10392352938652039,
            0.5946891903877258
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.7611865997314453,
          "mean_weights": [
            0.1573912799358368,
            0.10317619144916534,
            0.0853322446346283,
            0.08218386769294739,
            0.09052872657775879,
            0.11341015249490738,
            0.3679775595664978
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.476489782333374,
          "mean_weights": [
            0.09070298075675964,
            0.037138283252716064,
            0.030013620853424072,
            0.03533655032515526,
            0.020730752497911453,
            0.06278065592050552,
            0.3836256265640259,
            0.3396714925765991
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.7213938236236572,
          "mean_weights": [
            0.10584281384944916,
            0.06887699663639069,
            0.05207539349794388,
            0.050678279250860214,
            0.04768219590187073,
            0.06809806823730469,
            0.2275032103061676,
            0.37924304604530334
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.5405172109603882,
          "mean_weights": [
            0.07428755611181259,
            0.04491765797138214,
            0.03515764698386192,
            0.046252522617578506,
            0.023427214473485947,
            0.07749318331480026,
            0.29666566848754883,
            0.4017985761165619
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.8063910007476807,
          "mean_weights": [
            0.11433833837509155,
            0.07374332845211029,
            0.05932536721229553,
            0.05389238893985748,
            0.061260174959897995,
            0.07349519431591034,
            0.22022446990013123,
            0.3437207341194153
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.4859801530838013,
          "mean_weights": [
            0.08387120068073273,
            0.041406162083148956,
            0.032444946467876434,
            0.03686721250414848,
            0.025052767246961594,
            0.060124777257442474,
            0.27774184942245483,
            0.4424910843372345
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.7777256965637207,
          "mean_weights": [
            0.10586117208003998,
            0.07087038457393646,
            0.05850498005747795,
            0.06360965967178345,
            0.04672512412071228,
            0.07485505938529968,
            0.2385886013507843,
            0.3409850001335144
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.6318418979644775,
          "mean_weights": [
            0.09244748204946518,
            0.05762210488319397,
            0.045099519193172455,
            0.056236449629068375,
            0.029373466968536377,
            0.0723743885755539,
            0.31761035323143005,
            0.3292362093925476
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.8334553241729736,
          "mean_weights": [
            0.1157165914773941,
            0.07161639630794525,
            0.06577154994010925,
            0.07425849884748459,
            0.04881399869918823,
            0.08197253942489624,
            0.27155569195747375,
            0.270294725894928
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.6227548122406006,
          "mean_weights": [
            0.0605507455766201,
            0.040063511580228806,
            0.032619886100292206,
            0.03868011012673378,
            0.02029048651456833,
            0.04760073497891426,
            0.19551873207092285,
            0.11294309794902802,
            0.45173269510269165
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.9787883758544922,
          "mean_weights": [
            0.0788881778717041,
            0.08108963817358017,
            0.057054273784160614,
            0.08211588114500046,
            0.026040803641080856,
            0.10537256300449371,
            0.24502354860305786,
            0.12593510746955872,
            0.1984800100326538
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.7590525150299072,
          "mean_weights": [
            0.07131639868021011,
            0.044687941670417786,
            0.033290669322013855,
            0.043597716838121414,
            0.02658817172050476,
            0.06922741234302521,
            0.23554196953773499,
            0.14986532926559448,
            0.3258844017982483
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.9736928939819336,
          "mean_weights": [
            0.09849467873573303,
            0.068942591547966,
            0.0599900484085083,
            0.06335455179214478,
            0.04671281576156616,
            0.0713595598936081,
            0.22560220956802368,
            0.1557777225971222,
            0.20976582169532776
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.809161901473999,
          "mean_weights": [
            0.08854075521230698,
            0.05356752127408981,
            0.04243364930152893,
            0.04702243581414223,
            0.02648795023560524,
            0.05820769816637039,
            0.18489578366279602,
            0.21069815754890442,
            0.2881460189819336
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.8544633388519287,
          "mean_weights": [
            0.07977832853794098,
            0.05858254432678223,
            0.053744811564683914,
            0.07626348733901978,
            0.031090255826711655,
            0.0994817242026329,
            0.3525030016899109,
            0.1274903416633606,
            0.12106549739837646
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.9456019401550293,
          "mean_weights": [
            0.08823960274457932,
            0.06737187504768372,
            0.05197303742170334,
            0.0631948709487915,
            0.02739548310637474,
            0.08291902393102646,
            0.2524128556251526,
            0.186361163854599,
            0.18013209104537964
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.06279198825359344,
        0.0596221424639225,
        0.07690861076116562,
        0.07634982466697693,
        0.18058517575263977,
        0.06968431174755096,
        0.15882407128810883,
        0.11288930475711823,
        0.19892723858356476,
        0.11885104328393936,
        0.15647411346435547,
        0.1495160460472107,
        0.1533493995666504,
        0.11554427444934845,
        0.1527898609638214,
        0.13194940984249115,
        0.16355258226394653,
        0.08884616196155548,
        0.11789151281118393,
        0.10897023230791092,
        0.11456070095300674,
        0.09843935817480087,
        0.12148918211460114,
        0.09431816637516022,
        0.15094918012619019,
        0.09025302529335022,
        0.09854138642549515,
        0.0895693451166153,
        0.11213528364896774,
        0.10807885229587555,
        0.16254977881908417,
        0.2204425185918808,
        0.2323070764541626,
        0.15380074083805084,
        0.1390199512243271,
        0.08548793941736221,
        0.14812250435352325,
        0.09071414917707443,
        0.15333011746406555,
        0.10860678553581238,
        0.1781008094549179,
        0.04808543995022774,
        0.07593096792697906,
        0.05631518363952637,
        0.09691466391086578,
        0.07238366454839706,
        0.09368696808815002,
        0.08572373539209366,
        0.10724716633558273,
        0.06546958535909653,
        0.06538715958595276,
        0.06323372572660446,
        0.07412076741456985,
        0.06869969516992569,
        0.07655765861272812,
        0.07729499042034149,
        0.08420819044113159,
        0.04753003641963005,
        0.07230556756258011,
        0.06484207510948181,
        0.07723049819469452,
        0.07835857570171356,
        0.08422473073005676,
        0.08203105628490448
      ],
      "loss_nats": 1.3013269901275635,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 1.596132755279541,
        "mean_weights": [
          0.09668220579624176,
          0.03708431497216225,
          0.034579962491989136,
          0.026012204587459564,
          0.0258483849465847,
          0.037477150559425354,
          0.0845583826303482,
          0.3670029640197754,
          0.29075440764427185
        ],
        "sources": 9
      },
      "step": 8000,
      "stream_state_rms_by_sublayer": [
        0.09253507107496262,
        0.15345558524131775,
        0.30484533309936523,
        0.3249167501926422,
        0.33420246839523315,
        0.3818626403808594,
        0.38682517409324646,
        0.40600064396858215,
        0.12552575767040253,
        0.1351035088300705,
        0.14749252796173096,
        0.15324510633945465,
        0.1726136952638626,
        0.18001793324947357,
        0.1892337054014206,
        0.20391276478767395,
        0.031880609691143036,
        0.04374876990914345,
        0.06123264133930206,
        0.08211024105548859,
        0.10403046011924744,
        0.1380448192358017,
        0.15778414905071259,
        0.2295975536108017,
        0.053015924990177155,
        0.08199761807918549,
        0.09510950744152069,
        0.13892096281051636,
        0.1597633957862854,
        0.21687114238739014,
        0.7120755910873413,
        0.7360114455223083,
        0.04075472429394722,
        0.057135771960020065,
        0.07218439877033234,
        0.08251863718032837,
        0.11449843645095825,
        0.12661415338516235,
        0.16539782285690308,
        0.19619159400463104,
        0.0493597537279129,
        0.06753352284431458,
        0.07179203629493713,
        0.095451220870018,
        0.10295005142688751,
        0.1274663656949997,
        0.1323672980070114,
        0.15628865361213684,
        0.023035654798150063,
        0.058436937630176544,
        0.06294231116771698,
        0.08794140070676804,
        0.09175154566764832,
        0.12247690558433533,
        0.1252463310956955,
        0.1640285700559616,
        0.03424545377492905,
        0.07537907361984253,
        0.07851024717092514,
        0.12290703505277634,
        0.12469635903835297,
        0.19157102704048157,
        0.19299884140491486,
        0.29249706864356995
      ]
    }
  ],
  "environment": {
    "autocast": "cuda-bfloat16-forward-fp32-cross-entropy",
    "compile": false,
    "compute_capability": [
      12,
      0
    ],
    "cublas_workspace_config": ":4096:8",
    "cuda": "12.8",
    "deterministic_algorithms": true,
    "gpu": "NVIDIA GeForce RTX 5090",
    "python": "3.10.14",
    "torch": "2.11.0+cu128"
  },
  "evaluations": [
    {
      "bits_per_byte": 8.034013546791773,
      "cross_entropy_nats": 5.5687538385391235,
      "step": 0
    },
    {
      "bits_per_byte": 5.132431212443787,
      "cross_entropy_nats": 3.5575302243232727,
      "step": 100
    },
    {
      "bits_per_byte": 3.0911456537212687,
      "cross_entropy_nats": 2.1426188945770264,
      "step": 500
    },
    {
      "bits_per_byte": 1.947918907793682,
      "cross_entropy_nats": 1.3501944988965988,
      "step": 2000
    },
    {
      "bits_per_byte": 1.7911587423283797,
      "cross_entropy_nats": 1.241536632180214,
      "step": 4000
    },
    {
      "bits_per_byte": 1.7093240655856785,
      "cross_entropy_nats": 1.1848131567239761,
      "step": 8000
    }
  ],
  "gradient_gate": null,
  "hashes": {
    "final_mixer_parameters": "91dda38f7c99ebfb81b2efc51836d81bcd8f26bf6deadbc08f763cb6225c959d",
    "final_model_state": "bd2556388aeaa211b798c283c7cbd8ccd29edf166a2922fa13d172e8dfdc38d1",
    "final_optimizer_state": "0b101eab3bc7d8d654be2ea335c86fc25563ce19912d721844ee4e639c569e77",
    "final_public_parameters": "f4d3594a7bf1e78aff69c12541a837933d83661dec81d23a744ee66756f670a9",
    "initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d",
    "initial_public_parameter_elements": 18985152,
    "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670",
    "initial_public_parameter_tensors": 227,
    "initial_public_parameters": "bbf2cfc5c0019fb54bf6927274ab9b185b564a77b4b8ef2dc6f5d81575f835c2"
  },
  "manifest": {
    "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716",
    "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371",
    "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
    "input_gate_tensor_hashes": {
      "0": "add6249175ac3bb3fa911069fbe8f25d17d9ab05df34b66ee89996983522ce83",
      "1": "619b5ee4f70e975312ead9adf5df4f9d8584ce2839369c749442e429923405cf",
      "7999": "fe22994f7750c2f8323fdb5ac51c9be1e208733abaf6a337f5b1ae1df1cd2730"
    },
    "path": "experiments/k3/attnres_gradient/manifest.json",
    "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338"
  },
  "model": {
    "attnres_aggregation_groups": 8,
    "context": 256,
    "d_ff": 768,
    "d_head": 32,
    "d_model": 192,
    "heads": 6,
    "layers": 32,
    "parameters": {
      "core": 18985152,
      "embedding": 98304,
      "mixer": 24960,
      "total": 19010112
    },
    "sublayers": 64,
    "sublayers_per_attnres_group": 8,
    "transformer_blocks_per_attnres_group": 4,
    "vocabulary": 256
  },
  "optimizer": {
    "betas": [
      0.9,
      0.95
    ],
    "epsilon": 1e-08,
    "grad_clip": 1.0,
    "min_lr": 3e-05,
    "name": "AdamW",
    "peak_lr": 0.0003,
    "warmup_steps": 400,
    "weight_decay_ndim_ge_2": 0.1
  },
  "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
  "run_kind": "formal",
  "schema_version": 1,
  "seed": 2026073002,
  "steps": 8000,
  "target_bytes_seen": 65536000,
  "timing": {
    "mean_ms": 109.03141246394478,
    "measured_steps": 7980,
    "median_ms": 108.93342149211094,
    "p95_ms": 109.53918776649516,
    "peak_allocated_bytes": 12815843328,
    "peak_reserved_bytes": 13337886720,
    "warmup_steps_excluded": 20
  },
  "training_history": [
    {
      "bits_per_byte": 8.018870646365954,
      "learning_rate": 7.499999999999999e-07,
      "loss_nats": 5.558257579803467,
      "step": 1,
      "unclipped_grad_norm": 26.834381103515625
    },
    {
      "bits_per_byte": 7.214299345307752,
      "learning_rate": 7.499999999999999e-06,
      "loss_nats": 5.000571250915527,
      "step": 10,
      "unclipped_grad_norm": 12.740906715393066
    },
    {
      "bits_per_byte": 6.684460697974439,
      "learning_rate": 1.4999999999999999e-05,
      "loss_nats": 4.633315086364746,
      "step": 20,
      "unclipped_grad_norm": 3.454355239868164
    },
    {
      "bits_per_byte": 6.541223915307444,
      "learning_rate": 2.2499999999999998e-05,
      "loss_nats": 4.534030914306641,
      "step": 30,
      "unclipped_grad_norm": 3.060823440551758
    },
    {
      "bits_per_byte": 6.411011034824987,
      "learning_rate": 2.9999999999999997e-05,
      "loss_nats": 4.443774223327637,
      "step": 40,
      "unclipped_grad_norm": 3.0002496242523193
    },
    {
      "bits_per_byte": 6.085588902515397,
      "learning_rate": 3.75e-05,
      "loss_nats": 4.2182087898254395,
      "step": 50,
      "unclipped_grad_norm": 2.976402997970581
    },
    {
      "bits_per_byte": 5.879418164452873,
      "learning_rate": 4.4999999999999996e-05,
      "loss_nats": 4.0753021240234375,
      "step": 60,
      "unclipped_grad_norm": 2.708517551422119
    },
    {
      "bits_per_byte": 5.710178980836628,
      "learning_rate": 5.2499999999999995e-05,
      "loss_nats": 3.9579944610595703,
      "step": 70,
      "unclipped_grad_norm": 2.559852361679077
    },
    {
      "bits_per_byte": 5.512089019607543,
      "learning_rate": 5.9999999999999995e-05,
      "loss_nats": 3.8206889629364014,
      "step": 80,
      "unclipped_grad_norm": 2.5563650131225586
    },
    {
      "bits_per_byte": 5.341444737733749,
      "learning_rate": 6.749999999999999e-05,
      "loss_nats": 3.7024073600769043,
      "step": 90,
      "unclipped_grad_norm": 2.3861565589904785
    },
    {
      "bits_per_byte": 5.169528816139475,
      "learning_rate": 7.5e-05,
      "loss_nats": 3.5832443237304688,
      "step": 100,
      "unclipped_grad_norm": 2.241894006729126
    },
    {
      "bits_per_byte": 5.000991665704052,
      "learning_rate": 8.249999999999999e-05,
      "loss_nats": 3.466423273086548,
      "step": 110,
      "unclipped_grad_norm": 2.1930899620056152
    },
    {
      "bits_per_byte": 4.733823840860015,
      "learning_rate": 8.999999999999999e-05,
      "loss_nats": 3.2812366485595703,
      "step": 120,
      "unclipped_grad_norm": 1.8853988647460938
    },
    {
      "bits_per_byte": 4.585053344418857,
      "learning_rate": 9.75e-05,
      "loss_nats": 3.178116798400879,
      "step": 130,
      "unclipped_grad_norm": 1.6627076864242554
    },
    {
      "bits_per_byte": 4.391709416422983,
      "learning_rate": 0.00010499999999999999,
      "loss_nats": 3.0441009998321533,
      "step": 140,
      "unclipped_grad_norm": 1.4586857557296753
    },
    {
      "bits_per_byte": 4.260228335873064,
      "learning_rate": 0.0001125,
      "loss_nats": 2.952965259552002,
      "step": 150,
      "unclipped_grad_norm": 1.235063076019287
    },
    {
      "bits_per_byte": 4.153197684831009,
      "learning_rate": 0.00011999999999999999,
      "loss_nats": 2.878777265548706,
      "step": 160,
      "unclipped_grad_norm": 1.0407519340515137
    },
    {
      "bits_per_byte": 3.997309514401244,
      "learning_rate": 0.00012749999999999998,
      "loss_nats": 2.770723819732666,
      "step": 170,
      "unclipped_grad_norm": 0.9049285054206848
    },
    {
      "bits_per_byte": 3.860879805055398,
      "learning_rate": 0.00013499999999999997,
      "loss_nats": 2.6761579513549805,
      "step": 180,
      "unclipped_grad_norm": 1.0858614444732666
    },
    {
      "bits_per_byte": 3.870312021559318,
      "learning_rate": 0.0001425,
      "loss_nats": 2.6826958656311035,
      "step": 190,
      "unclipped_grad_norm": 0.6896197199821472
    },
    {
      "bits_per_byte": 3.747811875037455,
      "learning_rate": 0.00015,
      "loss_nats": 2.597785234451294,
      "step": 200,
      "unclipped_grad_norm": 0.712465226650238
    },
    {
      "bits_per_byte": 3.69472358243852,
      "learning_rate": 0.0001575,
      "loss_nats": 2.5609872341156006,
      "step": 210,
      "unclipped_grad_norm": 0.8807040452957153
    },
    {
      "bits_per_byte": 3.662194783854328,
      "learning_rate": 0.00016499999999999997,
      "loss_nats": 2.538439989089966,
      "step": 220,
      "unclipped_grad_norm": 0.7631296515464783
    },
    {
      "bits_per_byte": 3.6752317567203328,
      "learning_rate": 0.0001725,
      "loss_nats": 2.5474765300750732,
      "step": 230,
      "unclipped_grad_norm": 0.6869579553604126
    },
    {
      "bits_per_byte": 3.6565805821995627,
      "learning_rate": 0.00017999999999999998,
      "loss_nats": 2.53454852104187,
      "step": 240,
      "unclipped_grad_norm": 0.781292736530304
    },
    {
      "bits_per_byte": 3.5245367744160205,
      "learning_rate": 0.0001875,
      "loss_nats": 2.4430227279663086,
      "step": 250,
      "unclipped_grad_norm": 1.0127525329589844
    },
    {
      "bits_per_byte": 3.524651658826797,
      "learning_rate": 0.000195,
      "loss_nats": 2.4431023597717285,
      "step": 260,
      "unclipped_grad_norm": 0.8615073561668396
    },
    {
      "bits_per_byte": 3.485441678221887,
      "learning_rate": 0.00020249999999999996,
      "loss_nats": 2.415924072265625,
      "step": 270,
      "unclipped_grad_norm": 0.7731815576553345
    },
    {
      "bits_per_byte": 3.4956767097400343,
      "learning_rate": 0.00020999999999999998,
      "loss_nats": 2.423018455505371,
      "step": 280,
      "unclipped_grad_norm": 0.9061474800109863
    },
    {
      "bits_per_byte": 3.3938416545805445,
      "learning_rate": 0.00021749999999999997,
      "loss_nats": 2.3524317741394043,
      "step": 290,
      "unclipped_grad_norm": 1.0149996280670166
    },
    {
      "bits_per_byte": 3.483576010425327,
      "learning_rate": 0.000225,
      "loss_nats": 2.414630889892578,
      "step": 300,
      "unclipped_grad_norm": 1.1414116621017456
    },
    {
      "bits_per_byte": 3.381058528107214,
      "learning_rate": 0.00023249999999999996,
      "loss_nats": 2.343571186065674,
      "step": 310,
      "unclipped_grad_norm": 1.1472890377044678
    },
    {
      "bits_per_byte": 3.3707543595634197,
      "learning_rate": 0.00023999999999999998,
      "loss_nats": 2.3364288806915283,
      "step": 320,
      "unclipped_grad_norm": 1.2828246355056763
    },
    {
      "bits_per_byte": 3.3462052120139605,
      "learning_rate": 0.0002475,
      "loss_nats": 2.3194127082824707,
      "step": 330,
      "unclipped_grad_norm": 0.9300209879875183
    },
    {
      "bits_per_byte": 3.3510090314178607,
      "learning_rate": 0.00025499999999999996,
      "loss_nats": 2.322742462158203,
      "step": 340,
      "unclipped_grad_norm": 0.7300398945808411
    },
    {
      "bits_per_byte": 3.324897249502662,
      "learning_rate": 0.0002625,
      "loss_nats": 2.304643154144287,
      "step": 350,
      "unclipped_grad_norm": 0.96440589427948
    },
    {
      "bits_per_byte": 3.301325651340707,
      "learning_rate": 0.00026999999999999995,
      "loss_nats": 2.288304567337036,
      "step": 360,
      "unclipped_grad_norm": 1.186806321144104
    },
    {
      "bits_per_byte": 3.330423052074884,
      "learning_rate": 0.00027749999999999997,
      "loss_nats": 2.3084733486175537,
      "step": 370,
      "unclipped_grad_norm": 1.40411376953125
    },
    {
      "bits_per_byte": 3.351273196769586,
      "learning_rate": 0.000285,
      "loss_nats": 2.322925567626953,
      "step": 380,
      "unclipped_grad_norm": 0.8571750521659851
    },
    {
      "bits_per_byte": 3.2969108566930587,
      "learning_rate": 0.0002925,
      "loss_nats": 2.2852444648742676,
      "step": 390,
      "unclipped_grad_norm": 1.172652006149292
    },
    {
      "bits_per_byte": 3.2316042286462263,
      "learning_rate": 0.0003,
      "loss_nats": 2.2399773597717285,
      "step": 400,
      "unclipped_grad_norm": 1.0324530601501465
    },
    {
      "bits_per_byte": 3.191360632309631,
      "learning_rate": 0.0002999988466112388,
      "loss_nats": 2.212082624435425,
      "step": 410,
      "unclipped_grad_norm": 0.9553669691085815
    },
    {
      "bits_per_byte": 3.2040835648551598,
      "learning_rate": 0.00029999538646466355,
      "loss_nats": 2.2209014892578125,
      "step": 420,
      "unclipped_grad_norm": 1.0212959051132202
    },
    {
      "bits_per_byte": 3.1886670400318176,
      "learning_rate": 0.0002999896196193985,
      "loss_nats": 2.2102155685424805,
      "step": 430,
      "unclipped_grad_norm": 0.8366522789001465
    },
    {
      "bits_per_byte": 3.2100664973133752,
      "learning_rate": 0.00029998154617398326,
      "loss_nats": 2.225048542022705,
      "step": 440,
      "unclipped_grad_norm": 1.1677532196044922
    },
    {
      "bits_per_byte": 3.1886219805772913,
      "learning_rate": 0.0002999711662663707,
      "loss_nats": 2.210184335708618,
      "step": 450,
      "unclipped_grad_norm": 1.3463479280471802
    },
    {
      "bits_per_byte": 3.136037597145384,
      "learning_rate": 0.0002999584800739247,
      "loss_nats": 2.1737356185913086,
      "step": 460,
      "unclipped_grad_norm": 1.704139232635498
    },
    {
      "bits_per_byte": 3.1589835224235,
      "learning_rate": 0.0002999434878134173,
      "loss_nats": 2.189640522003174,
      "step": 470,
      "unclipped_grad_norm": 1.1849488019943237
    },
    {
      "bits_per_byte": 3.1210654754918714,
      "learning_rate": 0.000299926189741025,
      "loss_nats": 2.163357734680176,
      "step": 480,
      "unclipped_grad_norm": 0.8924318552017212
    },
    {
      "bits_per_byte": 3.0823876092087312,
      "learning_rate": 0.000299906586152324,
      "loss_nats": 2.1365482807159424,
      "step": 490,
      "unclipped_grad_norm": 1.3279240131378174
    },
    {
      "bits_per_byte": 3.12389562199447,
      "learning_rate": 0.0002998846773822856,
      "loss_nats": 2.1653194427490234,
      "step": 500,
      "unclipped_grad_norm": 1.3776742219924927
    },
    {
      "bits_per_byte": 3.102860080002138,
      "learning_rate": 0.0002998604638052702,
      "loss_nats": 2.1507387161254883,
      "step": 510,
      "unclipped_grad_norm": 1.061196208000183
    },
    {
      "bits_per_byte": 2.961672642602824,
      "learning_rate": 0.0002998339458350211,
      "loss_nats": 2.05287504196167,
      "step": 520,
      "unclipped_grad_norm": 1.0864008665084839
    },
    {
      "bits_per_byte": 3.0521231342057993,
      "learning_rate": 0.0002998051239246572,
      "loss_nats": 2.115570545196533,
      "step": 530,
      "unclipped_grad_norm": 0.9000556468963623
    },
    {
      "bits_per_byte": 2.992112883874938,
      "learning_rate": 0.00029977399856666537,
      "loss_nats": 2.073974609375,
      "step": 540,
      "unclipped_grad_norm": 0.8655318021774292
    },
    {
      "bits_per_byte": 2.9456885750316344,
      "learning_rate": 0.0002997405702928922,
      "loss_nats": 2.0417957305908203,
      "step": 550,
      "unclipped_grad_norm": 0.8011268973350525
    },
    {
      "bits_per_byte": 2.9526635034200264,
      "learning_rate": 0.0002997048396745345,
      "loss_nats": 2.046630382537842,
      "step": 560,
      "unclipped_grad_norm": 1.2049078941345215
    },
    {
      "bits_per_byte": 2.943776471919342,
      "learning_rate": 0.0002996668073221302,
      "loss_nats": 2.0404703617095947,
      "step": 570,
      "unclipped_grad_norm": 0.9911091923713684
    },
    {
      "bits_per_byte": 2.836173462615215,
      "learning_rate": 0.000299626473885547,
      "loss_nats": 1.9658856391906738,
      "step": 580,
      "unclipped_grad_norm": 0.9621384143829346
    },
    {
      "bits_per_byte": 2.898327304706431,
      "learning_rate": 0.00029958384005397226,
      "loss_nats": 2.008967399597168,
      "step": 590,
      "unclipped_grad_norm": 1.0092018842697144
    },
    {
      "bits_per_byte": 2.807946982025008,
      "learning_rate": 0.0002995389065559004,
      "loss_nats": 1.9463205337524414,
      "step": 600,
      "unclipped_grad_norm": 1.041255235671997
    },
    {
      "bits_per_byte": 2.8662821213364653,
      "learning_rate": 0.000299491674159121,
      "loss_nats": 1.98675537109375,
      "step": 610,
      "unclipped_grad_norm": 1.0585267543792725
    },
    {
      "bits_per_byte": 2.7525066546928727,
      "learning_rate": 0.00029944214367070535,
      "loss_nats": 1.9078922271728516,
      "step": 620,
      "unclipped_grad_norm": 0.9587937593460083
    },
    {
      "bits_per_byte": 2.7676626258001575,
      "learning_rate": 0.00029939031593699284,
      "loss_nats": 1.9183975458145142,
      "step": 630,
      "unclipped_grad_norm": 0.9836012721061707
    },
    {
      "bits_per_byte": 2.7750938241612886,
      "learning_rate": 0.00029933619184357654,
      "loss_nats": 1.9235484600067139,
      "step": 640,
      "unclipped_grad_norm": 0.9689151644706726
    },
    {
      "bits_per_byte": 2.6921524390602856,
      "learning_rate": 0.00029927977231528774,
      "loss_nats": 1.8660578727722168,
      "step": 650,
      "unclipped_grad_norm": 1.0390667915344238
    },
    {
      "bits_per_byte": 2.6851900654054064,
      "learning_rate": 0.00029922105831618057,
      "loss_nats": 1.8612319231033325,
      "step": 660,
      "unclipped_grad_norm": 1.6513670682907104
    },
    {
      "bits_per_byte": 2.6846809967588525,
      "learning_rate": 0.0002991600508495153,
      "loss_nats": 1.8608790636062622,
      "step": 670,
      "unclipped_grad_norm": 0.9175130724906921
    },
    {
      "bits_per_byte": 2.7078205745018398,
      "learning_rate": 0.00029909675095774114,
      "loss_nats": 1.8769181966781616,
      "step": 680,
      "unclipped_grad_norm": 1.0189234018325806
    },
    {
      "bits_per_byte": 2.5875252455640383,
      "learning_rate": 0.0002990311597224787,
      "loss_nats": 1.793535828590393,
      "step": 690,
      "unclipped_grad_norm": 0.9444974064826965
    },
    {
      "bits_per_byte": 2.5413828162854992,
      "learning_rate": 0.00029896327826450103,
      "loss_nats": 1.761552333831787,
      "step": 700,
      "unclipped_grad_norm": 1.1904406547546387
    },
    {
      "bits_per_byte": 2.623998466245768,
      "learning_rate": 0.0002988931077437152,
      "loss_nats": 1.818817138671875,
      "step": 710,
      "unclipped_grad_norm": 1.1858620643615723
    },
    {
      "bits_per_byte": 2.683836561943116,
      "learning_rate": 0.00029882064935914164,
      "loss_nats": 1.8602937459945679,
      "step": 720,
      "unclipped_grad_norm": 0.9852609634399414
    },
    {
      "bits_per_byte": 2.4849188482154285,
      "learning_rate": 0.0002987459043488943,
      "loss_nats": 1.722414493560791,
      "step": 730,
      "unclipped_grad_norm": 1.1975579261779785
    },
    {
      "bits_per_byte": 2.5244201674309004,
      "learning_rate": 0.0002986688739901591,
      "loss_nats": 1.7497947216033936,
      "step": 740,
      "unclipped_grad_norm": 0.9646193385124207
    },
    {
      "bits_per_byte": 2.5296962511940695,
      "learning_rate": 0.0002985895595991723,
      "loss_nats": 1.7534518241882324,
      "step": 750,
      "unclipped_grad_norm": 1.191988468170166
    },
    {
      "bits_per_byte": 2.4988650933565446,
      "learning_rate": 0.00029850796253119824,
      "loss_nats": 1.7320812940597534,
      "step": 760,
      "unclipped_grad_norm": 1.092863917350769
    },
    {
      "bits_per_byte": 2.4838112799438723,
      "learning_rate": 0.00029842408418050544,
      "loss_nats": 1.721646785736084,
      "step": 770,
      "unclipped_grad_norm": 1.0348618030548096
    },
    {
      "bits_per_byte": 2.4223032327066463,
      "learning_rate": 0.0002983379259803436,
      "loss_nats": 1.679012656211853,
      "step": 780,
      "unclipped_grad_norm": 1.034103274345398
    },
    {
      "bits_per_byte": 2.485370818621896,
      "learning_rate": 0.0002982494894029186,
      "loss_nats": 1.7227277755737305,
      "step": 790,
      "unclipped_grad_norm": 1.060265302658081
    },
    {
      "bits_per_byte": 2.4233009040643014,
      "learning_rate": 0.0002981587759593675,
      "loss_nats": 1.679704189300537,
      "step": 800,
      "unclipped_grad_norm": 1.3169445991516113
    },
    {
      "bits_per_byte": 2.4599308009409677,
      "learning_rate": 0.00029806578719973283,
      "loss_nats": 1.7050940990447998,
      "step": 810,
      "unclipped_grad_norm": 0.9821087121963501
    },
    {
      "bits_per_byte": 2.425854846429614,
      "learning_rate": 0.00029797052471293596,
      "loss_nats": 1.6814744472503662,
      "step": 820,
      "unclipped_grad_norm": 1.0924170017242432
    },
    {
      "bits_per_byte": 2.4277851797028984,
      "learning_rate": 0.0002978729901267499,
      "loss_nats": 1.6828124523162842,
      "step": 830,
      "unclipped_grad_norm": 1.055789828300476
    },
    {
      "bits_per_byte": 2.3982261775338056,
      "learning_rate": 0.0002977731851077716,
      "loss_nats": 1.6623237133026123,
      "step": 840,
      "unclipped_grad_norm": 1.230089545249939
    },
    {
      "bits_per_byte": 2.339956047664374,
      "learning_rate": 0.0002976711113613935,
      "loss_nats": 1.621933937072754,
      "step": 850,
      "unclipped_grad_norm": 0.94926518201828
    },
    {
      "bits_per_byte": 2.3803994878412724,
      "learning_rate": 0.0002975667706317743,
      "loss_nats": 1.6499671936035156,
      "step": 860,
      "unclipped_grad_norm": 1.1705771684646606
    },
    {
      "bits_per_byte": 2.3436123988217177,
      "learning_rate": 0.00029746016470180915,
      "loss_nats": 1.6244683265686035,
      "step": 870,
      "unclipped_grad_norm": 1.1496235132217407
    },
    {
      "bits_per_byte": 2.3796778486382517,
      "learning_rate": 0.00029735129539309926,
      "loss_nats": 1.6494669914245605,
      "step": 880,
      "unclipped_grad_norm": 1.1764546632766724
    },
    {
      "bits_per_byte": 2.357840007579645,
      "learning_rate": 0.0002972401645659207,
      "loss_nats": 1.634330153465271,
      "step": 890,
      "unclipped_grad_norm": 0.9926145672798157
    },
    {
      "bits_per_byte": 2.2929997964819524,
      "learning_rate": 0.00029712677411919283,
      "loss_nats": 1.5893863439559937,
      "step": 900,
      "unclipped_grad_norm": 1.009945034980774
    },
    {
      "bits_per_byte": 2.3411444477818364,
      "learning_rate": 0.0002970111259904454,
      "loss_nats": 1.6227576732635498,
      "step": 910,
      "unclipped_grad_norm": 1.2830723524093628
    },
    {
      "bits_per_byte": 2.2736926801307917,
      "learning_rate": 0.0002968932221557859,
      "loss_nats": 1.5760036706924438,
      "step": 920,
      "unclipped_grad_norm": 1.0900828838348389
    },
    {
      "bits_per_byte": 2.3038844064724087,
      "learning_rate": 0.0002967730646298655,
      "loss_nats": 1.596930980682373,
      "step": 930,
      "unclipped_grad_norm": 1.1090145111083984
    },
    {
      "bits_per_byte": 2.2389674911124313,
      "learning_rate": 0.0002966506554658449,
      "loss_nats": 1.551934003829956,
      "step": 940,
      "unclipped_grad_norm": 0.9897664189338684
    },
    {
      "bits_per_byte": 2.2434025797128814,
      "learning_rate": 0.00029652599675535876,
      "loss_nats": 1.5550081729888916,
      "step": 950,
      "unclipped_grad_norm": 1.0138952732086182
    },
    {
      "bits_per_byte": 2.2236732459538526,
      "learning_rate": 0.0002963990906284808,
      "loss_nats": 1.5413328409194946,
      "step": 960,
      "unclipped_grad_norm": 1.0875126123428345
    },
    {
      "bits_per_byte": 2.2667958318658914,
      "learning_rate": 0.0002962699392536863,
      "loss_nats": 1.5712231397628784,
      "step": 970,
      "unclipped_grad_norm": 1.245374321937561
    },
    {
      "bits_per_byte": 2.2344890628839638,
      "learning_rate": 0.0002961385448378161,
      "loss_nats": 1.5488297939300537,
      "step": 980,
      "unclipped_grad_norm": 1.1757959127426147
    },
    {
      "bits_per_byte": 2.1613678871472266,
      "learning_rate": 0.00029600490962603817,
      "loss_nats": 1.4981460571289062,
      "step": 990,
      "unclipped_grad_norm": 1.066755771636963
    },
    {
      "bits_per_byte": 2.207448574654106,
      "learning_rate": 0.00029586903590180956,
      "loss_nats": 1.5300867557525635,
      "step": 1000,
      "unclipped_grad_norm": 0.9658553004264832
    },
    {
      "bits_per_byte": 2.213320750285166,
      "learning_rate": 0.00029573092598683735,
      "loss_nats": 1.5341570377349854,
      "step": 1010,
      "unclipped_grad_norm": 1.1267611980438232
    },
    {
      "bits_per_byte": 2.2186436133293697,
      "learning_rate": 0.00029559058224103895,
      "loss_nats": 1.537846565246582,
      "step": 1020,
      "unclipped_grad_norm": 0.9853240251541138
    },
    {
      "bits_per_byte": 2.1280228589020402,
      "learning_rate": 0.0002954480070625017,
      "loss_nats": 1.4750330448150635,
      "step": 1030,
      "unclipped_grad_norm": 0.972481369972229
    },
    {
      "bits_per_byte": 2.2322469250646653,
      "learning_rate": 0.00029530320288744205,
      "loss_nats": 1.5472756624221802,
      "step": 1040,
      "unclipped_grad_norm": 1.0531573295593262
    },
    {
      "bits_per_byte": 2.179971250491057,
      "learning_rate": 0.0002951561721901638,
      "loss_nats": 1.5110409259796143,
      "step": 1050,
      "unclipped_grad_norm": 0.9985654950141907
    },
    {
      "bits_per_byte": 2.1476089311129005,
      "learning_rate": 0.00029500691748301594,
      "loss_nats": 1.4886090755462646,
      "step": 1060,
      "unclipped_grad_norm": 1.100019931793213
    },
    {
      "bits_per_byte": 2.2640807417567204,
      "learning_rate": 0.00029485544131634963,
      "loss_nats": 1.5693411827087402,
      "step": 1070,
      "unclipped_grad_norm": 1.0959177017211914
    },
    {
      "bits_per_byte": 2.1797610876917024,
      "learning_rate": 0.0002947017462784745,
      "loss_nats": 1.5108952522277832,
      "step": 1080,
      "unclipped_grad_norm": 1.088409423828125
    },
    {
      "bits_per_byte": 2.1080023585151006,
      "learning_rate": 0.00029454583499561476,
      "loss_nats": 1.461155891418457,
      "step": 1090,
      "unclipped_grad_norm": 1.0188393592834473
    },
    {
      "bits_per_byte": 2.1918825969071642,
      "learning_rate": 0.00029438771013186413,
      "loss_nats": 1.5192972421646118,
      "step": 1100,
      "unclipped_grad_norm": 0.9501848816871643
    },
    {
      "bits_per_byte": 2.205643788715952,
      "learning_rate": 0.0002942273743891401,
      "loss_nats": 1.5288357734680176,
      "step": 1110,
      "unclipped_grad_norm": 0.9797388911247253
    },
    {
      "bits_per_byte": 2.0930500148664373,
      "learning_rate": 0.00029406483050713824,
      "loss_nats": 1.4507917165756226,
      "step": 1120,
      "unclipped_grad_norm": 0.8856902122497559
    },
    {
      "bits_per_byte": 2.095190166973774,
      "learning_rate": 0.00029390008126328504,
      "loss_nats": 1.4522751569747925,
      "step": 1130,
      "unclipped_grad_norm": 0.9447507262229919
    },
    {
      "bits_per_byte": 2.116739593126884,
      "learning_rate": 0.00029373312947269045,
      "loss_nats": 1.4672120809555054,
      "step": 1140,
      "unclipped_grad_norm": 1.0488460063934326
    },
    {
      "bits_per_byte": 2.1023439573190634,
      "learning_rate": 0.00029356397798809995,
      "loss_nats": 1.4572337865829468,
      "step": 1150,
      "unclipped_grad_norm": 0.9838997721672058
    },
    {
      "bits_per_byte": 2.0562952585852448,
      "learning_rate": 0.0002933926296998457,
      "loss_nats": 1.425315260887146,
      "step": 1160,
      "unclipped_grad_norm": 0.9282211065292358
    },
    {
      "bits_per_byte": 2.160992449020393,
      "learning_rate": 0.0002932190875357972,
      "loss_nats": 1.497885823249817,
      "step": 1170,
      "unclipped_grad_norm": 1.0253955125808716
    },
    {
      "bits_per_byte": 2.0716469459492117,
      "learning_rate": 0.00029304335446131126,
      "loss_nats": 1.4359562397003174,
      "step": 1180,
      "unclipped_grad_norm": 0.9352066516876221
    },
    {
      "bits_per_byte": 2.0893013090255805,
      "learning_rate": 0.00029286543347918116,
      "loss_nats": 1.4481933116912842,
      "step": 1190,
      "unclipped_grad_norm": 1.0826658010482788
    },
    {
      "bits_per_byte": 2.0752424152481495,
      "learning_rate": 0.0002926853276295856,
      "loss_nats": 1.438448429107666,
      "step": 1200,
      "unclipped_grad_norm": 0.9942638874053955
    },
    {
      "bits_per_byte": 2.2049758081000004,
      "learning_rate": 0.0002925030399900368,
      "loss_nats": 1.5283727645874023,
      "step": 1210,
      "unclipped_grad_norm": 1.1482313871383667
    },
    {
      "bits_per_byte": 2.0506430487646385,
      "learning_rate": 0.00029231857367532726,
      "loss_nats": 1.4213974475860596,
      "step": 1220,
      "unclipped_grad_norm": 0.9142818450927734
    },
    {
      "bits_per_byte": 2.00269480165204,
      "learning_rate": 0.0002921319318374776,
      "loss_nats": 1.3881622552871704,
      "step": 1230,
      "unclipped_grad_norm": 0.9387761354446411
    },
    {
      "bits_per_byte": 2.0166881700394925,
      "learning_rate": 0.00029194311766568174,
      "loss_nats": 1.3978617191314697,
      "step": 1240,
      "unclipped_grad_norm": 1.0545530319213867
    },
    {
      "bits_per_byte": 2.1787518934964396,
      "learning_rate": 0.000291752134386253,
      "loss_nats": 1.5101957321166992,
      "step": 1250,
      "unclipped_grad_norm": 1.1504697799682617
    },
    {
      "bits_per_byte": 2.1047604855463367,
      "learning_rate": 0.00029155898526256856,
      "loss_nats": 1.4589087963104248,
      "step": 1260,
      "unclipped_grad_norm": 0.9371413588523865
    },
    {
      "bits_per_byte": 1.9885089846782722,
      "learning_rate": 0.0002913636735950141,
      "loss_nats": 1.3783293962478638,
      "step": 1270,
      "unclipped_grad_norm": 0.9821643233299255
    },
    {
      "bits_per_byte": 2.1346452228561636,
      "learning_rate": 0.0002911662027209272,
      "loss_nats": 1.4796233177185059,
      "step": 1280,
      "unclipped_grad_norm": 0.9382128715515137
    },
    {
      "bits_per_byte": 2.022505311222276,
      "learning_rate": 0.00029096657601454013,
      "loss_nats": 1.4018938541412354,
      "step": 1290,
      "unclipped_grad_norm": 0.9164487719535828
    },
    {
      "bits_per_byte": 2.050535903573151,
      "learning_rate": 0.00029076479688692266,
      "loss_nats": 1.4213231801986694,
      "step": 1300,
      "unclipped_grad_norm": 0.9040083885192871
    },
    {
      "bits_per_byte": 2.128253659619498,
      "learning_rate": 0.00029056086878592323,
      "loss_nats": 1.4751930236816406,
      "step": 1310,
      "unclipped_grad_norm": 1.045804738998413
    },
    {
      "bits_per_byte": 2.0627867437420613,
      "learning_rate": 0.00029035479519611045,
      "loss_nats": 1.4298148155212402,
      "step": 1320,
      "unclipped_grad_norm": 1.0330690145492554
    },
    {
      "bits_per_byte": 1.954123977343318,
      "learning_rate": 0.0002901465796387134,
      "loss_nats": 1.3544955253601074,
      "step": 1330,
      "unclipped_grad_norm": 0.9727330803871155
    },
    {
      "bits_per_byte": 1.9791288789175636,
      "learning_rate": 0.00028993622567156154,
      "loss_nats": 1.3718276023864746,
      "step": 1340,
      "unclipped_grad_norm": 0.966886043548584
    },
    {
      "bits_per_byte": 2.1481475807754027,
      "learning_rate": 0.0002897237368890237,
      "loss_nats": 1.4889824390411377,
      "step": 1350,
      "unclipped_grad_norm": 0.9399362802505493
    },
    {
      "bits_per_byte": 1.9898208683390537,
      "learning_rate": 0.0002895091169219469,
      "loss_nats": 1.3792387247085571,
      "step": 1360,
      "unclipped_grad_norm": 0.9223746061325073
    },
    {
      "bits_per_byte": 2.0666878262115826,
      "learning_rate": 0.00028929236943759425,
      "loss_nats": 1.4325188398361206,
      "step": 1370,
      "unclipped_grad_norm": 0.9086402654647827
    },
    {
      "bits_per_byte": 2.05495087020345,
      "learning_rate": 0.00028907349813958225,
      "loss_nats": 1.4243834018707275,
      "step": 1380,
      "unclipped_grad_norm": 0.954355776309967
    },
    {
      "bits_per_byte": 2.067550663170961,
      "learning_rate": 0.0002888525067678175,
      "loss_nats": 1.4331169128417969,
      "step": 1390,
      "unclipped_grad_norm": 0.8977138996124268
    },
    {
      "bits_per_byte": 2.0721085473841274,
      "learning_rate": 0.00028862939909843273,
      "loss_nats": 1.4362761974334717,
      "step": 1400,
      "unclipped_grad_norm": 0.8833977580070496
    },
    {
      "bits_per_byte": 1.9799110560136877,
      "learning_rate": 0.0002884041789437225,
      "loss_nats": 1.3723697662353516,
      "step": 1410,
      "unclipped_grad_norm": 0.9529625177383423
    },
    {
      "bits_per_byte": 2.038817521691312,
      "learning_rate": 0.0002881768501520778,
      "loss_nats": 1.4132006168365479,
      "step": 1420,
      "unclipped_grad_norm": 1.083083152770996
    },
    {
      "bits_per_byte": 2.0292872750764,
      "learning_rate": 0.0002879474166079205,
      "loss_nats": 1.4065947532653809,
      "step": 1430,
      "unclipped_grad_norm": 0.9355389475822449
    },
    {
      "bits_per_byte": 1.987968787171912,
      "learning_rate": 0.00028771588223163694,
      "loss_nats": 1.3779549598693848,
      "step": 1440,
      "unclipped_grad_norm": 0.8836045265197754
    },
    {
      "bits_per_byte": 2.0423230440638482,
      "learning_rate": 0.00028748225097951054,
      "loss_nats": 1.4156304597854614,
      "step": 1450,
      "unclipped_grad_norm": 0.924534022808075
    },
    {
      "bits_per_byte": 1.9816687187055055,
      "learning_rate": 0.000287246526843655,
      "loss_nats": 1.3735880851745605,
      "step": 1460,
      "unclipped_grad_norm": 0.8437807559967041
    },
    {
      "bits_per_byte": 1.9797782854072217,
      "learning_rate": 0.00028700871385194533,
      "loss_nats": 1.3722777366638184,
      "step": 1470,
      "unclipped_grad_norm": 0.9143778085708618
    },
    {
      "bits_per_byte": 1.8767507025466492,
      "learning_rate": 0.00028676881606794945,
      "loss_nats": 1.3008644580841064,
      "step": 1480,
      "unclipped_grad_norm": 0.8930306434631348
    },
    {
      "bits_per_byte": 2.001770738868954,
      "learning_rate": 0.00028652683759085854,
      "loss_nats": 1.387521743774414,
      "step": 1490,
      "unclipped_grad_norm": 0.8246955871582031
    },
    {
      "bits_per_byte": 1.9125255016832254,
      "learning_rate": 0.0002862827825554171,
      "loss_nats": 1.3256616592407227,
      "step": 1500,
      "unclipped_grad_norm": 0.9061107635498047
    },
    {
      "bits_per_byte": 1.973825621895558,
      "learning_rate": 0.00028603665513185245,
      "loss_nats": 1.3681516647338867,
      "step": 1510,
      "unclipped_grad_norm": 0.9889123439788818
    },
    {
      "bits_per_byte": 1.980869687309597,
      "learning_rate": 0.000285788459525803,
      "loss_nats": 1.3730342388153076,
      "step": 1520,
      "unclipped_grad_norm": 0.9045401811599731
    },
    {
      "bits_per_byte": 1.9378121108395843,
      "learning_rate": 0.00028553819997824707,
      "loss_nats": 1.343189001083374,
      "step": 1530,
      "unclipped_grad_norm": 0.8896906971931458
    },
    {
      "bits_per_byte": 2.017761685746178,
      "learning_rate": 0.00028528588076542964,
      "loss_nats": 1.3986058235168457,
      "step": 1540,
      "unclipped_grad_norm": 0.9600386619567871
    },
    {
      "bits_per_byte": 1.9800400430018348,
      "learning_rate": 0.00028503150619878996,
      "loss_nats": 1.3724591732025146,
      "step": 1550,
      "unclipped_grad_norm": 0.8903042078018188
    },
    {
      "bits_per_byte": 2.037820882229562,
      "learning_rate": 0.0002847750806248876,
      "loss_nats": 1.412509799003601,
      "step": 1560,
      "unclipped_grad_norm": 0.9084417819976807
    },
    {
      "bits_per_byte": 2.015308525214271,
      "learning_rate": 0.000284516608425328,
      "loss_nats": 1.3969054222106934,
      "step": 1570,
      "unclipped_grad_norm": 0.9122006893157959
    },
    {
      "bits_per_byte": 1.985575476602505,
      "learning_rate": 0.000284256094016688,
      "loss_nats": 1.376296043395996,
      "step": 1580,
      "unclipped_grad_norm": 0.8903481960296631
    },
    {
      "bits_per_byte": 1.996281568601366,
      "learning_rate": 0.0002839935418504399,
      "loss_nats": 1.3837169408798218,
      "step": 1590,
      "unclipped_grad_norm": 0.933029294013977
    },
    {
      "bits_per_byte": 2.002402775110875,
      "learning_rate": 0.000283728956412876,
      "loss_nats": 1.3879598379135132,
      "step": 1600,
      "unclipped_grad_norm": 0.9777830243110657
    },
    {
      "bits_per_byte": 2.0119596790366705,
      "learning_rate": 0.00028346234222503136,
      "loss_nats": 1.3945841789245605,
      "step": 1610,
      "unclipped_grad_norm": 0.9627588391304016
    },
    {
      "bits_per_byte": 2.0159194075901357,
      "learning_rate": 0.00028319370384260675,
      "loss_nats": 1.3973288536071777,
      "step": 1620,
      "unclipped_grad_norm": 0.8731178641319275
    },
    {
      "bits_per_byte": 1.8905791395719242,
      "learning_rate": 0.00028292304585589105,
      "loss_nats": 1.3104496002197266,
      "step": 1630,
      "unclipped_grad_norm": 0.8998029828071594
    },
    {
      "bits_per_byte": 2.0235616286638756,
      "learning_rate": 0.0002826503728896824,
      "loss_nats": 1.4026260375976562,
      "step": 1640,
      "unclipped_grad_norm": 0.9714035391807556
    },
    {
      "bits_per_byte": 1.9588359580116574,
      "learning_rate": 0.0002823756896032095,
      "loss_nats": 1.3577616214752197,
      "step": 1650,
      "unclipped_grad_norm": 0.9218164682388306
    },
    {
      "bits_per_byte": 1.936817019221692,
      "learning_rate": 0.0002820990006900518,
      "loss_nats": 1.3424992561340332,
      "step": 1660,
      "unclipped_grad_norm": 0.8974215984344482
    },
    {
      "bits_per_byte": 2.096671109580354,
      "learning_rate": 0.00028182031087805947,
      "loss_nats": 1.4533016681671143,
      "step": 1670,
      "unclipped_grad_norm": 0.9378093481063843
    },
    {
      "bits_per_byte": 2.0150522710644854,
      "learning_rate": 0.0002815396249292725,
      "loss_nats": 1.3967278003692627,
      "step": 1680,
      "unclipped_grad_norm": 0.8720370531082153
    },
    {
      "bits_per_byte": 1.8838673446393523,
      "learning_rate": 0.0002812569476398391,
      "loss_nats": 1.3057973384857178,
      "step": 1690,
      "unclipped_grad_norm": 0.8577803373336792
    },
    {
      "bits_per_byte": 1.927341291107104,
      "learning_rate": 0.00028097228383993446,
      "loss_nats": 1.3359311819076538,
      "step": 1700,
      "unclipped_grad_norm": 0.9562329053878784
    },
    {
      "bits_per_byte": 1.9962937793695772,
      "learning_rate": 0.0002806856383936774,
      "loss_nats": 1.3837254047393799,
      "step": 1710,
      "unclipped_grad_norm": 0.9462636113166809
    },
    {
      "bits_per_byte": 1.995081645646296,
      "learning_rate": 0.00028039701619904774,
      "loss_nats": 1.382885217666626,
      "step": 1720,
      "unclipped_grad_norm": 0.8408851027488708
    },
    {
      "bits_per_byte": 1.894888164889292,
      "learning_rate": 0.0002801064221878024,
      "loss_nats": 1.3134363889694214,
      "step": 1730,
      "unclipped_grad_norm": 0.8187386989593506
    },
    {
      "bits_per_byte": 1.9475287866480313,
      "learning_rate": 0.00027981386132539134,
      "loss_nats": 1.349924087524414,
      "step": 1740,
      "unclipped_grad_norm": 0.9181671142578125
    },
    {
      "bits_per_byte": 2.0484275682562236,
      "learning_rate": 0.0002795193386108724,
      "loss_nats": 1.4198617935180664,
      "step": 1750,
      "unclipped_grad_norm": 0.9259082674980164
    },
    {
      "bits_per_byte": 1.9089950418263137,
      "learning_rate": 0.00027922285907682625,
      "loss_nats": 1.3232145309448242,
      "step": 1760,
      "unclipped_grad_norm": 0.9000674486160278
    },
    {
      "bits_per_byte": 1.9653874651136254,
      "learning_rate": 0.00027892442778927005,
      "loss_nats": 1.3623027801513672,
      "step": 1770,
      "unclipped_grad_norm": 0.8686579465866089
    },
    {
      "bits_per_byte": 1.9032847018697159,
      "learning_rate": 0.00027862404984757116,
      "loss_nats": 1.3192564249038696,
      "step": 1780,
      "unclipped_grad_norm": 0.8219738006591797
    },
    {
      "bits_per_byte": 1.8754542973244455,
      "learning_rate": 0.0002783217303843597,
      "loss_nats": 1.2999658584594727,
      "step": 1790,
      "unclipped_grad_norm": 0.8401536345481873
    },
    {
      "bits_per_byte": 1.9695277754504965,
      "learning_rate": 0.00027801747456544134,
      "loss_nats": 1.3651726245880127,
      "step": 1800,
      "unclipped_grad_norm": 0.9855192303657532
    },
    {
      "bits_per_byte": 1.952977025044714,
      "learning_rate": 0.00027771128758970843,
      "loss_nats": 1.3537005186080933,
      "step": 1810,
      "unclipped_grad_norm": 0.9152796268463135
    },
    {
      "bits_per_byte": 2.0236675699768067,
      "learning_rate": 0.00027740317468905165,
      "loss_nats": 1.4026994705200195,
      "step": 1820,
      "unclipped_grad_norm": 0.9160932898521423
    },
    {
      "bits_per_byte": 1.863515605666851,
      "learning_rate": 0.0002770931411282704,
      "loss_nats": 1.2916905879974365,
      "step": 1830,
      "unclipped_grad_norm": 0.8749093413352966
    },
    {
      "bits_per_byte": 1.9532654399502112,
      "learning_rate": 0.0002767811922049827,
      "loss_nats": 1.35390043258667,
      "step": 1840,
      "unclipped_grad_norm": 0.8691527843475342
    },
    {
      "bits_per_byte": 1.8425976998077092,
      "learning_rate": 0.00027646733324953503,
      "loss_nats": 1.277191400527954,
      "step": 1850,
      "unclipped_grad_norm": 0.918012797832489
    },
    {
      "bits_per_byte": 1.9936586611930571,
      "learning_rate": 0.00027615156962491097,
      "loss_nats": 1.3818988800048828,
      "step": 1860,
      "unclipped_grad_norm": 0.9178117513656616
    },
    {
      "bits_per_byte": 1.995530520365048,
      "learning_rate": 0.0002758339067266396,
      "loss_nats": 1.3831963539123535,
      "step": 1870,
      "unclipped_grad_norm": 0.8769021034240723
    },
    {
      "bits_per_byte": 1.8636951555543517,
      "learning_rate": 0.0002755143499827035,
      "loss_nats": 1.2918150424957275,
      "step": 1880,
      "unclipped_grad_norm": 0.8614026308059692
    },
    {
      "bits_per_byte": 1.9426209177403593,
      "learning_rate": 0.0002751929048534456,
      "loss_nats": 1.3465222120285034,
      "step": 1890,
      "unclipped_grad_norm": 0.8975904583930969
    },
    {
      "bits_per_byte": 1.861964838104021,
      "learning_rate": 0.0002748695768314763,
      "loss_nats": 1.2906156778335571,
      "step": 1900,
      "unclipped_grad_norm": 0.8269307613372803
    },
    {
      "bits_per_byte": 1.8502344174366214,
      "learning_rate": 0.00027454437144157947,
      "loss_nats": 1.282484769821167,
      "step": 1910,
      "unclipped_grad_norm": 0.8237605690956116
    },
    {
      "bits_per_byte": 1.9149965484108225,
      "learning_rate": 0.00027421729424061787,
      "loss_nats": 1.3273744583129883,
      "step": 1920,
      "unclipped_grad_norm": 0.8641052842140198
    },
    {
      "bits_per_byte": 1.8801597426520518,
      "learning_rate": 0.0002738883508174385,
      "loss_nats": 1.303227424621582,
      "step": 1930,
      "unclipped_grad_norm": 0.8907292485237122
    },
    {
      "bits_per_byte": 1.8629336163763313,
      "learning_rate": 0.0002735575467927768,
      "loss_nats": 1.2912871837615967,
      "step": 1940,
      "unclipped_grad_norm": 0.8837904930114746
    },
    {
      "bits_per_byte": 1.923144226495448,
      "learning_rate": 0.0002732248878191609,
      "loss_nats": 1.3330219984054565,
      "step": 1950,
      "unclipped_grad_norm": 1.0997384786605835
    },
    {
      "bits_per_byte": 1.88083511852199,
      "learning_rate": 0.0002728903795808148,
      "loss_nats": 1.303695559501648,
      "step": 1960,
      "unclipped_grad_norm": 0.8385953307151794
    },
    {
      "bits_per_byte": 1.8359603733629608,
      "learning_rate": 0.0002725540277935614,
      "loss_nats": 1.2725907564163208,
      "step": 1970,
      "unclipped_grad_norm": 0.839948832988739
    },
    {
      "bits_per_byte": 1.7988822896981462,
      "learning_rate": 0.00027221583820472456,
      "loss_nats": 1.2468901872634888,
      "step": 1980,
      "unclipped_grad_norm": 0.7658324837684631
    },
    {
      "bits_per_byte": 1.9198664091526536,
      "learning_rate": 0.0002718758165930314,
      "loss_nats": 1.3307499885559082,
      "step": 1990,
      "unclipped_grad_norm": 0.8840731382369995
    },
    {
      "bits_per_byte": 1.863330208369221,
      "learning_rate": 0.0002715339687685131,
      "loss_nats": 1.2915620803833008,
      "step": 2000,
      "unclipped_grad_norm": 0.8192090392112732
    },
    {
      "bits_per_byte": 1.9331472534175815,
      "learning_rate": 0.00027119030057240576,
      "loss_nats": 1.3399555683135986,
      "step": 2010,
      "unclipped_grad_norm": 0.8343153595924377
    },
    {
      "bits_per_byte": 1.9117808168049895,
      "learning_rate": 0.0002708448178770506,
      "loss_nats": 1.3251454830169678,
      "step": 2020,
      "unclipped_grad_norm": 0.913158118724823
    },
    {
      "bits_per_byte": 1.8970498148279866,
      "learning_rate": 0.00027049752658579376,
      "loss_nats": 1.3149347305297852,
      "step": 2030,
      "unclipped_grad_norm": 0.8877862691879272
    },
    {
      "bits_per_byte": 1.9115138997308505,
      "learning_rate": 0.00027014843263288514,
      "loss_nats": 1.324960470199585,
      "step": 2040,
      "unclipped_grad_norm": 0.8749900460243225
    },
    {
      "bits_per_byte": 1.8657040848990802,
      "learning_rate": 0.00026979754198337724,
      "loss_nats": 1.2932075262069702,
      "step": 2050,
      "unclipped_grad_norm": 0.9185486435890198
    },
    {
      "bits_per_byte": 1.7675556498434857,
      "learning_rate": 0.00026944486063302305,
      "loss_nats": 1.225176215171814,
      "step": 2060,
      "unclipped_grad_norm": 0.7904725670814514
    },
    {
      "bits_per_byte": 1.8617049723185672,
      "learning_rate": 0.0002690903946081738,
      "loss_nats": 1.290435552597046,
      "step": 2070,
      "unclipped_grad_norm": 0.8348630666732788
    },
    {
      "bits_per_byte": 1.973646415973359,
      "learning_rate": 0.00026873414996567596,
      "loss_nats": 1.3680274486541748,
      "step": 2080,
      "unclipped_grad_norm": 0.8117388486862183
    },
    {
      "bits_per_byte": 1.8279640400110946,
      "learning_rate": 0.00026837613279276736,
      "loss_nats": 1.2670481204986572,
      "step": 2090,
      "unclipped_grad_norm": 0.7758682370185852
    },
    {
      "bits_per_byte": 1.8962239541385433,
      "learning_rate": 0.0002680163492069739,
      "loss_nats": 1.3143622875213623,
      "step": 2100,
      "unclipped_grad_norm": 0.8137099146842957
    },
    {
      "bits_per_byte": 1.8350517890184526,
      "learning_rate": 0.00026765480535600414,
      "loss_nats": 1.271960973739624,
      "step": 2110,
      "unclipped_grad_norm": 0.9251447319984436
    },
    {
      "bits_per_byte": 1.870564486595114,
      "learning_rate": 0.0002672915074176452,
      "loss_nats": 1.2965764999389648,
      "step": 2120,
      "unclipped_grad_norm": 0.9138389825820923
    },
    {
      "bits_per_byte": 1.8882444750864615,
      "learning_rate": 0.00026692646159965624,
      "loss_nats": 1.3088313341140747,
      "step": 2130,
      "unclipped_grad_norm": 0.8262670040130615
    },
    {
      "bits_per_byte": 1.7726164113277747,
      "learning_rate": 0.0002665596741396633,
      "loss_nats": 1.2286840677261353,
      "step": 2140,
      "unclipped_grad_norm": 0.7359873056411743
    },
    {
      "bits_per_byte": 1.8268258588276844,
      "learning_rate": 0.000266191151305052,
      "loss_nats": 1.2662591934204102,
      "step": 2150,
      "unclipped_grad_norm": 0.8173519372940063
    },
    {
      "bits_per_byte": 1.8637190611428216,
      "learning_rate": 0.00026582089939286075,
      "loss_nats": 1.291831612586975,
      "step": 2160,
      "unclipped_grad_norm": 0.8174240589141846
    },
    {
      "bits_per_byte": 1.8528119214251022,
      "learning_rate": 0.0002654489247296732,
      "loss_nats": 1.2842713594436646,
      "step": 2170,
      "unclipped_grad_norm": 0.8957990407943726
    },
    {
      "bits_per_byte": 1.782856774308099,
      "learning_rate": 0.00026507523367151007,
      "loss_nats": 1.2357821464538574,
      "step": 2180,
      "unclipped_grad_norm": 0.8048707246780396
    },
    {
      "bits_per_byte": 1.8923490130319534,
      "learning_rate": 0.00026469983260372037,
      "loss_nats": 1.3116763830184937,
      "step": 2190,
      "unclipped_grad_norm": 0.8952655792236328
    },
    {
      "bits_per_byte": 1.9442205283760343,
      "learning_rate": 0.0002643227279408727,
      "loss_nats": 1.3476309776306152,
      "step": 2200,
      "unclipped_grad_norm": 0.8895230889320374
    },
    {
      "bits_per_byte": 1.9876000563684622,
      "learning_rate": 0.0002639439261266453,
      "loss_nats": 1.377699375152588,
      "step": 2210,
      "unclipped_grad_norm": 0.8686981797218323
    },
    {
      "bits_per_byte": 1.852814845130167,
      "learning_rate": 0.000263563433633716,
      "loss_nats": 1.284273386001587,
      "step": 2220,
      "unclipped_grad_norm": 0.8053512573242188
    },
    {
      "bits_per_byte": 1.859576858996791,
      "learning_rate": 0.00026318125696365156,
      "loss_nats": 1.2889604568481445,
      "step": 2230,
      "unclipped_grad_norm": 0.8349927663803101
    },
    {
      "bits_per_byte": 1.8206912376714082,
      "learning_rate": 0.000262797402646797,
      "loss_nats": 1.2620069980621338,
      "step": 2240,
      "unclipped_grad_norm": 0.8471757769584656
    },
    {
      "bits_per_byte": 1.8270817690121683,
      "learning_rate": 0.00026241187724216334,
      "loss_nats": 1.2664365768432617,
      "step": 2250,
      "unclipped_grad_norm": 0.8009039163589478
    },
    {
      "bits_per_byte": 1.8453999851208678,
      "learning_rate": 0.0002620246873373162,
      "loss_nats": 1.2791337966918945,
      "step": 2260,
      "unclipped_grad_norm": 0.8003636002540588
    },
    {
      "bits_per_byte": 1.9009572606555896,
      "learning_rate": 0.00026163583954826264,
      "loss_nats": 1.317643165588379,
      "step": 2270,
      "unclipped_grad_norm": 0.8468067646026611
    },
    {
      "bits_per_byte": 1.7920475271702074,
      "learning_rate": 0.0002612453405193386,
      "loss_nats": 1.2421526908874512,
      "step": 2280,
      "unclipped_grad_norm": 0.9071624875068665
    },
    {
      "bits_per_byte": 1.8281408381761814,
      "learning_rate": 0.000260853196923095,
      "loss_nats": 1.2671706676483154,
      "step": 2290,
      "unclipped_grad_norm": 0.822325587272644
    },
    {
      "bits_per_byte": 1.8827026781277094,
      "learning_rate": 0.0002604594154601839,
      "loss_nats": 1.3049900531768799,
      "step": 2300,
      "unclipped_grad_norm": 0.8915430307388306
    },
    {
      "bits_per_byte": 1.831042185496237,
      "learning_rate": 0.0002600640028592441,
      "loss_nats": 1.269181728363037,
      "step": 2310,
      "unclipped_grad_norm": 0.8500391840934753
    },
    {
      "bits_per_byte": 1.8431096921593277,
      "learning_rate": 0.0002596669658767859,
      "loss_nats": 1.2775462865829468,
      "step": 2320,
      "unclipped_grad_norm": 0.8417274355888367
    },
    {
      "bits_per_byte": 1.7906957435344257,
      "learning_rate": 0.000259268311297076,
      "loss_nats": 1.241215705871582,
      "step": 2330,
      "unclipped_grad_norm": 0.777140736579895
    },
    {
      "bits_per_byte": 1.825728265549878,
      "learning_rate": 0.00025886804593202114,
      "loss_nats": 1.265498399734497,
      "step": 2340,
      "unclipped_grad_norm": 0.8567083477973938
    },
    {
      "bits_per_byte": 1.7348746466132805,
      "learning_rate": 0.00025846617662105193,
      "loss_nats": 1.2025234699249268,
      "step": 2350,
      "unclipped_grad_norm": 0.7593752145767212
    },
    {
      "bits_per_byte": 1.8440781264663364,
      "learning_rate": 0.00025806271023100637,
      "loss_nats": 1.2782175540924072,
      "step": 2360,
      "unclipped_grad_norm": 0.8551052212715149
    },
    {
      "bits_per_byte": 1.871989878805465,
      "learning_rate": 0.00025765765365601165,
      "loss_nats": 1.2975645065307617,
      "step": 2370,
      "unclipped_grad_norm": 0.8494746685028076
    },
    {
      "bits_per_byte": 1.7757177744707837,
      "learning_rate": 0.00025725101381736707,
      "loss_nats": 1.2308337688446045,
      "step": 2380,
      "unclipped_grad_norm": 0.7841326594352722
    },
    {
      "bits_per_byte": 1.7735628318554726,
      "learning_rate": 0.00025684279766342567,
      "loss_nats": 1.2293400764465332,
      "step": 2390,
      "unclipped_grad_norm": 0.8951802849769592
    },
    {
      "bits_per_byte": 1.8680005692360513,
      "learning_rate": 0.00025643301216947504,
      "loss_nats": 1.2947993278503418,
      "step": 2400,
      "unclipped_grad_norm": 0.8339796662330627
    },
    {
      "bits_per_byte": 1.8459044102358484,
      "learning_rate": 0.00025602166433761866,
      "loss_nats": 1.279483437538147,
      "step": 2410,
      "unclipped_grad_norm": 0.9356110692024231
    },
    {
      "bits_per_byte": 1.931278833898608,
      "learning_rate": 0.0002556087611966561,
      "loss_nats": 1.338660478591919,
      "step": 2420,
      "unclipped_grad_norm": 0.8573734760284424
    },
    {
      "bits_per_byte": 1.825236739133712,
      "learning_rate": 0.00025519430980196277,
      "loss_nats": 1.265157699584961,
      "step": 2430,
      "unclipped_grad_norm": 0.8783476948738098
    },
    {
      "bits_per_byte": 1.7732133631089195,
      "learning_rate": 0.0002547783172353695,
      "loss_nats": 1.229097843170166,
      "step": 2440,
      "unclipped_grad_norm": 0.8393950462341309
    },
    {
      "bits_per_byte": 1.8696945983470499,
      "learning_rate": 0.0002543607906050415,
      "loss_nats": 1.295973539352417,
      "step": 2450,
      "unclipped_grad_norm": 0.8328628540039062
    },
    {
      "bits_per_byte": 1.8721511985319743,
      "learning_rate": 0.0002539417370453568,
      "loss_nats": 1.2976763248443604,
      "step": 2460,
      "unclipped_grad_norm": 0.8645251989364624
    },
    {
      "bits_per_byte": 1.8203225068679583,
      "learning_rate": 0.00025352116371678464,
      "loss_nats": 1.261751413345337,
      "step": 2470,
      "unclipped_grad_norm": 0.8000954985618591
    },
    {
      "bits_per_byte": 1.8206716316492098,
      "learning_rate": 0.0002530990778057627,
      "loss_nats": 1.261993408203125,
      "step": 2480,
      "unclipped_grad_norm": 0.7867690920829773
    },
    {
      "bits_per_byte": 1.7380601092725614,
      "learning_rate": 0.00025267548652457475,
      "loss_nats": 1.2047314643859863,
      "step": 2490,
      "unclipped_grad_norm": 0.7984139919281006
    },
    {
      "bits_per_byte": 1.822108202731867,
      "learning_rate": 0.00025225039711122696,
      "loss_nats": 1.2629891633987427,
      "step": 2500,
      "unclipped_grad_norm": 0.7798662781715393
    },
    {
      "bits_per_byte": 1.7878630172920644,
      "learning_rate": 0.00025182381682932464,
      "loss_nats": 1.2392522096633911,
      "step": 2510,
      "unclipped_grad_norm": 0.8921790719032288
    },
    {
      "bits_per_byte": 1.7912486677569497,
      "learning_rate": 0.0002513957529679477,
      "loss_nats": 1.2415989637374878,
      "step": 2520,
      "unclipped_grad_norm": 0.8250295519828796
    },
    {
      "bits_per_byte": 1.7871405181757898,
      "learning_rate": 0.00025096621284152656,
      "loss_nats": 1.2387514114379883,
      "step": 2530,
      "unclipped_grad_norm": 0.7922425270080566
    },
    {
      "bits_per_byte": 1.7956409326773348,
      "learning_rate": 0.0002505352037897168,
      "loss_nats": 1.2446434497833252,
      "step": 2540,
      "unclipped_grad_norm": 0.8244069218635559
    },
    {
      "bits_per_byte": 1.833951787983534,
      "learning_rate": 0.0002501027331772739,
      "loss_nats": 1.2711985111236572,
      "step": 2550,
      "unclipped_grad_norm": 0.8060020804405212
    },
    {
      "bits_per_byte": 1.7851159384098327,
      "learning_rate": 0.00024966880839392746,
      "loss_nats": 1.2373480796813965,
      "step": 2560,
      "unclipped_grad_norm": 0.7821792960166931
    },
    {
      "bits_per_byte": 1.768823849910948,
      "learning_rate": 0.0002492334368542548,
      "loss_nats": 1.2260552644729614,
      "step": 2570,
      "unclipped_grad_norm": 0.7924907207489014
    },
    {
      "bits_per_byte": 1.8169237857216074,
      "learning_rate": 0.00024879662599755416,
      "loss_nats": 1.2593955993652344,
      "step": 2580,
      "unclipped_grad_norm": 0.813592255115509
    },
    {
      "bits_per_byte": 1.7917042497990852,
      "learning_rate": 0.00024835838328771803,
      "loss_nats": 1.2419147491455078,
      "step": 2590,
      "unclipped_grad_norm": 0.8239418268203735
    },
    {
      "bits_per_byte": 1.8340940176357976,
      "learning_rate": 0.0002479187162131051,
      "loss_nats": 1.2712970972061157,
      "step": 2600,
      "unclipped_grad_norm": 0.7961696982383728
    },
    {
      "bits_per_byte": 1.7288738279593754,
      "learning_rate": 0.00024747763228641264,
      "loss_nats": 1.198364019393921,
      "step": 2610,
      "unclipped_grad_norm": 0.7893867492675781
    },
    {
      "bits_per_byte": 1.7710386424887592,
      "learning_rate": 0.00024703513904454795,
      "loss_nats": 1.2275904417037964,
      "step": 2620,
      "unclipped_grad_norm": 0.7791327238082886
    },
    {
      "bits_per_byte": 1.783225161146247,
      "learning_rate": 0.00024659124404849956,
      "loss_nats": 1.2360374927520752,
      "step": 2630,
      "unclipped_grad_norm": 0.7614359259605408
    },
    {
      "bits_per_byte": 1.8200944578729141,
      "learning_rate": 0.00024614595488320833,
      "loss_nats": 1.2615933418273926,
      "step": 2640,
      "unclipped_grad_norm": 0.831463098526001
    },
    {
      "bits_per_byte": 1.7566091260987187,
      "learning_rate": 0.00024569927915743745,
      "loss_nats": 1.2175886631011963,
      "step": 2650,
      "unclipped_grad_norm": 0.7739477157592773
    },
    {
      "bits_per_byte": 1.7470520501902724,
      "learning_rate": 0.0002452512245036427,
      "loss_nats": 1.2109642028808594,
      "step": 2660,
      "unclipped_grad_norm": 0.7929992079734802
    },
    {
      "bits_per_byte": 1.775358846678433,
      "learning_rate": 0.00024480179857784177,
      "loss_nats": 1.230584979057312,
      "step": 2670,
      "unclipped_grad_norm": 0.7438913583755493
    },
    {
      "bits_per_byte": 1.7678870604116983,
      "learning_rate": 0.00024435100905948387,
      "loss_nats": 1.2254059314727783,
      "step": 2680,
      "unclipped_grad_norm": 0.8285056352615356
    },
    {
      "bits_per_byte": 1.8436208245976922,
      "learning_rate": 0.00024389886365131793,
      "loss_nats": 1.2779005765914917,
      "step": 2690,
      "unclipped_grad_norm": 0.7684076428413391
    },
    {
      "bits_per_byte": 1.825501592416041,
      "learning_rate": 0.00024344537007926163,
      "loss_nats": 1.2653412818908691,
      "step": 2700,
      "unclipped_grad_norm": 0.8987715840339661
    },
    {
      "bits_per_byte": 1.7152053347994027,
      "learning_rate": 0.00024299053609226878,
      "loss_nats": 1.188889741897583,
      "step": 2710,
      "unclipped_grad_norm": 0.8146651983261108
    },
    {
      "bits_per_byte": 1.8198911743795942,
      "learning_rate": 0.00024253436946219733,
      "loss_nats": 1.2614524364471436,
      "step": 2720,
      "unclipped_grad_norm": 0.8013731241226196
    },
    {
      "bits_per_byte": 1.7448410412307802,
      "learning_rate": 0.0002420768779836764,
      "loss_nats": 1.2094316482543945,
      "step": 2730,
      "unclipped_grad_norm": 0.7481213212013245
    },
    {
      "bits_per_byte": 1.7681415947349752,
      "learning_rate": 0.00024161806947397305,
      "loss_nats": 1.2255823612213135,
      "step": 2740,
      "unclipped_grad_norm": 0.8155582547187805
    },
    {
      "bits_per_byte": 1.781660806953999,
      "learning_rate": 0.00024115795177285886,
      "loss_nats": 1.2349531650543213,
      "step": 2750,
      "unclipped_grad_norm": 0.816902756690979
    },
    {
      "bits_per_byte": 1.869134278870539,
      "learning_rate": 0.0002406965327424758,
      "loss_nats": 1.2955851554870605,
      "step": 2760,
      "unclipped_grad_norm": 0.8422117233276367
    },
    {
      "bits_per_byte": 1.7766440730283308,
      "learning_rate": 0.00024023382026720197,
      "loss_nats": 1.231475830078125,
      "step": 2770,
      "unclipped_grad_norm": 0.8072130084037781
    },
    {
      "bits_per_byte": 1.778421169759696,
      "learning_rate": 0.00023976982225351698,
      "loss_nats": 1.2327076196670532,
      "step": 2780,
      "unclipped_grad_norm": 0.8882554173469543
    },
    {
      "bits_per_byte": 1.7671284449387423,
      "learning_rate": 0.0002393045466298666,
      "loss_nats": 1.2248800992965698,
      "step": 2790,
      "unclipped_grad_norm": 0.8406802415847778
    },
    {
      "bits_per_byte": 1.719355792112675,
      "learning_rate": 0.00023883800134652763,
      "loss_nats": 1.191766619682312,
      "step": 2800,
      "unclipped_grad_norm": 0.8179776072502136
    },
    {
      "bits_per_byte": 1.7902867687906738,
      "learning_rate": 0.0002383701943754715,
      "loss_nats": 1.2409322261810303,
      "step": 2810,
      "unclipped_grad_norm": 0.8046079874038696
    },
    {
      "bits_per_byte": 1.7916516231079211,
      "learning_rate": 0.00023790113371022884,
      "loss_nats": 1.2418782711029053,
      "step": 2820,
      "unclipped_grad_norm": 0.7889978885650635
    },
    {
      "bits_per_byte": 1.717342391219024,
      "learning_rate": 0.0002374308273657521,
      "loss_nats": 1.190371036529541,
      "step": 2830,
      "unclipped_grad_norm": 0.7270697355270386
    },
    {
      "bits_per_byte": 1.8021411889493455,
      "learning_rate": 0.00023695928337827911,
      "loss_nats": 1.2491490840911865,
      "step": 2840,
      "unclipped_grad_norm": 0.8529788255691528
    },
    {
      "bits_per_byte": 1.7264863648000979,
      "learning_rate": 0.00023648650980519564,
      "loss_nats": 1.196709156036377,
      "step": 2850,
      "unclipped_grad_norm": 0.7487845420837402
    },
    {
      "bits_per_byte": 1.698717702026487,
      "learning_rate": 0.00023601251472489757,
      "loss_nats": 1.1774613857269287,
      "step": 2860,
      "unclipped_grad_norm": 0.8107967972755432
    },
    {
      "bits_per_byte": 1.8323354950307253,
      "learning_rate": 0.00023553730623665305,
      "loss_nats": 1.270078182220459,
      "step": 2870,
      "unclipped_grad_norm": 0.822874128818512
    },
    {
      "bits_per_byte": 1.7418315168233318,
      "learning_rate": 0.00023506089246046403,
      "loss_nats": 1.2073456048965454,
      "step": 2880,
      "unclipped_grad_norm": 0.7772940993309021
    },
    {
      "bits_per_byte": 1.7973201712803593,
      "learning_rate": 0.00023458328153692736,
      "loss_nats": 1.245807409286499,
      "step": 2890,
      "unclipped_grad_norm": 0.7877682447433472
    },
    {
      "bits_per_byte": 1.7076477291898875,
      "learning_rate": 0.0002341044816270961,
      "loss_nats": 1.1836512088775635,
      "step": 2900,
      "unclipped_grad_norm": 0.8104777932167053
    },
    {
      "bits_per_byte": 1.7337485042154308,
      "learning_rate": 0.00023362450091233953,
      "loss_nats": 1.2017428874969482,
      "step": 2910,
      "unclipped_grad_norm": 0.8031578063964844
    },
    {
      "bits_per_byte": 1.8229534974608574,
      "learning_rate": 0.00023314334759420388,
      "loss_nats": 1.2635750770568848,
      "step": 2920,
      "unclipped_grad_norm": 0.7966766953468323
    },
    {
      "bits_per_byte": 1.766033431400699,
      "learning_rate": 0.0002326610298942717,
      "loss_nats": 1.22412109375,
      "step": 2930,
      "unclipped_grad_norm": 0.8636581301689148
    },
    {
      "bits_per_byte": 1.8135838826418518,
      "learning_rate": 0.00023217755605402175,
      "loss_nats": 1.2570805549621582,
      "step": 2940,
      "unclipped_grad_norm": 0.8673250675201416
    },
    {
      "bits_per_byte": 1.7613176671140407,
      "learning_rate": 0.00023169293433468802,
      "loss_nats": 1.2208523750305176,
      "step": 2950,
      "unclipped_grad_norm": 0.7721947431564331
    },
    {
      "bits_per_byte": 1.7305283010606756,
      "learning_rate": 0.00023120717301711857,
      "loss_nats": 1.1995108127593994,
      "step": 2960,
      "unclipped_grad_norm": 0.7505601644515991
    },
    {
      "bits_per_byte": 1.8081494028572378,
      "learning_rate": 0.00023072028040163408,
      "loss_nats": 1.253313660621643,
      "step": 2970,
      "unclipped_grad_norm": 0.7939547896385193
    },
    {
      "bits_per_byte": 1.72656255311443,
      "learning_rate": 0.00023023226480788593,
      "loss_nats": 1.196761965751648,
      "step": 2980,
      "unclipped_grad_norm": 0.7610272765159607
    },
    {
      "bits_per_byte": 1.8584056571444152,
      "learning_rate": 0.00022974313457471416,
      "loss_nats": 1.2881486415863037,
      "step": 2990,
      "unclipped_grad_norm": 0.833854615688324
    },
    {
      "bits_per_byte": 1.7959412143857414,
      "learning_rate": 0.0002292528980600049,
      "loss_nats": 1.2448515892028809,
      "step": 3000,
      "unclipped_grad_norm": 0.7871666550636292
    },
    {
      "bits_per_byte": 1.7543190051198296,
      "learning_rate": 0.00022876156364054758,
      "loss_nats": 1.216001272201538,
      "step": 3010,
      "unclipped_grad_norm": 0.7730440497398376
    },
    {
      "bits_per_byte": 1.7304921847039942,
      "learning_rate": 0.0002282691397118918,
      "loss_nats": 1.1994857788085938,
      "step": 3020,
      "unclipped_grad_norm": 0.8199000954627991
    },
    {
      "bits_per_byte": 1.7578830015936595,
      "learning_rate": 0.00022777563468820376,
      "loss_nats": 1.218471646308899,
      "step": 3030,
      "unclipped_grad_norm": 0.9139093160629272
    },
    {
      "bits_per_byte": 1.7370483353375354,
      "learning_rate": 0.00022728105700212282,
      "loss_nats": 1.204030156135559,
      "step": 3040,
      "unclipped_grad_norm": 0.7832452058792114
    },
    {
      "bits_per_byte": 1.745498014957076,
      "learning_rate": 0.00022678541510461698,
      "loss_nats": 1.2098870277404785,
      "step": 3050,
      "unclipped_grad_norm": 0.7990915179252625
    },
    {
      "bits_per_byte": 1.7370017280391516,
      "learning_rate": 0.00022628871746483876,
      "loss_nats": 1.2039978504180908,
      "step": 3060,
      "unclipped_grad_norm": 0.8140543103218079
    },
    {
      "bits_per_byte": 1.7324813360438736,
      "learning_rate": 0.0002257909725699804,
      "loss_nats": 1.200864553451538,
      "step": 3070,
      "unclipped_grad_norm": 0.8290132284164429
    },
    {
      "bits_per_byte": 1.73554830265671,
      "learning_rate": 0.00022529218892512886,
      "loss_nats": 1.2029904127120972,
      "step": 3080,
      "unclipped_grad_norm": 0.8363073468208313
    },
    {
      "bits_per_byte": 1.7064868462965634,
      "learning_rate": 0.00022479237505312034,
      "loss_nats": 1.1828465461730957,
      "step": 3090,
      "unclipped_grad_norm": 0.7568578124046326
    },
    {
      "bits_per_byte": 1.8015361539836103,
      "learning_rate": 0.00022429153949439496,
      "loss_nats": 1.2487297058105469,
      "step": 3100,
      "unclipped_grad_norm": 0.8538148999214172
    },
    {
      "bits_per_byte": 1.7730212584879057,
      "learning_rate": 0.00022378969080685053,
      "loss_nats": 1.2289646863937378,
      "step": 3110,
      "unclipped_grad_norm": 0.8123647570610046
    },
    {
      "bits_per_byte": 1.664611306585931,
      "learning_rate": 0.0002232868375656965,
      "loss_nats": 1.1538206338882446,
      "step": 3120,
      "unclipped_grad_norm": 0.814990758895874
    },
    {
      "bits_per_byte": 1.7603762340832176,
      "learning_rate": 0.00022278298836330733,
      "loss_nats": 1.2201998233795166,
      "step": 3130,
      "unclipped_grad_norm": 0.8308927416801453
    },
    {
      "bits_per_byte": 1.713619138810495,
      "learning_rate": 0.00022227815180907572,
      "loss_nats": 1.1877902746200562,
      "step": 3140,
      "unclipped_grad_norm": 0.7710813879966736
    },
    {
      "bits_per_byte": 1.7093383186478688,
      "learning_rate": 0.00022177233652926546,
      "loss_nats": 1.1848230361938477,
      "step": 3150,
      "unclipped_grad_norm": 0.7859835624694824
    },
    {
      "bits_per_byte": 1.7563532159142348,
      "learning_rate": 0.00022126555116686416,
      "loss_nats": 1.2174112796783447,
      "step": 3160,
      "unclipped_grad_norm": 0.811743974685669
    },
    {
      "bits_per_byte": 1.8161149513145995,
      "learning_rate": 0.00022075780438143532,
      "loss_nats": 1.258834958076477,
      "step": 3170,
      "unclipped_grad_norm": 0.8695415258407593
    },
    {
      "bits_per_byte": 1.6889931150161004,
      "learning_rate": 0.00022024910484897065,
      "loss_nats": 1.1707208156585693,
      "step": 3180,
      "unclipped_grad_norm": 0.8201394081115723
    },
    {
      "bits_per_byte": 1.7704115937437135,
      "learning_rate": 0.00021973946126174158,
      "loss_nats": 1.2271558046340942,
      "step": 3190,
      "unclipped_grad_norm": 0.8031932711601257
    },
    {
      "bits_per_byte": 1.6638831320421776,
      "learning_rate": 0.00021922888232815086,
      "loss_nats": 1.1533159017562866,
      "step": 3200,
      "unclipped_grad_norm": 0.7531365156173706
    },
    {
      "bits_per_byte": 1.7080410535124106,
      "learning_rate": 0.0002187173767725837,
      "loss_nats": 1.1839238405227661,
      "step": 3210,
      "unclipped_grad_norm": 0.7989531755447388
    },
    {
      "bits_per_byte": 1.7142219380017691,
      "learning_rate": 0.0002182049533352588,
      "loss_nats": 1.1882081031799316,
      "step": 3220,
      "unclipped_grad_norm": 0.8516196012496948
    },
    {
      "bits_per_byte": 1.687988048404458,
      "learning_rate": 0.00021769162077207884,
      "loss_nats": 1.1700241565704346,
      "step": 3230,
      "unclipped_grad_norm": 0.8453141450881958
    },
    {
      "bits_per_byte": 1.725164850110868,
      "learning_rate": 0.00021717738785448102,
      "loss_nats": 1.1957931518554688,
      "step": 3240,
      "unclipped_grad_norm": 0.8131914734840393
    },
    {
      "bits_per_byte": 1.7438406181507111,
      "learning_rate": 0.00021666226336928708,
      "loss_nats": 1.2087382078170776,
      "step": 3250,
      "unclipped_grad_norm": 0.8390238285064697
    },
    {
      "bits_per_byte": 1.8351064795014271,
      "learning_rate": 0.0002161462561185532,
      "loss_nats": 1.2719988822937012,
      "step": 3260,
      "unclipped_grad_norm": 0.8244672417640686
    },
    {
      "bits_per_byte": 1.6914478233918653,
      "learning_rate": 0.00021562937491941957,
      "loss_nats": 1.1724222898483276,
      "step": 3270,
      "unclipped_grad_norm": 0.8076629042625427
    },
    {
      "bits_per_byte": 1.6199020087370317,
      "learning_rate": 0.00021511162860395975,
      "loss_nats": 1.1228305101394653,
      "step": 3280,
      "unclipped_grad_norm": 0.7831076979637146
    },
    {
      "bits_per_byte": 1.6792207168287738,
      "learning_rate": 0.0002145930260190298,
      "loss_nats": 1.1639471054077148,
      "step": 3290,
      "unclipped_grad_norm": 0.8022177219390869
    },
    {
      "bits_per_byte": 1.7567814527148833,
      "learning_rate": 0.00021407357602611698,
      "loss_nats": 1.2177081108093262,
      "step": 3300,
      "unclipped_grad_norm": 0.8066262602806091
    },
    {
      "bits_per_byte": 1.745768887632185,
      "learning_rate": 0.0002135532875011885,
      "loss_nats": 1.210074782371521,
      "step": 3310,
      "unclipped_grad_norm": 0.8170367479324341
    },
    {
      "bits_per_byte": 1.7488066171943748,
      "learning_rate": 0.0002130321693345397,
      "loss_nats": 1.2121803760528564,
      "step": 3320,
      "unclipped_grad_norm": 0.8205949664115906
    },
    {
      "bits_per_byte": 1.6972405430382262,
      "learning_rate": 0.00021251023043064224,
      "loss_nats": 1.176437497138977,
      "step": 3330,
      "unclipped_grad_norm": 0.7750418782234192
    },
    {
      "bits_per_byte": 1.7083963696690934,
      "learning_rate": 0.00021198747970799188,
      "loss_nats": 1.184170126914978,
      "step": 3340,
      "unclipped_grad_norm": 0.7907351851463318
    },
    {
      "bits_per_byte": 1.7221589373390878,
      "learning_rate": 0.00021146392609895618,
      "loss_nats": 1.1937096118927002,
      "step": 3350,
      "unclipped_grad_norm": 0.7935147285461426
    },
    {
      "bits_per_byte": 1.6939794080125659,
      "learning_rate": 0.00021093957854962178,
      "loss_nats": 1.1741770505905151,
      "step": 3360,
      "unclipped_grad_norm": 0.7715696096420288
    },
    {
      "bits_per_byte": 1.6536716621471985,
      "learning_rate": 0.00021041444601964157,
      "loss_nats": 1.146237850189209,
      "step": 3370,
      "unclipped_grad_norm": 0.7847044467926025
    },
    {
      "bits_per_byte": 1.7430156173745224,
      "learning_rate": 0.00020988853748208164,
      "loss_nats": 1.2081663608551025,
      "step": 3380,
      "unclipped_grad_norm": 0.9182605743408203
    },
    {
      "bits_per_byte": 1.7282608817917002,
      "learning_rate": 0.00020936186192326782,
      "loss_nats": 1.197939157485962,
      "step": 3390,
      "unclipped_grad_norm": 0.8387140035629272
    },
    {
      "bits_per_byte": 1.7168453613580306,
      "learning_rate": 0.00020883442834263225,
      "loss_nats": 1.1900265216827393,
      "step": 3400,
      "unclipped_grad_norm": 0.827458918094635
    },
    {
      "bits_per_byte": 1.744452876387783,
      "learning_rate": 0.0002083062457525596,
      "loss_nats": 1.2091625928878784,
      "step": 3410,
      "unclipped_grad_norm": 0.8461803197860718
    },
    {
      "bits_per_byte": 1.7169175940713928,
      "learning_rate": 0.00020777732317823306,
      "loss_nats": 1.1900765895843506,
      "step": 3420,
      "unclipped_grad_norm": 0.7754979133605957
    },
    {
      "bits_per_byte": 1.665198971303929,
      "learning_rate": 0.00020724766965748005,
      "loss_nats": 1.1542279720306396,
      "step": 3430,
      "unclipped_grad_norm": 0.784263551235199
    },
    {
      "bits_per_byte": 1.6807204055442975,
      "learning_rate": 0.00020671729424061788,
      "loss_nats": 1.1649866104125977,
      "step": 3440,
      "unclipped_grad_norm": 0.768701434135437
    },
    {
      "bits_per_byte": 1.674807642007634,
      "learning_rate": 0.00020618620599029906,
      "loss_nats": 1.1608881950378418,
      "step": 3450,
      "unclipped_grad_norm": 0.8573304414749146
    },
    {
      "bits_per_byte": 1.6607244986764313,
      "learning_rate": 0.00020565441398135649,
      "loss_nats": 1.151126503944397,
      "step": 3460,
      "unclipped_grad_norm": 0.7546505331993103
    },
    {
      "bits_per_byte": 1.7168016777647113,
      "learning_rate": 0.00020512192730064827,
      "loss_nats": 1.1899962425231934,
      "step": 3470,
      "unclipped_grad_norm": 0.8346678614616394
    },
    {
      "bits_per_byte": 1.7794081781929978,
      "learning_rate": 0.00020458875504690273,
      "loss_nats": 1.2333917617797852,
      "step": 3480,
      "unclipped_grad_norm": 0.8080501556396484
    },
    {
      "bits_per_byte": 1.7098234817059528,
      "learning_rate": 0.00020405490633056245,
      "loss_nats": 1.1851593255996704,
      "step": 3490,
      "unclipped_grad_norm": 0.7828541398048401
    },
    {
      "bits_per_byte": 1.6618220919542377,
      "learning_rate": 0.00020352039027362917,
      "loss_nats": 1.15188729763031,
      "step": 3500,
      "unclipped_grad_norm": 0.8354499936103821
    },
    {
      "bits_per_byte": 1.670962969847596,
      "learning_rate": 0.0002029852160095075,
      "loss_nats": 1.158223271369934,
      "step": 3510,
      "unclipped_grad_norm": 0.7484581470489502
    },
    {
      "bits_per_byte": 1.7077951183216769,
      "learning_rate": 0.000202449392682849,
      "loss_nats": 1.1837533712387085,
      "step": 3520,
      "unclipped_grad_norm": 0.8215554356575012
    },
    {
      "bits_per_byte": 1.6640239858332344,
      "learning_rate": 0.0002019129294493959,
      "loss_nats": 1.1534135341644287,
      "step": 3530,
      "unclipped_grad_norm": 0.7679988741874695
    },
    {
      "bits_per_byte": 1.6934656958344385,
      "learning_rate": 0.00020137583547582478,
      "loss_nats": 1.173820972442627,
      "step": 3540,
      "unclipped_grad_norm": 0.849611759185791
    },
    {
      "bits_per_byte": 1.7380399873024104,
      "learning_rate": 0.00020083811993958973,
      "loss_nats": 1.2047175168991089,
      "step": 3550,
      "unclipped_grad_norm": 0.8348926305770874
    },
    {
      "bits_per_byte": 1.6724150193688307,
      "learning_rate": 0.00020029979202876563,
      "loss_nats": 1.1592297554016113,
      "step": 3560,
      "unclipped_grad_norm": 0.82209312915802
    },
    {
      "bits_per_byte": 1.707797526078789,
      "learning_rate": 0.00019976086094189118,
      "loss_nats": 1.1837550401687622,
      "step": 3570,
      "unclipped_grad_norm": 0.814924955368042
    },
    {
      "bits_per_byte": 1.725196322935976,
      "learning_rate": 0.00019922133588781175,
      "loss_nats": 1.1958149671554565,
      "step": 3580,
      "unclipped_grad_norm": 0.8165888786315918
    },
    {
      "bits_per_byte": 1.7086591591596116,
      "learning_rate": 0.0001986812260855219,
      "loss_nats": 1.1843522787094116,
      "step": 3590,
      "unclipped_grad_norm": 0.8172020316123962
    },
    {
      "bits_per_byte": 1.7535423314685332,
      "learning_rate": 0.00019814054076400787,
      "loss_nats": 1.2154629230499268,
      "step": 3600,
      "unclipped_grad_norm": 0.8490144610404968
    },
    {
      "bits_per_byte": 1.7320164669385913,
      "learning_rate": 0.00019759928916209004,
      "loss_nats": 1.2005423307418823,
      "step": 3610,
      "unclipped_grad_norm": 0.8558303117752075
    },
    {
      "bits_per_byte": 1.671523289324107,
      "learning_rate": 0.00019705748052826493,
      "loss_nats": 1.1586116552352905,
      "step": 3620,
      "unclipped_grad_norm": 0.7700185179710388
    },
    {
      "bits_per_byte": 1.6783824733884685,
      "learning_rate": 0.00019651512412054723,
      "loss_nats": 1.1633660793304443,
      "step": 3630,
      "unclipped_grad_norm": 0.8705776333808899
    },
    {
      "bits_per_byte": 1.6085100499265301,
      "learning_rate": 0.00019597222920631144,
      "loss_nats": 1.1149342060089111,
      "step": 3640,
      "unclipped_grad_norm": 0.7363900542259216
    },
    {
      "bits_per_byte": 1.69346002040696,
      "learning_rate": 0.0001954288050621338,
      "loss_nats": 1.1738170385360718,
      "step": 3650,
      "unclipped_grad_norm": 0.7923496961593628
    },
    {
      "bits_per_byte": 1.6919579239343243,
      "learning_rate": 0.00019488486097363363,
      "loss_nats": 1.1727758646011353,
      "step": 3660,
      "unclipped_grad_norm": 0.8016297221183777
    },
    {
      "bits_per_byte": 1.7219477426438283,
      "learning_rate": 0.00019434040623531452,
      "loss_nats": 1.1935632228851318,
      "step": 3670,
      "unclipped_grad_norm": 0.8946071863174438
    },
    {
      "bits_per_byte": 1.6611216066172736,
      "learning_rate": 0.0001937954501504058,
      "loss_nats": 1.1514017581939697,
      "step": 3680,
      "unclipped_grad_norm": 0.7832621932029724
    },
    {
      "bits_per_byte": 1.7299588665036687,
      "learning_rate": 0.00019325000203070333,
      "loss_nats": 1.1991161108016968,
      "step": 3690,
      "unclipped_grad_norm": 0.8317174911499023
    },
    {
      "bits_per_byte": 1.6188276331170917,
      "learning_rate": 0.0001927040711964106,
      "loss_nats": 1.1220858097076416,
      "step": 3700,
      "unclipped_grad_norm": 0.7848891615867615
    },
    {
      "bits_per_byte": 1.69280218676741,
      "learning_rate": 0.00019215766697597923,
      "loss_nats": 1.17336106300354,
      "step": 3710,
      "unclipped_grad_norm": 0.7887690663337708
    },
    {
      "bits_per_byte": 1.7133850704226707,
      "learning_rate": 0.00019161079870594978,
      "loss_nats": 1.1876280307769775,
      "step": 3720,
      "unclipped_grad_norm": 0.8315525650978088
    },
    {
      "bits_per_byte": 1.6847540866376338,
      "learning_rate": 0.0001910634757307922,
      "loss_nats": 1.1677825450897217,
      "step": 3730,
      "unclipped_grad_norm": 0.8175208568572998
    },
    {
      "bits_per_byte": 1.6646517225088835,
      "learning_rate": 0.0001905157074027459,
      "loss_nats": 1.153848648071289,
      "step": 3740,
      "unclipped_grad_norm": 0.795343816280365
    },
    {
      "bits_per_byte": 1.6976003307438312,
      "learning_rate": 0.00018996750308166037,
      "loss_nats": 1.1766868829727173,
      "step": 3750,
      "unclipped_grad_norm": 0.8138338327407837
    },
    {
      "bits_per_byte": 1.7200370153927425,
      "learning_rate": 0.00018941887213483482,
      "loss_nats": 1.1922388076782227,
      "step": 3760,
      "unclipped_grad_norm": 0.8444334268569946
    },
    {
      "bits_per_byte": 1.7215936303657018,
      "learning_rate": 0.00018886982393685845,
      "loss_nats": 1.1933177709579468,
      "step": 3770,
      "unclipped_grad_norm": 0.7583279609680176
    },
    {
      "bits_per_byte": 1.7717356881727062,
      "learning_rate": 0.00018832036786945008,
      "loss_nats": 1.2280735969543457,
      "step": 3780,
      "unclipped_grad_norm": 0.8467698693275452
    },
    {
      "bits_per_byte": 1.6571842358084203,
      "learning_rate": 0.0001877705133212979,
      "loss_nats": 1.1486725807189941,
      "step": 3790,
      "unclipped_grad_norm": 0.769040584564209
    },
    {
      "bits_per_byte": 1.6766210270783317,
      "learning_rate": 0.00018722026968789906,
      "loss_nats": 1.1621451377868652,
      "step": 3800,
      "unclipped_grad_norm": 0.7948989272117615
    },
    {
      "bits_per_byte": 1.6280214816669156,
      "learning_rate": 0.00018666964637139907,
      "loss_nats": 1.1284584999084473,
      "step": 3810,
      "unclipped_grad_norm": 0.8209450840950012
    },
    {
      "bits_per_byte": 1.6241315780697008,
      "learning_rate": 0.00018611865278043115,
      "loss_nats": 1.1257622241973877,
      "step": 3820,
      "unclipped_grad_norm": 0.81658935546875
    },
    {
      "bits_per_byte": 1.6981439679032087,
      "learning_rate": 0.00018556729832995555,
      "loss_nats": 1.1770637035369873,
      "step": 3830,
      "unclipped_grad_norm": 0.8478888273239136
    },
    {
      "bits_per_byte": 1.649875145129402,
      "learning_rate": 0.00018501559244109864,
      "loss_nats": 1.1436063051223755,
      "step": 3840,
      "unclipped_grad_norm": 0.8229350447654724
    },
    {
      "bits_per_byte": 1.621423539249215,
      "learning_rate": 0.00018446354454099178,
      "loss_nats": 1.123885154724121,
      "step": 3850,
      "unclipped_grad_norm": 0.8153557181358337
    },
    {
      "bits_per_byte": 1.6024271955481633,
      "learning_rate": 0.00018391116406261048,
      "loss_nats": 1.1107178926467896,
      "step": 3860,
      "unclipped_grad_norm": 0.794911801815033
    },
    {
      "bits_per_byte": 1.5664059452729007,
      "learning_rate": 0.00018335846044461305,
      "loss_nats": 1.085749864578247,
      "step": 3870,
      "unclipped_grad_norm": 0.75998854637146
    },
    {
      "bits_per_byte": 1.6739675067581694,
      "learning_rate": 0.0001828054431311793,
      "loss_nats": 1.1603058576583862,
      "step": 3880,
      "unclipped_grad_norm": 0.8571500182151794
    },
    {
      "bits_per_byte": 1.6411153807903556,
      "learning_rate": 0.00018225212157184934,
      "loss_nats": 1.137534499168396,
      "step": 3890,
      "unclipped_grad_norm": 0.8254691958427429
    },
    {
      "bits_per_byte": 1.6427378651185953,
      "learning_rate": 0.00018169850522136192,
      "loss_nats": 1.138659119606018,
      "step": 3900,
      "unclipped_grad_norm": 0.8312620520591736
    },
    {
      "bits_per_byte": 1.6573687731927962,
      "learning_rate": 0.000181144603539493,
      "loss_nats": 1.1488004922866821,
      "step": 3910,
      "unclipped_grad_norm": 0.8051927089691162
    },
    {
      "bits_per_byte": 1.6875147521492837,
      "learning_rate": 0.00018059042599089412,
      "loss_nats": 1.1696960926055908,
      "step": 3920,
      "unclipped_grad_norm": 0.8500935435295105
    },
    {
      "bits_per_byte": 1.685968628118027,
      "learning_rate": 0.00018003598204493048,
      "loss_nats": 1.1686244010925293,
      "step": 3930,
      "unclipped_grad_norm": 0.8186569213867188
    },
    {
      "bits_per_byte": 1.6142946863883005,
      "learning_rate": 0.00017948128117551936,
      "loss_nats": 1.1189438104629517,
      "step": 3940,
      "unclipped_grad_norm": 0.7923241853713989
    },
    {
      "bits_per_byte": 1.7195771337843353,
      "learning_rate": 0.00017892633286096816,
      "loss_nats": 1.1919200420379639,
      "step": 3950,
      "unclipped_grad_norm": 0.8632136583328247
    },
    {
      "bits_per_byte": 1.7081789835984027,
      "learning_rate": 0.00017837114658381247,
      "loss_nats": 1.1840194463729858,
      "step": 3960,
      "unclipped_grad_norm": 0.8128528594970703
    },
    {
      "bits_per_byte": 1.6418629893736563,
      "learning_rate": 0.00017781573183065386,
      "loss_nats": 1.1380527019500732,
      "step": 3970,
      "unclipped_grad_norm": 0.8211596608161926
    },
    {
      "bits_per_byte": 1.6416170541829225,
      "learning_rate": 0.0001772600980919981,
      "loss_nats": 1.1378822326660156,
      "step": 3980,
      "unclipped_grad_norm": 0.8518010973930359
    },
    {
      "bits_per_byte": 1.672920992327669,
      "learning_rate": 0.0001767042548620927,
      "loss_nats": 1.1595804691314697,
      "step": 3990,
      "unclipped_grad_norm": 0.8055341243743896
    },
    {
      "bits_per_byte": 1.6017095119461129,
      "learning_rate": 0.00017614821163876486,
      "loss_nats": 1.1102204322814941,
      "step": 4000,
      "unclipped_grad_norm": 0.7798590064048767
    },
    {
      "bits_per_byte": 1.6685314791296977,
      "learning_rate": 0.00017559197792325906,
      "loss_nats": 1.1565378904342651,
      "step": 4010,
      "unclipped_grad_norm": 0.8833235502243042
    },
    {
      "bits_per_byte": 1.6832612772281446,
      "learning_rate": 0.0001750355632200748,
      "loss_nats": 1.166747808456421,
      "step": 4020,
      "unclipped_grad_norm": 0.8783014416694641
    },
    {
      "bits_per_byte": 1.6355704881438877,
      "learning_rate": 0.00017447897703680418,
      "loss_nats": 1.1336910724639893,
      "step": 4030,
      "unclipped_grad_norm": 0.831271231174469
    },
    {
      "bits_per_byte": 1.6221185211413516,
      "learning_rate": 0.0001739222288839694,
      "loss_nats": 1.1243668794631958,
      "step": 4040,
      "unclipped_grad_norm": 0.828468382358551
    },
    {
      "bits_per_byte": 1.6444031731269,
      "learning_rate": 0.00017336532827486026,
      "loss_nats": 1.1398134231567383,
      "step": 4050,
      "unclipped_grad_norm": 0.8378818035125732
    },
    {
      "bits_per_byte": 1.6213299806871455,
      "learning_rate": 0.00017280828472537163,
      "loss_nats": 1.1238203048706055,
      "step": 4060,
      "unclipped_grad_norm": 0.808568000793457
    },
    {
      "bits_per_byte": 1.749336667724334,
      "learning_rate": 0.0001722511077538408,
      "loss_nats": 1.212547779083252,
      "step": 4070,
      "unclipped_grad_norm": 0.8772934675216675
    },
    {
      "bits_per_byte": 1.5885019322904526,
      "learning_rate": 0.00017169380688088497,
      "loss_nats": 1.1010656356811523,
      "step": 4080,
      "unclipped_grad_norm": 0.8058938980102539
    },
    {
      "bits_per_byte": 1.6013793052564562,
      "learning_rate": 0.0001711363916292384,
      "loss_nats": 1.1099915504455566,
      "step": 4090,
      "unclipped_grad_norm": 0.8040578365325928
    },
    {
      "bits_per_byte": 1.737544161319973,
      "learning_rate": 0.00017057887152358974,
      "loss_nats": 1.204373836517334,
      "step": 4100,
      "unclipped_grad_norm": 0.8918697834014893
    },
    {
      "bits_per_byte": 1.6258001537483717,
      "learning_rate": 0.0001700212560904194,
      "loss_nats": 1.1269187927246094,
      "step": 4110,
      "unclipped_grad_norm": 0.8254836797714233
    },
    {
      "bits_per_byte": 1.6466504704257243,
      "learning_rate": 0.00016946355485783656,
      "loss_nats": 1.1413711309432983,
      "step": 4120,
      "unclipped_grad_norm": 0.8797640800476074
    },
    {
      "bits_per_byte": 1.7012891866221878,
      "learning_rate": 0.0001689057773554166,
      "loss_nats": 1.179243803024292,
      "step": 4130,
      "unclipped_grad_norm": 0.8626418709754944
    },
    {
      "bits_per_byte": 1.6871897049391529,
      "learning_rate": 0.00016834793311403806,
      "loss_nats": 1.1694707870483398,
      "step": 4140,
      "unclipped_grad_norm": 0.8501971364021301
    },
    {
      "bits_per_byte": 1.629858600343432,
      "learning_rate": 0.00016779003166571981,
      "loss_nats": 1.1297318935394287,
      "step": 4150,
      "unclipped_grad_norm": 0.8614740371704102
    },
    {
      "bits_per_byte": 1.6422026551091102,
      "learning_rate": 0.00016723208254345838,
      "loss_nats": 1.138288140296936,
      "step": 4160,
      "unclipped_grad_norm": 0.8621925115585327
    },
    {
      "bits_per_byte": 1.6599294228814925,
      "learning_rate": 0.00016667409528106481,
      "loss_nats": 1.1505753993988037,
      "step": 4170,
      "unclipped_grad_norm": 0.8789071440696716
    },
    {
      "bits_per_byte": 1.6412854716320588,
      "learning_rate": 0.00016611607941300184,
      "loss_nats": 1.1376523971557617,
      "step": 4180,
      "unclipped_grad_norm": 0.8332215547561646
    },
    {
      "bits_per_byte": 1.6753517951149641,
      "learning_rate": 0.0001655580444742211,
      "loss_nats": 1.1612653732299805,
      "step": 4190,
      "unclipped_grad_norm": 0.8483413457870483
    },
    {
      "bits_per_byte": 1.6524258198243482,
      "learning_rate": 0.00016499999999999997,
      "loss_nats": 1.1453742980957031,
      "step": 4200,
      "unclipped_grad_norm": 0.8514559268951416
    },
    {
      "bits_per_byte": 1.6764456047744516,
      "learning_rate": 0.0001644419555257789,
      "loss_nats": 1.1620235443115234,
      "step": 4210,
      "unclipped_grad_norm": 0.7994921803474426
    },
    {
      "bits_per_byte": 1.6553684429806108,
      "learning_rate": 0.0001638839205869981,
      "loss_nats": 1.147413969039917,
      "step": 4220,
      "unclipped_grad_norm": 0.8137004375457764
    },
    {
      "bits_per_byte": 1.654246428166382,
      "learning_rate": 0.0001633259047189352,
      "loss_nats": 1.1466362476348877,
      "step": 4230,
      "unclipped_grad_norm": 0.8624707460403442
    },
    {
      "bits_per_byte": 1.670982919835096,
      "learning_rate": 0.0001627679174565416,
      "loss_nats": 1.158237099647522,
      "step": 4240,
      "unclipped_grad_norm": 0.83209228515625
    },
    {
      "bits_per_byte": 1.605450134602379,
      "learning_rate": 0.00016220996833428016,
      "loss_nats": 1.1128132343292236,
      "step": 4250,
      "unclipped_grad_norm": 0.8165753483772278
    },
    {
      "bits_per_byte": 1.64798711958823,
      "learning_rate": 0.00016165206688596191,
      "loss_nats": 1.142297625541687,
      "step": 4260,
      "unclipped_grad_norm": 0.8390073180198669
    },
    {
      "bits_per_byte": 1.5630770490828,
      "learning_rate": 0.00016109422264458335,
      "loss_nats": 1.0834424495697021,
      "step": 4270,
      "unclipped_grad_norm": 0.7599806189537048
    },
    {
      "bits_per_byte": 1.634252757072977,
      "learning_rate": 0.00016053644514216336,
      "loss_nats": 1.1327776908874512,
      "step": 4280,
      "unclipped_grad_norm": 0.8349722027778625
    },
    {
      "bits_per_byte": 1.622665597953746,
      "learning_rate": 0.00015997874390958058,
      "loss_nats": 1.1247460842132568,
      "step": 4290,
      "unclipped_grad_norm": 0.8544122576713562
    },
    {
      "bits_per_byte": 1.5775992641216556,
      "learning_rate": 0.00015942112847641018,
      "loss_nats": 1.0935084819793701,
      "step": 4300,
      "unclipped_grad_norm": 0.8515903353691101
    },
    {
      "bits_per_byte": 1.5682213941354084,
      "learning_rate": 0.00015886360837076156,
      "loss_nats": 1.0870082378387451,
      "step": 4310,
      "unclipped_grad_norm": 0.8175749182701111
    },
    {
      "bits_per_byte": 1.709660270170284,
      "learning_rate": 0.00015830619311911495,
      "loss_nats": 1.1850461959838867,
      "step": 4320,
      "unclipped_grad_norm": 0.9185000061988831
    },
    {
      "bits_per_byte": 1.6398645509706304,
      "learning_rate": 0.00015774889224615915,
      "loss_nats": 1.1366674900054932,
      "step": 4330,
      "unclipped_grad_norm": 0.8850803971290588
    },
    {
      "bits_per_byte": 1.5869262272432476,
      "learning_rate": 0.00015719171527462834,
      "loss_nats": 1.099973440170288,
      "step": 4340,
      "unclipped_grad_norm": 0.8476384878158569
    },
    {
      "bits_per_byte": 1.5594358323987327,
      "learning_rate": 0.0001566346717251397,
      "loss_nats": 1.080918550491333,
      "step": 4350,
      "unclipped_grad_norm": 0.8173791170120239
    },
    {
      "bits_per_byte": 1.6383724294917448,
      "learning_rate": 0.00015607777111603056,
      "loss_nats": 1.1356332302093506,
      "step": 4360,
      "unclipped_grad_norm": 0.8429195880889893
    },
    {
      "bits_per_byte": 1.6201880158854165,
      "learning_rate": 0.00015552102296319576,
      "loss_nats": 1.1230287551879883,
      "step": 4370,
      "unclipped_grad_norm": 0.8339970707893372
    },
    {
      "bits_per_byte": 1.5560377991829855,
      "learning_rate": 0.00015496443677992516,
      "loss_nats": 1.0785632133483887,
      "step": 4380,
      "unclipped_grad_norm": 0.7866214513778687
    },
    {
      "bits_per_byte": 1.7055524645544258,
      "learning_rate": 0.0001544080220767409,
      "loss_nats": 1.1821988821029663,
      "step": 4390,
      "unclipped_grad_norm": 0.8178918361663818
    },
    {
      "bits_per_byte": 1.6491636529027824,
      "learning_rate": 0.00015385178836123511,
      "loss_nats": 1.143113136291504,
      "step": 4400,
      "unclipped_grad_norm": 0.8444706797599792
    },
    {
      "bits_per_byte": 1.6592194784987309,
      "learning_rate": 0.00015329574513790726,
      "loss_nats": 1.150083303451538,
      "step": 4410,
      "unclipped_grad_norm": 0.8914594054222107
    },
    {
      "bits_per_byte": 1.676016508060549,
      "learning_rate": 0.00015273990190800186,
      "loss_nats": 1.1617261171340942,
      "step": 4420,
      "unclipped_grad_norm": 0.894481897354126
    },
    {
      "bits_per_byte": 1.6150391992838857,
      "learning_rate": 0.0001521842681693461,
      "loss_nats": 1.119459867477417,
      "step": 4430,
      "unclipped_grad_norm": 0.8697190880775452
    },
    {
      "bits_per_byte": 1.63267068866769,
      "learning_rate": 0.00015162885341618752,
      "loss_nats": 1.1316810846328735,
      "step": 4440,
      "unclipped_grad_norm": 0.8085072636604309
    },
    {
      "bits_per_byte": 1.6597337066248108,
      "learning_rate": 0.00015107366713903178,
      "loss_nats": 1.150439739227295,
      "step": 4450,
      "unclipped_grad_norm": 0.8258711099624634
    },
    {
      "bits_per_byte": 1.6374223973283786,
      "learning_rate": 0.0001505187188244806,
      "loss_nats": 1.134974718093872,
      "step": 4460,
      "unclipped_grad_norm": 0.8935436010360718
    },
    {
      "bits_per_byte": 1.6156965169754833,
      "learning_rate": 0.00014996401795506952,
      "loss_nats": 1.11991548538208,
      "step": 4470,
      "unclipped_grad_norm": 0.8515243530273438
    },
    {
      "bits_per_byte": 1.663532975365021,
      "learning_rate": 0.00014940957400910585,
      "loss_nats": 1.1530731916427612,
      "step": 4480,
      "unclipped_grad_norm": 0.8462327718734741
    },
    {
      "bits_per_byte": 1.6896239473794652,
      "learning_rate": 0.00014885539646050699,
      "loss_nats": 1.1711580753326416,
      "step": 4490,
      "unclipped_grad_norm": 0.8624916672706604
    },
    {
      "bits_per_byte": 1.6220770733224936,
      "learning_rate": 0.00014830149477863805,
      "loss_nats": 1.124338150024414,
      "step": 4500,
      "unclipped_grad_norm": 0.830740213394165
    },
    {
      "bits_per_byte": 1.6091536090060587,
      "learning_rate": 0.00014774787842815066,
      "loss_nats": 1.1153802871704102,
      "step": 4510,
      "unclipped_grad_norm": 0.8425999879837036
    },
    {
      "bits_per_byte": 1.6507618876772505,
      "learning_rate": 0.00014719455686882066,
      "loss_nats": 1.1442209482192993,
      "step": 4520,
      "unclipped_grad_norm": 0.8468135595321655
    },
    {
      "bits_per_byte": 1.6238777316770272,
      "learning_rate": 0.00014664153955538695,
      "loss_nats": 1.1255862712860107,
      "step": 4530,
      "unclipped_grad_norm": 0.8532826900482178
    },
    {
      "bits_per_byte": 1.6469891042652731,
      "learning_rate": 0.00014608883593738946,
      "loss_nats": 1.1416058540344238,
      "step": 4540,
      "unclipped_grad_norm": 0.8736191987991333
    },
    {
      "bits_per_byte": 1.6559469926181134,
      "learning_rate": 0.00014553645545900817,
      "loss_nats": 1.1478149890899658,
      "step": 4550,
      "unclipped_grad_norm": 0.8442186713218689
    },
    {
      "bits_per_byte": 1.6446332859137545,
      "learning_rate": 0.0001449844075589013,
      "loss_nats": 1.1399729251861572,
      "step": 4560,
      "unclipped_grad_norm": 0.9025750756263733
    },
    {
      "bits_per_byte": 1.637351540476223,
      "learning_rate": 0.0001444327016700444,
      "loss_nats": 1.1349256038665771,
      "step": 4570,
      "unclipped_grad_norm": 0.8963979482650757
    },
    {
      "bits_per_byte": 1.6074537324849307,
      "learning_rate": 0.0001438813472195688,
      "loss_nats": 1.1142020225524902,
      "step": 4580,
      "unclipped_grad_norm": 0.8524222373962402
    },
    {
      "bits_per_byte": 1.592079171428399,
      "learning_rate": 0.00014333035362860093,
      "loss_nats": 1.1035451889038086,
      "step": 4590,
      "unclipped_grad_norm": 0.8329559564590454
    },
    {
      "bits_per_byte": 1.593515914493707,
      "learning_rate": 0.00014277973031210088,
      "loss_nats": 1.1045410633087158,
      "step": 4600,
      "unclipped_grad_norm": 0.856568455696106
    },
    {
      "bits_per_byte": 1.6099590037600493,
      "learning_rate": 0.00014222948667870207,
      "loss_nats": 1.1159385442733765,
      "step": 4610,
      "unclipped_grad_norm": 0.8378997445106506
    },
    {
      "bits_per_byte": 1.6641679353120065,
      "learning_rate": 0.00014167963213054991,
      "loss_nats": 1.1535133123397827,
      "step": 4620,
      "unclipped_grad_norm": 0.8407614827156067
    },
    {
      "bits_per_byte": 1.64286238055782,
      "learning_rate": 0.00014113017606314152,
      "loss_nats": 1.1387454271316528,
      "step": 4630,
      "unclipped_grad_norm": 0.9177886247634888
    },
    {
      "bits_per_byte": 1.6574955244064822,
      "learning_rate": 0.00014058112786516518,
      "loss_nats": 1.148888349533081,
      "step": 4640,
      "unclipped_grad_norm": 0.8678322434425354
    },
    {
      "bits_per_byte": 1.6864685816840852,
      "learning_rate": 0.0001400324969183396,
      "loss_nats": 1.1689709424972534,
      "step": 4650,
      "unclipped_grad_norm": 0.8395454287528992
    },
    {
      "bits_per_byte": 1.6055636431519484,
      "learning_rate": 0.00013948429259725406,
      "loss_nats": 1.1128919124603271,
      "step": 4660,
      "unclipped_grad_norm": 0.8130927681922913
    },
    {
      "bits_per_byte": 1.6111651180905502,
      "learning_rate": 0.00013893652426920776,
      "loss_nats": 1.116774559020996,
      "step": 4670,
      "unclipped_grad_norm": 0.8343866467475891
    },
    {
      "bits_per_byte": 1.6120890088909852,
      "learning_rate": 0.00013838920129405016,
      "loss_nats": 1.117414951324463,
      "step": 4680,
      "unclipped_grad_norm": 0.9091700911521912
    },
    {
      "bits_per_byte": 1.598018764258597,
      "learning_rate": 0.0001378423330240207,
      "loss_nats": 1.1076622009277344,
      "step": 4690,
      "unclipped_grad_norm": 0.8087259531021118
    },
    {
      "bits_per_byte": 1.6257145063882419,
      "learning_rate": 0.00013729592880358936,
      "loss_nats": 1.126859426498413,
      "step": 4700,
      "unclipped_grad_norm": 0.8473510146141052
    },
    {
      "bits_per_byte": 1.6037934257266175,
      "learning_rate": 0.00013674999796929659,
      "loss_nats": 1.111664891242981,
      "step": 4710,
      "unclipped_grad_norm": 0.8299785256385803
    },
    {
      "bits_per_byte": 1.6360324335441052,
      "learning_rate": 0.00013620454984959417,
      "loss_nats": 1.1340112686157227,
      "step": 4720,
      "unclipped_grad_norm": 0.8852627873420715
    },
    {
      "bits_per_byte": 1.6613310814860245,
      "learning_rate": 0.00013565959376468542,
      "loss_nats": 1.1515469551086426,
      "step": 4730,
      "unclipped_grad_norm": 0.8318192958831787
    },
    {
      "bits_per_byte": 1.6077376758415052,
      "learning_rate": 0.00013511513902636634,
      "loss_nats": 1.1143988370895386,
      "step": 4740,
      "unclipped_grad_norm": 0.9029622673988342
    },
    {
      "bits_per_byte": 1.5803602735986073,
      "learning_rate": 0.0001345711949378662,
      "loss_nats": 1.0954222679138184,
      "step": 4750,
      "unclipped_grad_norm": 0.8787683248519897
    },
    {
      "bits_per_byte": 1.5545300272828713,
      "learning_rate": 0.00013402777079368853,
      "loss_nats": 1.077518105506897,
      "step": 4760,
      "unclipped_grad_norm": 0.8357442021369934
    },
    {
      "bits_per_byte": 1.5563697256991507,
      "learning_rate": 0.0001334848758794528,
      "loss_nats": 1.0787932872772217,
      "step": 4770,
      "unclipped_grad_norm": 0.8196295499801636
    },
    {
      "bits_per_byte": 1.593790398804484,
      "learning_rate": 0.00013294251947173504,
      "loss_nats": 1.1047313213348389,
      "step": 4780,
      "unclipped_grad_norm": 0.9009830951690674
    },
    {
      "bits_per_byte": 1.5906680537780697,
      "learning_rate": 0.00013240071083790996,
      "loss_nats": 1.1025670766830444,
      "step": 4790,
      "unclipped_grad_norm": 0.8538159728050232
    },
    {
      "bits_per_byte": 1.62577985979557,
      "learning_rate": 0.0001318594592359921,
      "loss_nats": 1.1269047260284424,
      "step": 4800,
      "unclipped_grad_norm": 0.9271508455276489
    },
    {
      "bits_per_byte": 1.5952574108163438,
      "learning_rate": 0.0001313187739144781,
      "loss_nats": 1.105748176574707,
      "step": 4810,
      "unclipped_grad_norm": 0.8505629301071167
    },
    {
      "bits_per_byte": 1.5898409892100704,
      "learning_rate": 0.0001307786641121882,
      "loss_nats": 1.1019937992095947,
      "step": 4820,
      "unclipped_grad_norm": 0.8469997048377991
    },
    {
      "bits_per_byte": 1.5923758415011373,
      "learning_rate": 0.00013023913905810876,
      "loss_nats": 1.1037508249282837,
      "step": 4830,
      "unclipped_grad_norm": 0.8425012826919556
    },
    {
      "bits_per_byte": 1.5390662072305417,
      "learning_rate": 0.00012970020797123432,
      "loss_nats": 1.0667994022369385,
      "step": 4840,
      "unclipped_grad_norm": 0.8439065217971802
    },
    {
      "bits_per_byte": 1.6086853002477592,
      "learning_rate": 0.00012916188006041027,
      "loss_nats": 1.1150556802749634,
      "step": 4850,
      "unclipped_grad_norm": 0.8472049236297607
    },
    {
      "bits_per_byte": 1.5891265732783864,
      "learning_rate": 0.00012862416452417516,
      "loss_nats": 1.1014986038208008,
      "step": 4860,
      "unclipped_grad_norm": 0.8380921483039856
    },
    {
      "bits_per_byte": 1.603959560967351,
      "learning_rate": 0.00012808707055060405,
      "loss_nats": 1.111780047416687,
      "step": 4870,
      "unclipped_grad_norm": 0.8809474110603333
    },
    {
      "bits_per_byte": 1.5692639529649388,
      "learning_rate": 0.000127550607317151,
      "loss_nats": 1.087730884552002,
      "step": 4880,
      "unclipped_grad_norm": 0.7983198761940002
    },
    {
      "bits_per_byte": 1.525049621185223,
      "learning_rate": 0.00012701478399049248,
      "loss_nats": 1.0570838451385498,
      "step": 4890,
      "unclipped_grad_norm": 0.874304473400116
    },
    {
      "bits_per_byte": 1.5072996357549742,
      "learning_rate": 0.0001264796097263708,
      "loss_nats": 1.0447804927825928,
      "step": 4900,
      "unclipped_grad_norm": 0.8955963253974915
    },
    {
      "bits_per_byte": 1.5786693401753242,
      "learning_rate": 0.0001259450936694375,
      "loss_nats": 1.094250202178955,
      "step": 4910,
      "unclipped_grad_norm": 0.8447177410125732
    },
    {
      "bits_per_byte": 1.585648052182035,
      "learning_rate": 0.00012541124495309727,
      "loss_nats": 1.0990874767303467,
      "step": 4920,
      "unclipped_grad_norm": 0.8577683568000793
    },
    {
      "bits_per_byte": 1.5737733380705616,
      "learning_rate": 0.00012487807269935167,
      "loss_nats": 1.0908565521240234,
      "step": 4930,
      "unclipped_grad_norm": 0.8645251393318176
    },
    {
      "bits_per_byte": 1.6512401714293001,
      "learning_rate": 0.00012434558601864349,
      "loss_nats": 1.14455246925354,
      "step": 4940,
      "unclipped_grad_norm": 0.9303934574127197
    },
    {
      "bits_per_byte": 1.5141001737353912,
      "learning_rate": 0.0001238137940097009,
      "loss_nats": 1.0494942665100098,
      "step": 4950,
      "unclipped_grad_norm": 0.8813614249229431
    },
    {
      "bits_per_byte": 1.592528046147151,
      "learning_rate": 0.0001232827057593821,
      "loss_nats": 1.1038563251495361,
      "step": 4960,
      "unclipped_grad_norm": 0.915719747543335
    },
    {
      "bits_per_byte": 1.6794828183886887,
      "learning_rate": 0.00012275233034251992,
      "loss_nats": 1.1641287803649902,
      "step": 4970,
      "unclipped_grad_norm": 0.9402682781219482
    },
    {
      "bits_per_byte": 1.543928500735735,
      "learning_rate": 0.0001222226768217669,
      "loss_nats": 1.0701696872711182,
      "step": 4980,
      "unclipped_grad_norm": 0.9095892310142517
    },
    {
      "bits_per_byte": 1.6067152389821258,
      "learning_rate": 0.00012169375424744034,
      "loss_nats": 1.1136901378631592,
      "step": 4990,
      "unclipped_grad_norm": 0.934682309627533
    },
    {
      "bits_per_byte": 1.6283919322968743,
      "learning_rate": 0.00012116557165736772,
      "loss_nats": 1.1287152767181396,
      "step": 5000,
      "unclipped_grad_norm": 0.8894299268722534
    },
    {
      "bits_per_byte": 1.5848010376265358,
      "learning_rate": 0.00012063813807673214,
      "loss_nats": 1.098500370979309,
      "step": 5010,
      "unclipped_grad_norm": 0.8762696385383606
    },
    {
      "bits_per_byte": 1.5972207647585936,
      "learning_rate": 0.00012011146251791832,
      "loss_nats": 1.1071090698242188,
      "step": 5020,
      "unclipped_grad_norm": 0.87384033203125
    },
    {
      "bits_per_byte": 1.5759778116893213,
      "learning_rate": 0.0001195855539803584,
      "loss_nats": 1.0923845767974854,
      "step": 5030,
      "unclipped_grad_norm": 0.875623881816864
    },
    {
      "bits_per_byte": 1.5723771829108575,
      "learning_rate": 0.00011906042145037817,
      "loss_nats": 1.0898888111114502,
      "step": 5040,
      "unclipped_grad_norm": 0.8759424686431885
    },
    {
      "bits_per_byte": 1.6142312247901323,
      "learning_rate": 0.0001185360739010438,
      "loss_nats": 1.1188998222351074,
      "step": 5050,
      "unclipped_grad_norm": 0.886820912361145
    },
    {
      "bits_per_byte": 1.5584469321562717,
      "learning_rate": 0.00011801252029200808,
      "loss_nats": 1.080233097076416,
      "step": 5060,
      "unclipped_grad_norm": 0.880901038646698
    },
    {
      "bits_per_byte": 1.6299523308881525,
      "learning_rate": 0.00011748976956935777,
      "loss_nats": 1.1297968626022339,
      "step": 5070,
      "unclipped_grad_norm": 0.9343949556350708
    },
    {
      "bits_per_byte": 1.651922254622622,
      "learning_rate": 0.00011696783066546024,
      "loss_nats": 1.1450252532958984,
      "step": 5080,
      "unclipped_grad_norm": 0.9880313277244568
    },
    {
      "bits_per_byte": 1.6199948793684975,
      "learning_rate": 0.00011644671249881147,
      "loss_nats": 1.1228948831558228,
      "step": 5090,
      "unclipped_grad_norm": 1.0230180025100708
    },
    {
      "bits_per_byte": 1.5480533326340287,
      "learning_rate": 0.00011592642397388293,
      "loss_nats": 1.073028802871704,
      "step": 5100,
      "unclipped_grad_norm": 0.9155668020248413
    },
    {
      "bits_per_byte": 1.5785750936826513,
      "learning_rate": 0.00011540697398097015,
      "loss_nats": 1.0941848754882812,
      "step": 5110,
      "unclipped_grad_norm": 0.8749560713768005
    },
    {
      "bits_per_byte": 1.553970911684916,
      "learning_rate": 0.00011488837139604016,
      "loss_nats": 1.0771305561065674,
      "step": 5120,
      "unclipped_grad_norm": 0.9114576578140259
    },
    {
      "bits_per_byte": 1.6072428817549729,
      "learning_rate": 0.00011437062508058039,
      "loss_nats": 1.114055871963501,
      "step": 5130,
      "unclipped_grad_norm": 0.89686119556427
    },
    {
      "bits_per_byte": 1.5469013928385495,
      "learning_rate": 0.00011385374388144673,
      "loss_nats": 1.072230339050293,
      "step": 5140,
      "unclipped_grad_norm": 0.9089201092720032
    },
    {
      "bits_per_byte": 1.5509646549478342,
      "learning_rate": 0.00011333773663071286,
      "loss_nats": 1.0750467777252197,
      "step": 5150,
      "unclipped_grad_norm": 0.9353929162025452
    },
    {
      "bits_per_byte": 1.5786652125917033,
      "learning_rate": 0.00011282261214551897,
      "loss_nats": 1.0942473411560059,
      "step": 5160,
      "unclipped_grad_norm": 0.9100342988967896
    },
    {
      "bits_per_byte": 1.5893422395225683,
      "learning_rate": 0.00011230837922792112,
      "loss_nats": 1.1016480922698975,
      "step": 5170,
      "unclipped_grad_norm": 0.8825995922088623
    },
    {
      "bits_per_byte": 1.611850984902191,
      "learning_rate": 0.0001117950466647412,
      "loss_nats": 1.1172499656677246,
      "step": 5180,
      "unclipped_grad_norm": 0.8858869671821594
    },
    {
      "bits_per_byte": 1.5765690880429875,
      "learning_rate": 0.00011128262322741627,
      "loss_nats": 1.092794418334961,
      "step": 5190,
      "unclipped_grad_norm": 0.893822431564331
    },
    {
      "bits_per_byte": 1.5180597303062058,
      "learning_rate": 0.00011077111767184913,
      "loss_nats": 1.0522388219833374,
      "step": 5200,
      "unclipped_grad_norm": 0.9365838766098022
    },
    {
      "bits_per_byte": 1.6136103674205176,
      "learning_rate": 0.00011026053873825837,
      "loss_nats": 1.118469476699829,
      "step": 5210,
      "unclipped_grad_norm": 0.9187290668487549
    },
    {
      "bits_per_byte": 1.578151672396227,
      "learning_rate": 0.0001097508951510293,
      "loss_nats": 1.0938913822174072,
      "step": 5220,
      "unclipped_grad_norm": 0.9324937462806702
    },
    {
      "bits_per_byte": 1.6149896682804372,
      "learning_rate": 0.0001092421956185646,
      "loss_nats": 1.1194255352020264,
      "step": 5230,
      "unclipped_grad_norm": 0.9387596249580383
    },
    {
      "bits_per_byte": 1.5729148006774545,
      "learning_rate": 0.00010873444883313581,
      "loss_nats": 1.090261459350586,
      "step": 5240,
      "unclipped_grad_norm": 0.8688122630119324
    },
    {
      "bits_per_byte": 1.5356076361216906,
      "learning_rate": 0.00010822766347073447,
      "loss_nats": 1.0644021034240723,
      "step": 5250,
      "unclipped_grad_norm": 0.890831708908081
    },
    {
      "bits_per_byte": 1.5141386978491844,
      "learning_rate": 0.00010772184819092424,
      "loss_nats": 1.0495209693908691,
      "step": 5260,
      "unclipped_grad_norm": 0.8596166968345642
    },
    {
      "bits_per_byte": 1.6116535488190005,
      "learning_rate": 0.0001072170116366926,
      "loss_nats": 1.1171131134033203,
      "step": 5270,
      "unclipped_grad_norm": 0.92216956615448
    },
    {
      "bits_per_byte": 1.5994467362087108,
      "learning_rate": 0.00010671316243430345,
      "loss_nats": 1.1086519956588745,
      "step": 5280,
      "unclipped_grad_norm": 0.9085080623626709
    },
    {
      "bits_per_byte": 1.5769201046333985,
      "learning_rate": 0.00010621030919314944,
      "loss_nats": 1.093037724494934,
      "step": 5290,
      "unclipped_grad_norm": 0.8931360244750977
    },
    {
      "bits_per_byte": 1.537008606795619,
      "learning_rate": 0.000105708460505605,
      "loss_nats": 1.065373182296753,
      "step": 5300,
      "unclipped_grad_norm": 0.8922452330589294
    },
    {
      "bits_per_byte": 1.5906943671236518,
      "learning_rate": 0.00010520762494687964,
      "loss_nats": 1.1025853157043457,
      "step": 5310,
      "unclipped_grad_norm": 0.887484073638916
    },
    {
      "bits_per_byte": 1.5623235930893702,
      "learning_rate": 0.00010470781107487111,
      "loss_nats": 1.0829201936721802,
      "step": 5320,
      "unclipped_grad_norm": 0.9026566743850708
    },
    {
      "bits_per_byte": 1.6297041599229574,
      "learning_rate": 0.00010420902743001955,
      "loss_nats": 1.129624843597412,
      "step": 5330,
      "unclipped_grad_norm": 0.9325461387634277
    },
    {
      "bits_per_byte": 1.5387120949524151,
      "learning_rate": 0.00010371128253516117,
      "loss_nats": 1.0665539503097534,
      "step": 5340,
      "unclipped_grad_norm": 0.8923580050468445
    },
    {
      "bits_per_byte": 1.5882112816104945,
      "learning_rate": 0.000103214584895383,
      "loss_nats": 1.1008641719818115,
      "step": 5350,
      "unclipped_grad_norm": 0.9464812278747559
    },
    {
      "bits_per_byte": 1.4963152758270173,
      "learning_rate": 0.00010271894299787715,
      "loss_nats": 1.037166714668274,
      "step": 5360,
      "unclipped_grad_norm": 0.8723886609077454
    },
    {
      "bits_per_byte": 1.5320489711100373,
      "learning_rate": 0.00010222436531179622,
      "loss_nats": 1.0619354248046875,
      "step": 5370,
      "unclipped_grad_norm": 0.9227972626686096
    },
    {
      "bits_per_byte": 1.5933907111238788,
      "learning_rate": 0.0001017308602881082,
      "loss_nats": 1.1044542789459229,
      "step": 5380,
      "unclipped_grad_norm": 0.9233055710792542
    },
    {
      "bits_per_byte": 1.5140970780476757,
      "learning_rate": 0.00010123843635945242,
      "loss_nats": 1.0494921207427979,
      "step": 5390,
      "unclipped_grad_norm": 0.8707529902458191
    },
    {
      "bits_per_byte": 1.5792117734561455,
      "learning_rate": 0.00010074710193999505,
      "loss_nats": 1.0946261882781982,
      "step": 5400,
      "unclipped_grad_norm": 0.9207195043563843
    },
    {
      "bits_per_byte": 1.5488073045754112,
      "learning_rate": 0.0001002568654252858,
      "loss_nats": 1.0735514163970947,
      "step": 5410,
      "unclipped_grad_norm": 0.8852476477622986
    },
    {
      "bits_per_byte": 1.5115954184082252,
      "learning_rate": 9.976773519211404e-05,
      "loss_nats": 1.0477581024169922,
      "step": 5420,
      "unclipped_grad_norm": 0.9259113073348999
    },
    {
      "bits_per_byte": 1.5742989170515984,
      "learning_rate": 9.92797195983659e-05,
      "loss_nats": 1.0912208557128906,
      "step": 5430,
      "unclipped_grad_norm": 0.9123548865318298
    },
    {
      "bits_per_byte": 1.5198012266288425,
      "learning_rate": 9.879282698288142e-05,
      "loss_nats": 1.0534459352493286,
      "step": 5440,
      "unclipped_grad_norm": 0.8873674273490906
    },
    {
      "bits_per_byte": 1.5575636292614403,
      "learning_rate": 9.830706566531197e-05,
      "loss_nats": 1.0796208381652832,
      "step": 5450,
      "unclipped_grad_norm": 0.851129949092865
    },
    {
      "bits_per_byte": 1.5371788696199733,
      "learning_rate": 9.782244394597826e-05,
      "loss_nats": 1.0654911994934082,
      "step": 5460,
      "unclipped_grad_norm": 0.8905500173568726
    },
    {
      "bits_per_byte": 1.5449696837040585,
      "learning_rate": 9.733897010572829e-05,
      "loss_nats": 1.0708913803100586,
      "step": 5470,
      "unclipped_grad_norm": 0.8742530345916748
    },
    {
      "bits_per_byte": 1.5184021477640735,
      "learning_rate": 9.685665240579612e-05,
      "loss_nats": 1.052476167678833,
      "step": 5480,
      "unclipped_grad_norm": 0.9149489998817444
    },
    {
      "bits_per_byte": 1.6054506505503316,
      "learning_rate": 9.637549908766041e-05,
      "loss_nats": 1.1128135919570923,
      "step": 5490,
      "unclipped_grad_norm": 1.0173310041427612
    },
    {
      "bits_per_byte": 1.5652256283400292,
      "learning_rate": 9.589551837290389e-05,
      "loss_nats": 1.08493173122406,
      "step": 5500,
      "unclipped_grad_norm": 0.9233388304710388
    },
    {
      "bits_per_byte": 1.5242242764437322,
      "learning_rate": 9.541671846307257e-05,
      "loss_nats": 1.0565117597579956,
      "step": 5510,
      "unclipped_grad_norm": 0.8623188734054565
    },
    {
      "bits_per_byte": 1.4956853033769069,
      "learning_rate": 9.493910753953597e-05,
      "loss_nats": 1.0367300510406494,
      "step": 5520,
      "unclipped_grad_norm": 0.8464505672454834
    },
    {
      "bits_per_byte": 1.501370017918526,
      "learning_rate": 9.446269376334689e-05,
      "loss_nats": 1.040670394897461,
      "step": 5530,
      "unclipped_grad_norm": 0.9552764296531677
    },
    {
      "bits_per_byte": 1.6303358521995766,
      "learning_rate": 9.39874852751024e-05,
      "loss_nats": 1.1300626993179321,
      "step": 5540,
      "unclipped_grad_norm": 0.930858314037323
    },
    {
      "bits_per_byte": 1.5750513411491232,
      "learning_rate": 9.351349019480433e-05,
      "loss_nats": 1.0917423963546753,
      "step": 5550,
      "unclipped_grad_norm": 0.9512235522270203
    },
    {
      "bits_per_byte": 1.6036597952068972,
      "learning_rate": 9.304071662172086e-05,
      "loss_nats": 1.111572265625,
      "step": 5560,
      "unclipped_grad_norm": 0.9797965884208679
    },
    {
      "bits_per_byte": 1.5364850916063928,
      "learning_rate": 9.256917263424787e-05,
      "loss_nats": 1.0650103092193604,
      "step": 5570,
      "unclipped_grad_norm": 0.8889045715332031
    },
    {
      "bits_per_byte": 1.5818145588943031,
      "learning_rate": 9.209886628977114e-05,
      "loss_nats": 1.0964303016662598,
      "step": 5580,
      "unclipped_grad_norm": 0.91837477684021
    },
    {
      "bits_per_byte": 1.571553042047923,
      "learning_rate": 9.162980562452843e-05,
      "loss_nats": 1.0893175601959229,
      "step": 5590,
      "unclipped_grad_norm": 0.9484079480171204
    },
    {
      "bits_per_byte": 1.5589711352761015,
      "learning_rate": 9.116199865347238e-05,
      "loss_nats": 1.0805964469909668,
      "step": 5600,
      "unclipped_grad_norm": 0.9200659394264221
    },
    {
      "bits_per_byte": 1.4898029807694462,
      "learning_rate": 9.069545337013333e-05,
      "loss_nats": 1.032652735710144,
      "step": 5610,
      "unclipped_grad_norm": 0.898281455039978
    },
    {
      "bits_per_byte": 1.5617056594248202,
      "learning_rate": 9.0230177746483e-05,
      "loss_nats": 1.0824918746948242,
      "step": 5620,
      "unclipped_grad_norm": 0.9515117406845093
    },
    {
      "bits_per_byte": 1.481016903084818,
      "learning_rate": 8.976617973279797e-05,
      "loss_nats": 1.0265626907348633,
      "step": 5630,
      "unclipped_grad_norm": 0.8537728786468506
    },
    {
      "bits_per_byte": 1.471526900410208,
      "learning_rate": 8.930346725752419e-05,
      "loss_nats": 1.0199847221374512,
      "step": 5640,
      "unclipped_grad_norm": 0.851723313331604
    },
    {
      "bits_per_byte": 1.5525594500692852,
      "learning_rate": 8.88420482271411e-05,
      "loss_nats": 1.0761522054672241,
      "step": 5650,
      "unclipped_grad_norm": 0.9591006636619568
    },
    {
      "bits_per_byte": 1.4925528113740916,
      "learning_rate": 8.838193052602691e-05,
      "loss_nats": 1.0345587730407715,
      "step": 5660,
      "unclipped_grad_norm": 0.9846746325492859
    },
    {
      "bits_per_byte": 1.5646704683430441,
      "learning_rate": 8.792312201632352e-05,
      "loss_nats": 1.0845469236373901,
      "step": 5670,
      "unclipped_grad_norm": 0.9337268471717834
    },
    {
      "bits_per_byte": 1.5783896963850212,
      "learning_rate": 8.746563053780261e-05,
      "loss_nats": 1.0940563678741455,
      "step": 5680,
      "unclipped_grad_norm": 1.0177373886108398
    },
    {
      "bits_per_byte": 1.5898387534356093,
      "learning_rate": 8.700946390773114e-05,
      "loss_nats": 1.1019922494888306,
      "step": 5690,
      "unclipped_grad_norm": 0.9374083876609802
    },
    {
      "bits_per_byte": 1.5923674143512452,
      "learning_rate": 8.655462992073833e-05,
      "loss_nats": 1.1037449836730957,
      "step": 5700,
      "unclipped_grad_norm": 0.9426813721656799
    },
    {
      "bits_per_byte": 1.5582474322812707,
      "learning_rate": 8.610113634868199e-05,
      "loss_nats": 1.080094814300537,
      "step": 5710,
      "unclipped_grad_norm": 0.968785285949707
    },
    {
      "bits_per_byte": 1.5819518010496918,
      "learning_rate": 8.564899094051613e-05,
      "loss_nats": 1.0965254306793213,
      "step": 5720,
      "unclipped_grad_norm": 0.9166642427444458
    },
    {
      "bits_per_byte": 1.5197234904706525,
      "learning_rate": 8.519820142215816e-05,
      "loss_nats": 1.0533920526504517,
      "step": 5730,
      "unclipped_grad_norm": 0.9203759431838989
    },
    {
      "bits_per_byte": 1.6015581672133536,
      "learning_rate": 8.474877549635728e-05,
      "loss_nats": 1.1101155281066895,
      "step": 5740,
      "unclipped_grad_norm": 0.934572160243988
    },
    {
      "bits_per_byte": 1.4943809869527633,
      "learning_rate": 8.430072084256247e-05,
      "loss_nats": 1.0358259677886963,
      "step": 5750,
      "unclipped_grad_norm": 0.8915490508079529
    },
    {
      "bits_per_byte": 1.50218005620409,
      "learning_rate": 8.385404511679161e-05,
      "loss_nats": 1.0412318706512451,
      "step": 5760,
      "unclipped_grad_norm": 0.9438080787658691
    },
    {
      "bits_per_byte": 1.5178872317073904,
      "learning_rate": 8.340875595150034e-05,
      "loss_nats": 1.052119255065918,
      "step": 5770,
      "unclipped_grad_norm": 0.9708113670349121
    },
    {
      "bits_per_byte": 1.6077987296825615,
      "learning_rate": 8.296486095545203e-05,
      "loss_nats": 1.114441156387329,
      "step": 5780,
      "unclipped_grad_norm": 1.0216169357299805
    },
    {
      "bits_per_byte": 1.454466909375221,
      "learning_rate": 8.252236771358729e-05,
      "loss_nats": 1.0081596374511719,
      "step": 5790,
      "unclipped_grad_norm": 0.8829967975616455
    },
    {
      "bits_per_byte": 1.5674770532224742,
      "learning_rate": 8.208128378689484e-05,
      "loss_nats": 1.0864923000335693,
      "step": 5800,
      "unclipped_grad_norm": 0.9498817920684814
    },
    {
      "bits_per_byte": 1.4688064788388602,
      "learning_rate": 8.16416167122819e-05,
      "loss_nats": 1.018099069595337,
      "step": 5810,
      "unclipped_grad_norm": 0.9071933031082153
    },
    {
      "bits_per_byte": 1.4812057400354652,
      "learning_rate": 8.120337400244578e-05,
      "loss_nats": 1.02669358253479,
      "step": 5820,
      "unclipped_grad_norm": 0.9959194660186768
    },
    {
      "bits_per_byte": 1.5459386339590195,
      "learning_rate": 8.076656314574521e-05,
      "loss_nats": 1.0715630054473877,
      "step": 5830,
      "unclipped_grad_norm": 0.9805806875228882
    },
    {
      "bits_per_byte": 1.5386066695874363,
      "learning_rate": 8.03311916060725e-05,
      "loss_nats": 1.0664808750152588,
      "step": 5840,
      "unclipped_grad_norm": 0.9813235998153687
    },
    {
      "bits_per_byte": 1.4904921152514534,
      "learning_rate": 7.989726682272606e-05,
      "loss_nats": 1.033130407333374,
      "step": 5850,
      "unclipped_grad_norm": 0.8884730339050293
    },
    {
      "bits_per_byte": 1.505957139182339,
      "learning_rate": 7.946479621028318e-05,
      "loss_nats": 1.0438499450683594,
      "step": 5860,
      "unclipped_grad_norm": 0.9569032192230225
    },
    {
      "bits_per_byte": 1.5707541826346652,
      "learning_rate": 7.903378715847343e-05,
      "loss_nats": 1.0887638330459595,
      "step": 5870,
      "unclipped_grad_norm": 0.9548988342285156
    },
    {
      "bits_per_byte": 1.5056532458382643,
      "learning_rate": 7.860424703205224e-05,
      "loss_nats": 1.0436393022537231,
      "step": 5880,
      "unclipped_grad_norm": 0.9100088477134705
    },
    {
      "bits_per_byte": 1.5621944341185723,
      "learning_rate": 7.817618317067534e-05,
      "loss_nats": 1.0828306674957275,
      "step": 5890,
      "unclipped_grad_norm": 1.0087047815322876
    },
    {
      "bits_per_byte": 1.564086587243365,
      "learning_rate": 7.774960288877295e-05,
      "loss_nats": 1.0841422080993652,
      "step": 5900,
      "unclipped_grad_norm": 0.9735300540924072
    },
    {
      "bits_per_byte": 1.5205851235514751,
      "learning_rate": 7.732451347542522e-05,
      "loss_nats": 1.053989291191101,
      "step": 5910,
      "unclipped_grad_norm": 0.928986668586731
    },
    {
      "bits_per_byte": 1.4827504882055151,
      "learning_rate": 7.690092219423722e-05,
      "loss_nats": 1.0277643203735352,
      "step": 5920,
      "unclipped_grad_norm": 0.9661484956741333
    },
    {
      "bits_per_byte": 1.5936927126587939,
      "learning_rate": 7.647883628321536e-05,
      "loss_nats": 1.104663610458374,
      "step": 5930,
      "unclipped_grad_norm": 0.9932063817977905
    },
    {
      "bits_per_byte": 1.5284586612906255,
      "learning_rate": 7.605826295464316e-05,
      "loss_nats": 1.0594468116760254,
      "step": 5940,
      "unclipped_grad_norm": 1.0194250345230103
    },
    {
      "bits_per_byte": 1.5388254315193337,
      "learning_rate": 7.563920939495851e-05,
      "loss_nats": 1.0666325092315674,
      "step": 5950,
      "unclipped_grad_norm": 0.9446147084236145
    },
    {
      "bits_per_byte": 1.4754302186541906,
      "learning_rate": 7.522168276463047e-05,
      "loss_nats": 1.0226902961730957,
      "step": 5960,
      "unclipped_grad_norm": 0.9241304993629456
    },
    {
      "bits_per_byte": 1.5277755462013984,
      "learning_rate": 7.480569019803719e-05,
      "loss_nats": 1.0589733123779297,
      "step": 5970,
      "unclipped_grad_norm": 0.9580891728401184
    },
    {
      "bits_per_byte": 1.5012076662961116,
      "learning_rate": 7.439123880334388e-05,
      "loss_nats": 1.040557861328125,
      "step": 5980,
      "unclipped_grad_norm": 0.954113245010376
    },
    {
      "bits_per_byte": 1.4896813890352862,
      "learning_rate": 7.39783356623813e-05,
      "loss_nats": 1.0325684547424316,
      "step": 5990,
      "unclipped_grad_norm": 0.8945097923278809
    },
    {
      "bits_per_byte": 1.5812009247960246,
      "learning_rate": 7.356698783052497e-05,
      "loss_nats": 1.0960049629211426,
      "step": 6000,
      "unclipped_grad_norm": 1.0032626390457153
    },
    {
      "bits_per_byte": 1.556720054358958,
      "learning_rate": 7.315720233657431e-05,
      "loss_nats": 1.0790361166000366,
      "step": 6010,
      "unclipped_grad_norm": 0.9715147614479065
    },
    {
      "bits_per_byte": 1.5090167105411882,
      "learning_rate": 7.274898618263286e-05,
      "loss_nats": 1.0459706783294678,
      "step": 6020,
      "unclipped_grad_norm": 0.9192096590995789
    },
    {
      "bits_per_byte": 1.4969094758857484,
      "learning_rate": 7.234234634398831e-05,
      "loss_nats": 1.0375785827636719,
      "step": 6030,
      "unclipped_grad_norm": 0.9740291833877563
    },
    {
      "bits_per_byte": 1.5263979651679873,
      "learning_rate": 7.193728976899362e-05,
      "loss_nats": 1.058018445968628,
      "step": 6040,
      "unclipped_grad_norm": 0.9185873866081238
    },
    {
      "bits_per_byte": 1.5301623214300724,
      "learning_rate": 7.153382337894797e-05,
      "loss_nats": 1.0606276988983154,
      "step": 6050,
      "unclipped_grad_norm": 0.9192250370979309
    },
    {
      "bits_per_byte": 1.5237477125181913,
      "learning_rate": 7.113195406797887e-05,
      "loss_nats": 1.0561814308166504,
      "step": 6060,
      "unclipped_grad_norm": 0.9670439958572388
    },
    {
      "bits_per_byte": 1.4068365721927425,
      "learning_rate": 7.073168870292397e-05,
      "loss_nats": 0.9751448035240173,
      "step": 6070,
      "unclipped_grad_norm": 0.917002260684967
    },
    {
      "bits_per_byte": 1.4967151354902735,
      "learning_rate": 7.033303412321404e-05,
      "loss_nats": 1.0374438762664795,
      "step": 6080,
      "unclipped_grad_norm": 0.9365084171295166
    },
    {
      "bits_per_byte": 1.4969400887976019,
      "learning_rate": 6.993599714075586e-05,
      "loss_nats": 1.037599802017212,
      "step": 6090,
      "unclipped_grad_norm": 0.9915803074836731
    },
    {
      "bits_per_byte": 1.5907201645212812,
      "learning_rate": 6.954058453981609e-05,
      "loss_nats": 1.1026031970977783,
      "step": 6100,
      "unclipped_grad_norm": 1.0093804597854614
    },
    {
      "bits_per_byte": 1.5313995646203795,
      "learning_rate": 6.914680307690504e-05,
      "loss_nats": 1.0614852905273438,
      "step": 6110,
      "unclipped_grad_norm": 1.0098049640655518
    },
    {
      "bits_per_byte": 1.4730523865233613,
      "learning_rate": 6.875465948066139e-05,
      "loss_nats": 1.0210421085357666,
      "step": 6120,
      "unclipped_grad_norm": 0.9365134239196777
    },
    {
      "bits_per_byte": 1.5512489422697104,
      "learning_rate": 6.836416045173733e-05,
      "loss_nats": 1.0752438306808472,
      "step": 6130,
      "unclipped_grad_norm": 0.9237281680107117
    },
    {
      "bits_per_byte": 1.497698360305256,
      "learning_rate": 6.797531266268377e-05,
      "loss_nats": 1.0381253957748413,
      "step": 6140,
      "unclipped_grad_norm": 0.9138990044593811
    },
    {
      "bits_per_byte": 1.4838755987074594,
      "learning_rate": 6.758812275783663e-05,
      "loss_nats": 1.0285441875457764,
      "step": 6150,
      "unclipped_grad_norm": 0.9429982900619507
    },
    {
      "bits_per_byte": 1.551023816979731,
      "learning_rate": 6.720259735320298e-05,
      "loss_nats": 1.0750877857208252,
      "step": 6160,
      "unclipped_grad_norm": 1.0046567916870117
    },
    {
      "bits_per_byte": 1.4928049379402566,
      "learning_rate": 6.681874303634842e-05,
      "loss_nats": 1.034733533859253,
      "step": 6170,
      "unclipped_grad_norm": 0.964278519153595
    },
    {
      "bits_per_byte": 1.4789379768011883,
      "learning_rate": 6.643656636628402e-05,
      "loss_nats": 1.0251216888427734,
      "step": 6180,
      "unclipped_grad_norm": 0.9606515169143677
    },
    {
      "bits_per_byte": 1.501445346319604,
      "learning_rate": 6.605607387335466e-05,
      "loss_nats": 1.0407226085662842,
      "step": 6190,
      "unclipped_grad_norm": 1.0252128839492798
    },
    {
      "bits_per_byte": 1.4915814533620186,
      "learning_rate": 6.567727205912724e-05,
      "loss_nats": 1.0338854789733887,
      "step": 6200,
      "unclipped_grad_norm": 0.8806238770484924
    },
    {
      "bits_per_byte": 1.5186611536362729,
      "learning_rate": 6.530016739627959e-05,
      "loss_nats": 1.0526556968688965,
      "step": 6210,
      "unclipped_grad_norm": 0.989686906337738
    },
    {
      "bits_per_byte": 1.539449556559315,
      "learning_rate": 6.492476632848993e-05,
      "loss_nats": 1.0670651197433472,
      "step": 6220,
      "unclipped_grad_norm": 0.9960981011390686
    },
    {
      "bits_per_byte": 1.4808485320696232,
      "learning_rate": 6.455107527032678e-05,
      "loss_nats": 1.026445984840393,
      "step": 6230,
      "unclipped_grad_norm": 0.9705929160118103
    },
    {
      "bits_per_byte": 1.552468127281677,
      "learning_rate": 6.417910060713926e-05,
      "loss_nats": 1.0760889053344727,
      "step": 6240,
      "unclipped_grad_norm": 1.0799938440322876
    },
    {
      "bits_per_byte": 1.4757860507588256,
      "learning_rate": 6.380884869494799e-05,
      "loss_nats": 1.0229369401931763,
      "step": 6250,
      "unclipped_grad_norm": 0.9792200922966003
    },
    {
      "bits_per_byte": 1.445374530589406,
      "learning_rate": 6.34403258603367e-05,
      "loss_nats": 1.0018572807312012,
      "step": 6260,
      "unclipped_grad_norm": 0.9422674179077148
    },
    {
      "bits_per_byte": 1.5013631386124917,
      "learning_rate": 6.30735384003437e-05,
      "loss_nats": 1.040665626525879,
      "step": 6270,
      "unclipped_grad_norm": 0.9675420522689819
    },
    {
      "bits_per_byte": 1.5119997496204038,
      "learning_rate": 6.270849258235484e-05,
      "loss_nats": 1.048038363456726,
      "step": 6280,
      "unclipped_grad_norm": 0.9240654110908508
    },
    {
      "bits_per_byte": 1.5072342823476461,
      "learning_rate": 6.234519464399581e-05,
      "loss_nats": 1.0447351932525635,
      "step": 6290,
      "unclipped_grad_norm": 0.9724128842353821
    },
    {
      "bits_per_byte": 1.4760084243263911,
      "learning_rate": 6.198365079302614e-05,
      "loss_nats": 1.0230910778045654,
      "step": 6300,
      "unclipped_grad_norm": 0.9724760055541992
    },
    {
      "bits_per_byte": 1.539534687971492,
      "learning_rate": 6.16238672072326e-05,
      "loss_nats": 1.0671241283416748,
      "step": 6310,
      "unclipped_grad_norm": 1.0172570943832397
    },
    {
      "bits_per_byte": 1.498671266161187,
      "learning_rate": 6.126585003432406e-05,
      "loss_nats": 1.03879976272583,
      "step": 6320,
      "unclipped_grad_norm": 0.9810369610786438
    },
    {
      "bits_per_byte": 1.5103010769778318,
      "learning_rate": 6.090960539182614e-05,
      "loss_nats": 1.046860933303833,
      "step": 6330,
      "unclipped_grad_norm": 0.9735086560249329
    },
    {
      "bits_per_byte": 1.4197871237766881,
      "learning_rate": 6.0555139366976935e-05,
      "loss_nats": 0.9841214418411255,
      "step": 6340,
      "unclipped_grad_norm": 0.9049187302589417
    },
    {
      "bits_per_byte": 1.5345103868091858,
      "learning_rate": 6.020245801662274e-05,
      "loss_nats": 1.0636415481567383,
      "step": 6350,
      "unclipped_grad_norm": 0.9579975008964539
    },
    {
      "bits_per_byte": 1.5509206273892133,
      "learning_rate": 5.985156736711483e-05,
      "loss_nats": 1.0750162601470947,
      "step": 6360,
      "unclipped_grad_norm": 0.989890456199646
    },
    {
      "bits_per_byte": 1.4729708667468522,
      "learning_rate": 5.950247341420618e-05,
      "loss_nats": 1.0209856033325195,
      "step": 6370,
      "unclipped_grad_norm": 1.021422266960144
    },
    {
      "bits_per_byte": 1.4625091620948676,
      "learning_rate": 5.915518212294935e-05,
      "loss_nats": 1.0137341022491455,
      "step": 6380,
      "unclipped_grad_norm": 1.0111291408538818
    },
    {
      "bits_per_byte": 1.576993369242666,
      "learning_rate": 5.880969942759425e-05,
      "loss_nats": 1.0930885076522827,
      "step": 6390,
      "unclipped_grad_norm": 1.0084502696990967
    },
    {
      "bits_per_byte": 1.5426105976821733,
      "learning_rate": 5.846603123148688e-05,
      "loss_nats": 1.0692561864852905,
      "step": 6400,
      "unclipped_grad_norm": 1.060890793800354
    },
    {
      "bits_per_byte": 1.542634847235945,
      "learning_rate": 5.812418340696858e-05,
      "loss_nats": 1.0692729949951172,
      "step": 6410,
      "unclipped_grad_norm": 1.0786292552947998
    },
    {
      "bits_per_byte": 1.4677484415707522,
      "learning_rate": 5.778416179527541e-05,
      "loss_nats": 1.0173656940460205,
      "step": 6420,
      "unclipped_grad_norm": 1.0185354948043823
    },
    {
      "bits_per_byte": 1.5249625979638863,
      "learning_rate": 5.7445972206438635e-05,
      "loss_nats": 1.057023525238037,
      "step": 6430,
      "unclipped_grad_norm": 0.9899823069572449
    },
    {
      "bits_per_byte": 1.447091949340922,
      "learning_rate": 5.710962041918516e-05,
      "loss_nats": 1.0030477046966553,
      "step": 6440,
      "unclipped_grad_norm": 0.9853183031082153
    },
    {
      "bits_per_byte": 1.4600725118974434,
      "learning_rate": 5.677511218083908e-05,
      "loss_nats": 1.01204514503479,
      "step": 6450,
      "unclipped_grad_norm": 0.9329874515533447
    },
    {
      "bits_per_byte": 1.4881399085356029,
      "learning_rate": 5.644245320722316e-05,
      "loss_nats": 1.031499981880188,
      "step": 6460,
      "unclipped_grad_norm": 0.9553210735321045
    },
    {
      "bits_per_byte": 1.5093300629310604,
      "learning_rate": 5.61116491825615e-05,
      "loss_nats": 1.0461878776550293,
      "step": 6470,
      "unclipped_grad_norm": 1.0880736112594604
    },
    {
      "bits_per_byte": 1.4620568477230984,
      "learning_rate": 5.5782705759382104e-05,
      "loss_nats": 1.013420581817627,
      "step": 6480,
      "unclipped_grad_norm": 0.978254497051239
    },
    {
      "bits_per_byte": 1.5301041912940807,
      "learning_rate": 5.5455628558420546e-05,
      "loss_nats": 1.0605874061584473,
      "step": 6490,
      "unclipped_grad_norm": 1.0214275121688843
    },
    {
      "bits_per_byte": 1.5703515712489953,
      "learning_rate": 5.513042316852368e-05,
      "loss_nats": 1.088484764099121,
      "step": 6500,
      "unclipped_grad_norm": 1.024044156074524
    },
    {
      "bits_per_byte": 1.5418132861127734,
      "learning_rate": 5.480709514655443e-05,
      "loss_nats": 1.068703532218933,
      "step": 6510,
      "unclipped_grad_norm": 1.0472792387008667
    },
    {
      "bits_per_byte": 1.5326505664227554,
      "learning_rate": 5.448565001729653e-05,
      "loss_nats": 1.0623524188995361,
      "step": 6520,
      "unclipped_grad_norm": 0.9873340129852295
    },
    {
      "bits_per_byte": 1.493115194642413,
      "learning_rate": 5.41660932733604e-05,
      "loss_nats": 1.0349485874176025,
      "step": 6530,
      "unclipped_grad_norm": 0.9528921246528625
    },
    {
      "bits_per_byte": 1.4717057623671055,
      "learning_rate": 5.384843037508902e-05,
      "loss_nats": 1.020108699798584,
      "step": 6540,
      "unclipped_grad_norm": 0.9596993327140808
    },
    {
      "bits_per_byte": 1.532975785615537,
      "learning_rate": 5.353266675046498e-05,
      "loss_nats": 1.0625778436660767,
      "step": 6550,
      "unclipped_grad_norm": 0.9932572245597839
    },
    {
      "bits_per_byte": 1.5382425823155597,
      "learning_rate": 5.321880779501729e-05,
      "loss_nats": 1.0662285089492798,
      "step": 6560,
      "unclipped_grad_norm": 1.0132641792297363
    },
    {
      "bits_per_byte": 1.5160394501065202,
      "learning_rate": 5.290685887172962e-05,
      "loss_nats": 1.0508384704589844,
      "step": 6570,
      "unclipped_grad_norm": 0.9644629955291748
    },
    {
      "bits_per_byte": 1.503854823258192,
      "learning_rate": 5.2596825310948295e-05,
      "loss_nats": 1.0423927307128906,
      "step": 6580,
      "unclipped_grad_norm": 0.9727261662483215
    },
    {
      "bits_per_byte": 1.5231672710715294,
      "learning_rate": 5.228871241029154e-05,
      "loss_nats": 1.0557790994644165,
      "step": 6590,
      "unclipped_grad_norm": 1.029261589050293
    },
    {
      "bits_per_byte": 1.415425385776831,
      "learning_rate": 5.198252543455865e-05,
      "loss_nats": 0.9810981154441833,
      "step": 6600,
      "unclipped_grad_norm": 0.9430753588676453
    },
    {
      "bits_per_byte": 1.4673490978554489,
      "learning_rate": 5.167826961564027e-05,
      "loss_nats": 1.0170888900756836,
      "step": 6610,
      "unclipped_grad_norm": 0.9699459671974182
    },
    {
      "bits_per_byte": 1.5770974187464382,
      "learning_rate": 5.137595015242884e-05,
      "loss_nats": 1.093160629272461,
      "step": 6620,
      "unclipped_grad_norm": 1.053364872932434
    },
    {
      "bits_per_byte": 1.520643081704816,
      "learning_rate": 5.1075572210729925e-05,
      "loss_nats": 1.0540294647216797,
      "step": 6630,
      "unclipped_grad_norm": 1.0116336345672607
    },
    {
      "bits_per_byte": 1.4843385760035823,
      "learning_rate": 5.07771409231737e-05,
      "loss_nats": 1.028865098953247,
      "step": 6640,
      "unclipped_grad_norm": 0.9483272433280945
    },
    {
      "bits_per_byte": 1.481005552229861,
      "learning_rate": 5.048066138912756e-05,
      "loss_nats": 1.026554822921753,
      "step": 6650,
      "unclipped_grad_norm": 0.9847526550292969
    },
    {
      "bits_per_byte": 1.5181830418668742,
      "learning_rate": 5.018613867460863e-05,
      "loss_nats": 1.0523242950439453,
      "step": 6660,
      "unclipped_grad_norm": 1.0222764015197754
    },
    {
      "bits_per_byte": 1.483456133022005,
      "learning_rate": 4.989357781219755e-05,
      "loss_nats": 1.028253436088562,
      "step": 6670,
      "unclipped_grad_norm": 0.9609436392784119
    },
    {
      "bits_per_byte": 1.5070292790278177,
      "learning_rate": 4.960298380095222e-05,
      "loss_nats": 1.044593095779419,
      "step": 6680,
      "unclipped_grad_norm": 1.0481340885162354
    },
    {
      "bits_per_byte": 1.4999456576040804,
      "learning_rate": 4.931436160632256e-05,
      "loss_nats": 1.0396831035614014,
      "step": 6690,
      "unclipped_grad_norm": 1.0560411214828491
    },
    {
      "bits_per_byte": 1.5028304945896531,
      "learning_rate": 4.9027716160065485e-05,
      "loss_nats": 1.0416827201843262,
      "step": 6700,
      "unclipped_grad_norm": 1.0250314474105835
    },
    {
      "bits_per_byte": 1.5313854620430087,
      "learning_rate": 4.874305236016084e-05,
      "loss_nats": 1.0614755153656006,
      "step": 6710,
      "unclipped_grad_norm": 0.9873788356781006
    },
    {
      "bits_per_byte": 1.5848861690387128,
      "learning_rate": 4.846037507072752e-05,
      "loss_nats": 1.0985593795776367,
      "step": 6720,
      "unclipped_grad_norm": 1.065596103668213
    },
    {
      "bits_per_byte": 1.4535899698384716,
      "learning_rate": 4.817968912194049e-05,
      "loss_nats": 1.0075517892837524,
      "step": 6730,
      "unclipped_grad_norm": 0.9767746329307556
    },
    {
      "bits_per_byte": 1.4828168735087481,
      "learning_rate": 4.790099930994813e-05,
      "loss_nats": 1.0278103351593018,
      "step": 6740,
      "unclipped_grad_norm": 0.9589064717292786
    },
    {
      "bits_per_byte": 1.4599156637198565,
      "learning_rate": 4.762431039679047e-05,
      "loss_nats": 1.0119364261627197,
      "step": 6750,
      "unclipped_grad_norm": 0.9927419424057007
    },
    {
      "bits_per_byte": 1.4484303183299363,
      "learning_rate": 4.734962711031755e-05,
      "loss_nats": 1.0039753913879395,
      "step": 6760,
      "unclipped_grad_norm": 0.9825301766395569
    },
    {
      "bits_per_byte": 1.469467236183475,
      "learning_rate": 4.707695414410892e-05,
      "loss_nats": 1.018557071685791,
      "step": 6770,
      "unclipped_grad_norm": 1.0425026416778564
    },
    {
      "bits_per_byte": 1.4605650702095145,
      "learning_rate": 4.680629615739322e-05,
      "loss_nats": 1.0123865604400635,
      "step": 6780,
      "unclipped_grad_norm": 1.0209197998046875
    },
    {
      "bits_per_byte": 1.494792713418929,
      "learning_rate": 4.653765777496863e-05,
      "loss_nats": 1.0361113548278809,
      "step": 6790,
      "unclipped_grad_norm": 0.9975507855415344
    },
    {
      "bits_per_byte": 1.4806505800384802,
      "learning_rate": 4.6271043587123986e-05,
      "loss_nats": 1.0263087749481201,
      "step": 6800,
      "unclipped_grad_norm": 1.0428087711334229
    },
    {
      "bits_per_byte": 1.4777557680591573,
      "learning_rate": 4.600645814956006e-05,
      "loss_nats": 1.0243022441864014,
      "step": 6810,
      "unclipped_grad_norm": 0.9905012249946594
    },
    {
      "bits_per_byte": 1.4914748241184836,
      "learning_rate": 4.574390598331204e-05,
      "loss_nats": 1.0338115692138672,
      "step": 6820,
      "unclipped_grad_norm": 1.0114696025848389
    },
    {
      "bits_per_byte": 1.4751165222990168,
      "learning_rate": 4.548339157467199e-05,
      "loss_nats": 1.022472858428955,
      "step": 6830,
      "unclipped_grad_norm": 0.9909161329269409
    },
    {
      "bits_per_byte": 1.462379831141419,
      "learning_rate": 4.52249193751124e-05,
      "loss_nats": 1.0136444568634033,
      "step": 6840,
      "unclipped_grad_norm": 1.0066962242126465
    },
    {
      "bits_per_byte": 1.542950607382929,
      "learning_rate": 4.4968493801209985e-05,
      "loss_nats": 1.0694918632507324,
      "step": 6850,
      "unclipped_grad_norm": 1.0904512405395508
    },
    {
      "bits_per_byte": 1.5094851912821385,
      "learning_rate": 4.471411923457034e-05,
      "loss_nats": 1.046295404434204,
      "step": 6860,
      "unclipped_grad_norm": 1.0577117204666138
    },
    {
      "bits_per_byte": 1.4431473552607326,
      "learning_rate": 4.446180002175291e-05,
      "loss_nats": 1.0003135204315186,
      "step": 6870,
      "unclipped_grad_norm": 0.931652843952179
    },
    {
      "bits_per_byte": 1.4721159409894133,
      "learning_rate": 4.421154047419693e-05,
      "loss_nats": 1.0203930139541626,
      "step": 6880,
      "unclipped_grad_norm": 1.0439471006393433
    },
    {
      "bits_per_byte": 1.4601951355275085,
      "learning_rate": 4.396334486814753e-05,
      "loss_nats": 1.0121301412582397,
      "step": 6890,
      "unclipped_grad_norm": 0.9945755004882812
    },
    {
      "bits_per_byte": 1.5078214311176918,
      "learning_rate": 4.371721744458285e-05,
      "loss_nats": 1.0451421737670898,
      "step": 6900,
      "unclipped_grad_norm": 1.0577389001846313
    },
    {
      "bits_per_byte": 1.51248938422741,
      "learning_rate": 4.347316240914145e-05,
      "loss_nats": 1.0483777523040771,
      "step": 6910,
      "unclipped_grad_norm": 1.042457938194275
    },
    {
      "bits_per_byte": 1.4546720846777004,
      "learning_rate": 4.3231183932050546e-05,
      "loss_nats": 1.008301854133606,
      "step": 6920,
      "unclipped_grad_norm": 1.0035841464996338
    },
    {
      "bits_per_byte": 1.4388924184870613,
      "learning_rate": 4.299128614805464e-05,
      "loss_nats": 0.9973642230033875,
      "step": 6930,
      "unclipped_grad_norm": 1.0526496171951294
    },
    {
      "bits_per_byte": 1.5114688391771902,
      "learning_rate": 4.2753473156344965e-05,
      "loss_nats": 1.0476703643798828,
      "step": 6940,
      "unclipped_grad_norm": 1.0117003917694092
    },
    {
      "bits_per_byte": 1.4256293744264978,
      "learning_rate": 4.251774902048942e-05,
      "loss_nats": 0.9881709814071655,
      "step": 6950,
      "unclipped_grad_norm": 1.020249843597412
    },
    {
      "bits_per_byte": 1.4392803253560824,
      "learning_rate": 4.228411776836306e-05,
      "loss_nats": 0.9976330995559692,
      "step": 6960,
      "unclipped_grad_norm": 1.007938265800476
    },
    {
      "bits_per_byte": 1.425250496646647,
      "learning_rate": 4.2052583392079427e-05,
      "loss_nats": 0.9879083633422852,
      "step": 6970,
      "unclipped_grad_norm": 1.0789695978164673
    },
    {
      "bits_per_byte": 1.5310960152416067,
      "learning_rate": 4.182314984792214e-05,
      "loss_nats": 1.0612748861312866,
      "step": 6980,
      "unclipped_grad_norm": 1.0816211700439453
    },
    {
      "bits_per_byte": 1.4834984407541174,
      "learning_rate": 4.159582105627748e-05,
      "loss_nats": 1.0282827615737915,
      "step": 6990,
      "unclipped_grad_norm": 1.0542293787002563
    },
    {
      "bits_per_byte": 1.494992041311279,
      "learning_rate": 4.137060090156724e-05,
      "loss_nats": 1.0362495183944702,
      "step": 7000,
      "unclipped_grad_norm": 1.040958046913147
    },
    {
      "bits_per_byte": 1.4039343649594325,
      "learning_rate": 4.114749323218249e-05,
      "loss_nats": 0.9731331467628479,
      "step": 7010,
      "unclipped_grad_norm": 1.0020650625228882
    },
    {
      "bits_per_byte": 1.4906596263533938,
      "learning_rate": 4.0926501860417706e-05,
      "loss_nats": 1.0332465171813965,
      "step": 7020,
      "unclipped_grad_norm": 1.0739529132843018
    },
    {
      "bits_per_byte": 1.4642052549976767,
      "learning_rate": 4.070763056240571e-05,
      "loss_nats": 1.0149097442626953,
      "step": 7030,
      "unclipped_grad_norm": 1.0350828170776367
    },
    {
      "bits_per_byte": 1.4574465088014192,
      "learning_rate": 4.049088307805306e-05,
      "loss_nats": 1.0102249383926392,
      "step": 7040,
      "unclipped_grad_norm": 1.0089728832244873
    },
    {
      "bits_per_byte": 1.4534154074478458,
      "learning_rate": 4.027626311097629e-05,
      "loss_nats": 1.0074307918548584,
      "step": 7050,
      "unclipped_grad_norm": 1.0608088970184326
    },
    {
      "bits_per_byte": 1.4416395833606184,
      "learning_rate": 4.006377432843846e-05,
      "loss_nats": 0.9992684125900269,
      "step": 7060,
      "unclipped_grad_norm": 0.9864199757575989
    },
    {
      "bits_per_byte": 1.4906718371216052,
      "learning_rate": 3.985342036128656e-05,
      "loss_nats": 1.0332549810409546,
      "step": 7070,
      "unclipped_grad_norm": 1.0150113105773926
    },
    {
      "bits_per_byte": 1.4931912109740944,
      "learning_rate": 3.964520480388956e-05,
      "loss_nats": 1.035001277923584,
      "step": 7080,
      "unclipped_grad_norm": 1.0090219974517822
    },
    {
      "bits_per_byte": 1.4509221749582877,
      "learning_rate": 3.943913121407678e-05,
      "loss_nats": 1.0057026147842407,
      "step": 7090,
      "unclipped_grad_norm": 1.0420514345169067
    },
    {
      "bits_per_byte": 1.4389093587781714,
      "learning_rate": 3.923520311307735e-05,
      "loss_nats": 0.9973759651184082,
      "step": 7100,
      "unclipped_grad_norm": 1.0253499746322632
    },
    {
      "bits_per_byte": 1.41304419998431,
      "learning_rate": 3.9033423985459816e-05,
      "loss_nats": 0.979447603225708,
      "step": 7110,
      "unclipped_grad_norm": 0.9803867936134338
    },
    {
      "bits_per_byte": 1.4722433801337027,
      "learning_rate": 3.88337972790728e-05,
      "loss_nats": 1.0204813480377197,
      "step": 7120,
      "unclipped_grad_norm": 1.0725315809249878
    },
    {
      "bits_per_byte": 1.5197811046586915,
      "learning_rate": 3.8636326404985845e-05,
      "loss_nats": 1.0534319877624512,
      "step": 7130,
      "unclipped_grad_norm": 1.0701161623001099
    },
    {
      "bits_per_byte": 1.465326925846604,
      "learning_rate": 3.844101473743143e-05,
      "loss_nats": 1.0156872272491455,
      "step": 7140,
      "unclipped_grad_norm": 1.0326207876205444
    },
    {
      "bits_per_byte": 1.5100518741167317,
      "learning_rate": 3.824786561374701e-05,
      "loss_nats": 1.046688199043274,
      "step": 7150,
      "unclipped_grad_norm": 1.0673091411590576
    },
    {
      "bits_per_byte": 1.5022746466620645,
      "learning_rate": 3.805688233431824e-05,
      "loss_nats": 1.041297435760498,
      "step": 7160,
      "unclipped_grad_norm": 1.059691071510315
    },
    {
      "bits_per_byte": 1.514605802728928,
      "learning_rate": 3.786806816252236e-05,
      "loss_nats": 1.049844741821289,
      "step": 7170,
      "unclipped_grad_norm": 1.085069179534912
    },
    {
      "bits_per_byte": 1.4489440305080636,
      "learning_rate": 3.768142632467269e-05,
      "loss_nats": 1.0043314695358276,
      "step": 7180,
      "unclipped_grad_norm": 0.9880333542823792
    },
    {
      "bits_per_byte": 1.5073161460894569,
      "learning_rate": 3.7496960009963236e-05,
      "loss_nats": 1.0447919368743896,
      "step": 7190,
      "unclipped_grad_norm": 1.0678606033325195
    },
    {
      "bits_per_byte": 1.4434393818018978,
      "learning_rate": 3.7314672370414325e-05,
      "loss_nats": 1.0005159378051758,
      "step": 7200,
      "unclipped_grad_norm": 1.0179466009140015
    },
    {
      "bits_per_byte": 1.4303239848470999,
      "learning_rate": 3.713456652081882e-05,
      "loss_nats": 0.9914250373840332,
      "step": 7210,
      "unclipped_grad_norm": 1.005952000617981
    },
    {
      "bits_per_byte": 1.4493132772594661,
      "learning_rate": 3.695664553868873e-05,
      "loss_nats": 1.0045874118804932,
      "step": 7220,
      "unclipped_grad_norm": 1.0151865482330322
    },
    {
      "bits_per_byte": 1.475039818036732,
      "learning_rate": 3.678091246420277e-05,
      "loss_nats": 1.0224196910858154,
      "step": 7230,
      "unclipped_grad_norm": 1.0841153860092163
    },
    {
      "bits_per_byte": 1.5119313005253603,
      "learning_rate": 3.660737030015427e-05,
      "loss_nats": 1.0479909181594849,
      "step": 7240,
      "unclipped_grad_norm": 1.09213387966156
    },
    {
      "bits_per_byte": 1.494972951237033,
      "learning_rate": 3.6436022011900056e-05,
      "loss_nats": 1.03623628616333,
      "step": 7250,
      "unclipped_grad_norm": 1.0454415082931519
    },
    {
      "bits_per_byte": 1.4613747645297765,
      "learning_rate": 3.626687052730954e-05,
      "loss_nats": 1.0129477977752686,
      "step": 7260,
      "unclipped_grad_norm": 1.0101300477981567
    },
    {
      "bits_per_byte": 1.4686286487778681,
      "learning_rate": 3.6099918736714955e-05,
      "loss_nats": 1.0179758071899414,
      "step": 7270,
      "unclipped_grad_norm": 1.0528877973556519
    },
    {
      "bits_per_byte": 1.4749450555961066,
      "learning_rate": 3.5935169492861716e-05,
      "loss_nats": 1.022354006767273,
      "step": 7280,
      "unclipped_grad_norm": 1.0265198945999146
    },
    {
      "bits_per_byte": 1.473522931056122,
      "learning_rate": 3.5772625610859886e-05,
      "loss_nats": 1.0213682651519775,
      "step": 7290,
      "unclipped_grad_norm": 1.066664457321167
    },
    {
      "bits_per_byte": 1.4559291058728565,
      "learning_rate": 3.5612289868135887e-05,
      "loss_nats": 1.0091731548309326,
      "step": 7300,
      "unclipped_grad_norm": 1.0080223083496094
    },
    {
      "bits_per_byte": 1.445694074354709,
      "learning_rate": 3.545416500438522e-05,
      "loss_nats": 1.0020787715911865,
      "step": 7310,
      "unclipped_grad_norm": 1.0805282592773438
    },
    {
      "bits_per_byte": 1.4250067972303744,
      "learning_rate": 3.52982537215255e-05,
      "loss_nats": 0.9877394437789917,
      "step": 7320,
      "unclipped_grad_norm": 1.0912657976150513
    },
    {
      "bits_per_byte": 1.4783119599520478,
      "learning_rate": 3.5144558683650396e-05,
      "loss_nats": 1.0246877670288086,
      "step": 7330,
      "unclipped_grad_norm": 1.0849530696868896
    },
    {
      "bits_per_byte": 1.4380544330207323,
      "learning_rate": 3.499308251698404e-05,
      "loss_nats": 0.9967833757400513,
      "step": 7340,
      "unclipped_grad_norm": 1.042580485343933
    },
    {
      "bits_per_byte": 1.4567071553853599,
      "learning_rate": 3.484382780983616e-05,
      "loss_nats": 1.0097124576568604,
      "step": 7350,
      "unclipped_grad_norm": 1.0958631038665771
    },
    {
      "bits_per_byte": 1.4481006275882324,
      "learning_rate": 3.469679711255795e-05,
      "loss_nats": 1.0037468671798706,
      "step": 7360,
      "unclipped_grad_norm": 1.0337251424789429
    },
    {
      "bits_per_byte": 1.4027950658887918,
      "learning_rate": 3.4551992937498284e-05,
      "loss_nats": 0.9723434448242188,
      "step": 7370,
      "unclipped_grad_norm": 1.0347329378128052
    },
    {
      "bits_per_byte": 1.4160821015291507,
      "learning_rate": 3.440941775896104e-05,
      "loss_nats": 0.981553316116333,
      "step": 7380,
      "unclipped_grad_norm": 1.0372161865234375
    },
    {
      "bits_per_byte": 1.421894943145663,
      "learning_rate": 3.4269074013162596e-05,
      "loss_nats": 0.9855824708938599,
      "step": 7390,
      "unclipped_grad_norm": 1.038377046585083
    },
    {
      "bits_per_byte": 1.4461825050831596,
      "learning_rate": 3.41309640981904e-05,
      "loss_nats": 1.0024173259735107,
      "step": 7400,
      "unclipped_grad_norm": 1.057859182357788
    },
    {
      "bits_per_byte": 1.475963364871865,
      "learning_rate": 3.3995090373961795e-05,
      "loss_nats": 1.0230598449707031,
      "step": 7410,
      "unclipped_grad_norm": 1.0708297491073608
    },
    {
      "bits_per_byte": 1.469500256852441,
      "learning_rate": 3.3861455162183876e-05,
      "loss_nats": 1.0185799598693848,
      "step": 7420,
      "unclipped_grad_norm": 1.062455654144287
    },
    {
      "bits_per_byte": 1.41574011402791,
      "learning_rate": 3.3730060746313664e-05,
      "loss_nats": 0.9813162684440613,
      "step": 7430,
      "unclipped_grad_norm": 1.0484251976013184
    },
    {
      "bits_per_byte": 1.3821044351027654,
      "learning_rate": 3.3600909371519225e-05,
      "loss_nats": 0.9580017924308777,
      "step": 7440,
      "unclipped_grad_norm": 1.030645728111267
    },
    {
      "bits_per_byte": 1.4987479704234719,
      "learning_rate": 3.3474003244641194e-05,
      "loss_nats": 1.0388529300689697,
      "step": 7450,
      "unclipped_grad_norm": 1.0465614795684814
    },
    {
      "bits_per_byte": 1.4339140366925356,
      "learning_rate": 3.334934453415514e-05,
      "loss_nats": 0.993913471698761,
      "step": 7460,
      "unclipped_grad_norm": 1.0371458530426025
    },
    {
      "bits_per_byte": 1.4012647642614147,
      "learning_rate": 3.3226935370134486e-05,
      "loss_nats": 0.9712827205657959,
      "step": 7470,
      "unclipped_grad_norm": 1.0733681917190552
    },
    {
      "bits_per_byte": 1.5010874504231586,
      "learning_rate": 3.31067778442141e-05,
      "loss_nats": 1.040474534034729,
      "step": 7480,
      "unclipped_grad_norm": 1.0303142070770264
    },
    {
      "bits_per_byte": 1.4650142613873354,
      "learning_rate": 3.298887400955457e-05,
      "loss_nats": 1.0154705047607422,
      "step": 7490,
      "unclipped_grad_norm": 1.0710126161575317
    },
    {
      "bits_per_byte": 1.472811954777455,
      "learning_rate": 3.287322588080714e-05,
      "loss_nats": 1.0208754539489746,
      "step": 7500,
      "unclipped_grad_norm": 1.0901119709014893
    },
    {
      "bits_per_byte": 1.4645168875610401,
      "learning_rate": 3.275983543407924e-05,
      "loss_nats": 1.0151257514953613,
      "step": 7510,
      "unclipped_grad_norm": 1.024397611618042
    },
    {
      "bits_per_byte": 1.4743771688829574,
      "learning_rate": 3.2648704606900735e-05,
      "loss_nats": 1.0219603776931763,
      "step": 7520,
      "unclipped_grad_norm": 1.0934079885482788
    },
    {
      "bits_per_byte": 1.46691879726299,
      "learning_rate": 3.253983529819086e-05,
      "loss_nats": 1.0167906284332275,
      "step": 7530,
      "unclipped_grad_norm": 1.081713318824768
    },
    {
      "bits_per_byte": 1.4640198577000465,
      "learning_rate": 3.243322936822571e-05,
      "loss_nats": 1.0147812366485596,
      "step": 7540,
      "unclipped_grad_norm": 1.0780924558639526
    },
    {
      "bits_per_byte": 1.4520784143200383,
      "learning_rate": 3.232888863860648e-05,
      "loss_nats": 1.0065040588378906,
      "step": 7550,
      "unclipped_grad_norm": 1.04312264919281
    },
    {
      "bits_per_byte": 1.3812490793886991,
      "learning_rate": 3.222681489222838e-05,
      "loss_nats": 0.9574089050292969,
      "step": 7560,
      "unclipped_grad_norm": 1.0638409852981567
    },
    {
      "bits_per_byte": 1.4099118799641457,
      "learning_rate": 3.212700987325009e-05,
      "loss_nats": 0.9772764444351196,
      "step": 7570,
      "unclipped_grad_norm": 1.0123975276947021
    },
    {
      "bits_per_byte": 1.3981073347742243,
      "learning_rate": 3.202947528706403e-05,
      "loss_nats": 0.9690941572189331,
      "step": 7580,
      "unclipped_grad_norm": 1.0492113828659058
    },
    {
      "bits_per_byte": 1.472455950690169,
      "learning_rate": 3.1934212800267116e-05,
      "loss_nats": 1.0206286907196045,
      "step": 7590,
      "unclipped_grad_norm": 1.0436540842056274
    },
    {
      "bits_per_byte": 1.5055485084038889,
      "learning_rate": 3.1841224040632484e-05,
      "loss_nats": 1.0435667037963867,
      "step": 7600,
      "unclipped_grad_norm": 1.140743374824524
    },
    {
      "bits_per_byte": 1.4701950667619266,
      "learning_rate": 3.175051059708143e-05,
      "loss_nats": 1.01906156539917,
      "step": 7610,
      "unclipped_grad_norm": 1.082565426826477
    },
    {
      "bits_per_byte": 1.458967179400348,
      "learning_rate": 3.1662074019656414e-05,
      "loss_nats": 1.0112789869308472,
      "step": 7620,
      "unclipped_grad_norm": 1.0694279670715332
    },
    {
      "bits_per_byte": 1.4490995028244436,
      "learning_rate": 3.157591581949455e-05,
      "loss_nats": 1.0044392347335815,
      "step": 7630,
      "unclipped_grad_norm": 1.0964275598526
    },
    {
      "bits_per_byte": 1.4314776444690875,
      "learning_rate": 3.149203746880175e-05,
      "loss_nats": 0.9922246932983398,
      "step": 7640,
      "unclipped_grad_norm": 1.15219247341156
    },
    {
      "bits_per_byte": 1.4066524647649938,
      "learning_rate": 3.141044040082763e-05,
      "loss_nats": 0.9750171899795532,
      "step": 7650,
      "unclipped_grad_norm": 1.064915418624878
    },
    {
      "bits_per_byte": 1.416584462852321,
      "learning_rate": 3.133112600984091e-05,
      "loss_nats": 0.9819015264511108,
      "step": 7660,
      "unclipped_grad_norm": 1.141569972038269
    },
    {
      "bits_per_byte": 1.4608237321164121,
      "learning_rate": 3.125409565110571e-05,
      "loss_nats": 1.0125658512115479,
      "step": 7670,
      "unclipped_grad_norm": 1.0876517295837402
    },
    {
      "bits_per_byte": 1.4539619683122877,
      "learning_rate": 3.117935064085835e-05,
      "loss_nats": 1.0078096389770508,
      "step": 7680,
      "unclipped_grad_norm": 1.0938481092453003
    },
    {
      "bits_per_byte": 1.433180616677931,
      "learning_rate": 3.110689225628479e-05,
      "loss_nats": 0.9934051036834717,
      "step": 7690,
      "unclipped_grad_norm": 1.070209264755249
    },
    {
      "bits_per_byte": 1.4952543148538446,
      "learning_rate": 3.1036721735498926e-05,
      "loss_nats": 1.0364313125610352,
      "step": 7700,
      "unclipped_grad_norm": 1.0864635705947876
    },
    {
      "bits_per_byte": 1.412102165014209,
      "learning_rate": 3.096884027752132e-05,
      "loss_nats": 0.9787946343421936,
      "step": 7710,
      "unclipped_grad_norm": 1.040504813194275
    },
    {
      "bits_per_byte": 1.4180753804526507,
      "learning_rate": 3.090324904225885e-05,
      "loss_nats": 0.9829349517822266,
      "step": 7720,
      "unclipped_grad_norm": 1.0190634727478027
    },
    {
      "bits_per_byte": 1.4255394275000965,
      "learning_rate": 3.083994915048472e-05,
      "loss_nats": 0.9881086349487305,
      "step": 7730,
      "unclipped_grad_norm": 1.0293750762939453
    },
    {
      "bits_per_byte": 1.468768986620972,
      "learning_rate": 3.077894168381942e-05,
      "loss_nats": 1.0180730819702148,
      "step": 7740,
      "unclipped_grad_norm": 1.0531129837036133
    },
    {
      "bits_per_byte": 1.4037740771288283,
      "learning_rate": 3.0720227684712254e-05,
      "loss_nats": 0.9730220437049866,
      "step": 7750,
      "unclipped_grad_norm": 1.1033527851104736
    },
    {
      "bits_per_byte": 1.4111542826339787,
      "learning_rate": 3.0663808156423456e-05,
      "loss_nats": 0.9781376123428345,
      "step": 7760,
      "unclipped_grad_norm": 1.0707266330718994
    },
    {
      "bits_per_byte": 1.4583370349675868,
      "learning_rate": 3.060968406300711e-05,
      "loss_nats": 1.010842204093933,
      "step": 7770,
      "unclipped_grad_norm": 1.0769577026367188
    },
    {
      "bits_per_byte": 1.4516912813729461,
      "learning_rate": 3.055785632929463e-05,
      "loss_nats": 1.0062357187271118,
      "step": 7780,
      "unclipped_grad_norm": 1.1653907299041748
    },
    {
      "bits_per_byte": 1.4586383485718983,
      "learning_rate": 3.0508325840878984e-05,
      "loss_nats": 1.011051058769226,
      "step": 7790,
      "unclipped_grad_norm": 1.0739812850952148
    },
    {
      "bits_per_byte": 1.4426457678594913,
      "learning_rate": 3.046109344409957e-05,
      "loss_nats": 0.9999658465385437,
      "step": 7800,
      "unclipped_grad_norm": 1.0972295999526978
    },
    {
      "bits_per_byte": 1.4198621082124645,
      "learning_rate": 3.0416159946027727e-05,
      "loss_nats": 0.9841734170913696,
      "step": 7810,
      "unclipped_grad_norm": 1.075659990310669
    },
    {
      "bits_per_byte": 1.4512271001982673,
      "learning_rate": 3.0373526114452946e-05,
      "loss_nats": 1.0059139728546143,
      "step": 7820,
      "unclipped_grad_norm": 1.1029958724975586
    },
    {
      "bits_per_byte": 1.4583721194283628,
      "learning_rate": 3.0333192677869803e-05,
      "loss_nats": 1.0108665227890015,
      "step": 7830,
      "unclipped_grad_norm": 1.081362009048462
    },
    {
      "bits_per_byte": 1.4242661539444337,
      "learning_rate": 3.029516032546544e-05,
      "loss_nats": 0.9872260689735413,
      "step": 7840,
      "unclipped_grad_norm": 1.0629743337631226
    },
    {
      "bits_per_byte": 1.423362471105475,
      "learning_rate": 3.02594297071078e-05,
      "loss_nats": 0.9865996837615967,
      "step": 7850,
      "unclipped_grad_norm": 1.0624163150787354
    },
    {
      "bits_per_byte": 1.4807702799634808,
      "learning_rate": 3.0226001433334603e-05,
      "loss_nats": 1.0263917446136475,
      "step": 7860,
      "unclipped_grad_norm": 1.0853880643844604
    },
    {
      "bits_per_byte": 1.4488509878939468,
      "learning_rate": 3.019487607534278e-05,
      "loss_nats": 1.0042669773101807,
      "step": 7870,
      "unclipped_grad_norm": 1.1049087047576904
    },
    {
      "bits_per_byte": 1.4220910033676464,
      "learning_rate": 3.016605416497886e-05,
      "loss_nats": 0.9857183694839478,
      "step": 7880,
      "unclipped_grad_norm": 1.0985115766525269
    },
    {
      "bits_per_byte": 1.4594506226319235,
      "learning_rate": 3.0139536194729734e-05,
      "loss_nats": 1.0116140842437744,
      "step": 7890,
      "unclipped_grad_norm": 1.0736068487167358
    },
    {
      "bits_per_byte": 1.4105420243969071,
      "learning_rate": 3.0115322617714362e-05,
      "loss_nats": 0.9777132272720337,
      "step": 7900,
      "unclipped_grad_norm": 1.1535842418670654
    },
    {
      "bits_per_byte": 1.463574250651661,
      "learning_rate": 3.0093413847675965e-05,
      "loss_nats": 1.0144723653793335,
      "step": 7910,
      "unclipped_grad_norm": 1.1884382963180542
    },
    {
      "bits_per_byte": 1.4830432026772835,
      "learning_rate": 3.0073810258974985e-05,
      "loss_nats": 1.0279672145843506,
      "step": 7920,
      "unclipped_grad_norm": 1.1013721227645874
    },
    {
      "bits_per_byte": 1.435883066062264,
      "learning_rate": 3.0056512186582658e-05,
      "loss_nats": 0.9952782988548279,
      "step": 7930,
      "unclipped_grad_norm": 1.0824542045593262
    },
    {
      "bits_per_byte": 1.4594231054077853,
      "learning_rate": 3.0041519926075316e-05,
      "loss_nats": 1.0115950107574463,
      "step": 7940,
      "unclipped_grad_norm": 1.1213319301605225
    },
    {
      "bits_per_byte": 1.408353201199376,
      "learning_rate": 3.0028833733629313e-05,
      "loss_nats": 0.9761960506439209,
      "step": 7950,
      "unclipped_grad_norm": 1.0871061086654663
    },
    {
      "bits_per_byte": 1.4735662706841393,
      "learning_rate": 3.0018453826016686e-05,
      "loss_nats": 1.0213983058929443,
      "step": 7960,
      "unclipped_grad_norm": 1.1091856956481934
    },
    {
      "bits_per_byte": 1.4267374586460069,
      "learning_rate": 3.001038038060143e-05,
      "loss_nats": 0.9889390468597412,
      "step": 7970,
      "unclipped_grad_norm": 1.0478838682174683
    },
    {
      "bits_per_byte": 1.472857186214632,
      "learning_rate": 3.0004613535336433e-05,
      "loss_nats": 1.0209068059921265,
      "step": 7980,
      "unclipped_grad_norm": 1.2172881364822388
    },
    {
      "bits_per_byte": 1.4136101948882995,
      "learning_rate": 3.000115338876117e-05,
      "loss_nats": 0.9798399209976196,
      "step": 7990,
      "unclipped_grad_norm": 1.1755717992782593
    },
    {
      "bits_per_byte": 1.46345644253582,
      "learning_rate": 3e-05,
      "loss_nats": 1.0143907070159912,
      "step": 8000,
      "unclipped_grad_norm": 1.0856302976608276
    }
  ]
}