{
  "architecture": "block",
  "batch_size": 32,
  "canonical_sha256_without_self": "513b85666fcdbf75424596e968cd73733314ec188345e453998de08b18dcef65",
  "depth": 32,
  "diagnostics": [
    {
      "activation_grad_rms_by_block": [
        0.00027819047681987286,
        0.0002560806169640273,
        0.0002331122086616233,
        0.00022053472639527172,
        0.00020666928321588784,
        0.00019684624567162246,
        0.0001890837011160329,
        0.00018263237143401057,
        0.00017469045997131616,
        0.0001688334159553051,
        0.0001649101759539917,
        0.00016157960635609925,
        0.00015717606584075838,
        0.00015400603297166526,
        0.00015046694898046553,
        0.00014767753600608557,
        0.00014588078192900866,
        0.00014366162940859795,
        0.00014116008242126554,
        0.00013849612150806934,
        0.0001361419854220003,
        0.00013430001854430884,
        0.00013224364374764264,
        0.00013045054220128804,
        0.00012888714263681322,
        0.00012767451698891819,
        0.0001263424928765744,
        0.00012512138346210122,
        0.00012419422273524106,
        0.00012343283742666245,
        0.0001227961911354214,
        0.00012195061572128907
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00022039370378479362,
        "first_to_last_ratio": 1.7624457042066763,
        "imbalance_abs_log_ratio": 0.5667024491058967,
        "last_quartile_mean": 0.00012504992537287762,
        "mean": 0.0001607882525149762,
        "normalized": [
          1.7301666786506156,
          1.592657504255016,
          1.4498087081325215,
          1.3715848200709229,
          1.2853506396348091,
          1.224257634451794,
          1.175979576607571,
          1.1358564358860717,
          1.0864628307036615,
          1.050035766385231,
          1.0256357250890054,
          1.0049217142965674,
          0.9775345112735686,
          0.9578189361646354,
          0.9358080993289649,
          0.9184597363064851,
          0.9072850761620224,
          0.8934833681037547,
          0.8779253472396409,
          0.8613572157279928,
          0.8467159963027754,
          0.8352601414820391,
          0.8224708066612338,
          0.81131886291965,
          0.8015955184587156,
          0.7940537632065271,
          0.7857694259399116,
          0.7781749070905981,
          0.7724085609032495,
          0.7676732316943716,
          0.7637136993200662,
          0.75845476155001
        ],
        "population_cv": 0.2498932502086342
      },
      "activation_output_rms_by_block": [
        0.0034565660171210766,
        0.004928664769977331,
        0.0060537648387253284,
        0.006960673723369837,
        0.0036145171616226435,
        0.0051584807224571705,
        0.006370538379997015,
        0.007546419743448496,
        0.00395671371370554,
        0.005665866658091545,
        0.006755253300070763,
        0.007946331985294819,
        0.004383862018585205,
        0.006652849726378918,
        0.007932198233902454,
        0.009347881190478802,
        0.004880861379206181,
        0.006906712893396616,
        0.008132148534059525,
        0.009816600009799004,
        0.005453306715935469,
        0.007622495759278536,
        0.009448722936213017,
        0.010670131072402,
        0.0055701653473079205,
        0.007738742511719465,
        0.009509840048849583,
        0.010865328833460808,
        0.005987230688333511,
        0.00815181527286768,
        0.009874933399260044,
        0.011666664853692055
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.005511203169589862,
        "first_to_last_ratio": 0.6356203088456116,
        "imbalance_abs_log_ratio": 0.45315389258709987,
        "last_quartile_mean": 0.008670590119436383,
        "mean": 0.007157071326219011,
        "normalized": [
          0.4829581625738996,
          0.6886426787339398,
          0.8458438602600115,
          0.9725589429115095,
          0.5050274053272774,
          0.7207530129760953,
          0.8901040788372985,
          1.0544005221525679,
          0.5528397766850007,
          0.7916459680002588,
          0.9438571997073384,
          1.1102770425361632,
          0.6125217730505899,
          0.9295491721602742,
          1.1083022471557418,
          1.3061042379491232,
          0.6819634955048963,
          0.9650194302374437,
          1.1362396940587196,
          1.3715945478756304,
          0.7619466772614072,
          1.0650300118366156,
          1.3201940438400879,
          1.490851576861243,
          0.7782743937317399,
          1.0812722353862223,
          1.3287334463206348,
          1.5181249897087195,
          0.8365475786723071,
          1.138987569259587,
          1.3797450031110483,
          1.6300892253166077
        ],
        "population_cv": 0.3025375276561286
      },
      "bits_per_byte": 8.06433666787865,
      "branch_output_rms_by_sublayer": [
        0.0021227982360869646,
        0.0027448106557130814,
        0.00224129855632782,
        0.0027243648655712605,
        0.0024903982412070036,
        0.0027103275060653687,
        0.0023722455371171236,
        0.0027234479784965515,
        0.0023577234242111444,
        0.0027595730498433113,
        0.0023993616923689842,
        0.0027272943407297134,
        0.002688874490559101,
        0.002728298306465149,
        0.0029664887115359306,
        0.0027007071767002344,
        0.0029547461308538914,
        0.002684502862393856,
        0.003052416956052184,
        0.002683063969016075,
        0.0028214550111442804,
        0.0027139110025018454,
        0.0032544767018407583,
        0.002666700165718794,
        0.00348988501355052,
        0.002679243916645646,
        0.003914494998753071,
        0.0026584581937640905,
        0.0035071265883743763,
        0.002684473292902112,
        0.004211232997477055,
        0.0026407993864268064,
        0.004072706215083599,
        0.002663099905475974,
        0.004050548188388348,
        0.0026283918414264917,
        0.003929397091269493,
        0.002597011858597398,
        0.004410331137478352,
        0.0026434636674821377,
        0.004824656527489424,
        0.0025708195753395557,
        0.00433384021744132,
        0.002691218862310052,
        0.004475373309105635,
        0.0026598076801747084,
        0.004755842965096235,
        0.0026596153620630503,
        0.004843301139771938,
        0.0026765079237520695,
        0.0050001065246760845,
        0.002582729561254382,
        0.005346097983419895,
        0.002614464145153761,
        0.004958339501172304,
        0.0025760054122656584,
        0.0054801348596811295,
        0.002634295728057623,
        0.005455824080854654,
        0.00259007909335196,
        0.005286100786179304,
        0.00257018837146461,
        0.005402065813541412,
        0.0025643527042120695
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.006159982634052032,
        0.005514663551817784,
        0.005191681274696802,
        0.00489831575692433,
        0.004666415075367917,
        0.004399813239791133,
        0.004483878567781538,
        0.004868281936291656,
        0.00443828064080646,
        0.004266304440803418,
        0.0039471188591477255,
        0.004494388758593749,
        0.0042334459038140095,
        0.00446434979681851,
        0.004322339302329385,
        0.004638471384339534,
        0.004165330760400444,
        0.004499937764754925,
        0.003981478974932938,
        0.004372386299982334,
        0.00458495381977442,
        0.00429501466057645,
        0.003963900484885007,
        0.004271473918508079,
        0.004041799650627023,
        0.004206875397956982,
        0.003955591472942362,
        0.003916464042690177,
        0.003989587235588314,
        0.003922309599766117,
        0.003679536678174494,
        0.003763244111925236
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.005022879004590398,
        "first_to_last_ratio": 1.2766484804448084,
        "imbalance_abs_log_ratio": 0.24423826934058726,
        "last_quartile_mean": 0.003934426023708838,
        "mean": 0.0043936754999019155,
        "normalized": [
          1.4020112851278042,
          1.255136741878387,
          1.1816260155791434,
          1.1148560600421402,
          1.0620754936207943,
          1.0013969488391563,
          1.0205302070855338,
          1.108020366183241,
          1.0101521245493756,
          0.9710103627130996,
          0.8983637638318627,
          1.0229223252136126,
          0.9635317637610054,
          1.0160854612313022,
          0.9837638902613262,
          1.0557155130011497,
          0.9480287655502622,
          1.0241852783291305,
          0.9061841219320409,
          0.995154580733134,
          1.0435349219296635,
          0.9775448051801577,
          0.9021832597727114,
          0.9721869351988865,
          0.9199131002545027,
          0.9574843199164109,
          0.9002921296829196,
          0.891386731400079,
          0.9080295610536958,
          0.892717179467095,
          0.8374620925593245,
          0.8565138941210488
        ],
        "population_cv": 0.11556590700625036
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.028097106143832207,
        0.014079716056585312,
        0.014136734418570995,
        0.014167090877890587,
        0.014224153943359852,
        0.014255538582801819,
        0.014337832108139992,
        0.014355218037962914,
        0.014408928342163563,
        0.009655896574258804,
        0.009704872034490108,
        0.009734091348946095,
        0.009773381985723972,
        0.009828771464526653,
        0.00987160298973322,
        0.009915035218000412,
        0.00994801800698042,
        0.007479202933609486,
        0.007515520788729191,
        0.007559895049780607,
        0.007602147292345762,
        0.0076358383521437645,
        0.007658426649868488,
        0.007706487085670233,
        0.007734050042927265,
        0.0062490664422512054,
        0.0062688966281712055,
        0.006315518636256456,
        0.006336068734526634,
        0.0063744354993104935,
        0.006399397272616625,
        0.006439714692533016,
        0.006462901830673218,
        0.0054116579703986645,
        0.005432929843664169,
        0.005481995176523924,
        0.005504137370735407,
        0.005541813559830189,
        0.005560056772083044,
        0.005614723544567823,
        0.005631812382489443,
        0.004862982779741287,
        0.004873365629464388,
        0.004923288244754076,
        0.004937449004501104,
        0.0049925679340958595,
        0.005005750339478254,
        0.005004179198294878,
        0.005022916477173567,
        0.004448203835636377,
        0.0044684382155537605,
        0.004490054678171873,
        0.004500201437622309,
        0.00455933902412653,
        0.004573327023535967,
        0.004625064320862293,
        0.00463611725717783,
        0.004160413984209299,
        0.004166214261204004,
        0.004242690280079842,
        0.0042496914975345135,
        0.004270957782864571,
        0.004283483140170574,
        0.004303553607314825
      ],
      "loss_nats": 5.5897722244262695,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 2.1972246170043945,
        "mean_weights": [
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519
        ],
        "sources": 9
      },
      "step": 0,
      "stream_state_rms_by_sublayer": [
        0.0021227982360869646,
        0.0034565660171210766,
        0.00413061399012804,
        0.004928664769977331,
        0.005416445434093475,
        0.0060537648387253284,
        0.006388718727976084,
        0.006960673723369837,
        0.0023577234242111444,
        0.0036145171616226435,
        0.004388594534248114,
        0.0051584807224571705,
        0.00574506726115942,
        0.006370538379997015,
        0.007066869642585516,
        0.007546419743448496,
        0.0029547461308538914,
        0.00395671371370554,
        0.004991721361875534,
        0.005665866658091545,
        0.006190058775246143,
        0.006755253300070763,
        0.0074937604367733,
        0.007946331985294819,
        0.00348988501355052,
        0.004383862018585205,
        0.006057556718587875,
        0.006652849726378918,
        0.0074848332442343235,
        0.007932198233902454,
        0.00897275935858488,
        0.009347881190478802,
        0.004072706215083599,
        0.004880861379206181,
        0.0063449000008404255,
        0.006906712893396616,
        0.007735534105449915,
        0.008132148534059525,
        0.00946879480034113,
        0.009816600009799004,
        0.004824656527489424,
        0.005453306715935469,
        0.007074959110468626,
        0.007622495759278536,
        0.009095527231693268,
        0.009448722936213017,
        0.010346688330173492,
        0.010670131072402,
        0.004843301139771938,
        0.0055701653473079205,
        0.007334613241255283,
        0.007738742511719465,
        0.009146629832684994,
        0.009509840048849583,
        0.010587968863546848,
        0.010865328833460808,
        0.0054801348596811295,
        0.005987230688333511,
        0.0077637480571866035,
        0.00815181527286768,
        0.009536821395158768,
        0.009874933399260044,
        0.011367839761078358,
        0.011666664853692055
      ]
    },
    {
      "activation_grad_rms_by_block": [
        6.886429764563218e-05,
        6.52737362543121e-05,
        6.186874816194177e-05,
        5.901425174670294e-05,
        5.411080928752199e-05,
        5.1034174248343334e-05,
        4.815354259335436e-05,
        4.550318044493906e-05,
        4.120725498069078e-05,
        3.8442405639216304e-05,
        3.604330777307041e-05,
        3.37670098815579e-05,
        3.108748569502495e-05,
        2.9203254598542117e-05,
        2.7565200070966966e-05,
        2.5956482204492204e-05,
        2.4566294086980633e-05,
        2.329334893147461e-05,
        2.2260202968027443e-05,
        2.14147730730474e-05,
        2.0902438336634077e-05,
        2.0279332602513023e-05,
        1.9777973648160696e-05,
        1.9328235794091597e-05,
        1.9035323930438608e-05,
        1.8655482563190162e-05,
        1.832095585996285e-05,
        1.8063024981529452e-05,
        1.7733544154907577e-05,
        1.752249227138236e-05,
        1.7320473489235155e-05,
        1.7174717868329026e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 5.6727842547843466e-05,
        "first_to_last_ratio": 3.155359202626439,
        "imbalance_abs_log_ratio": 1.1491023413751247,
        "last_quartile_mean": 1.79782518898719e-05,
        "mean": 3.227324236831919e-05,
        "normalized": [
          2.133789250541258,
          2.0225341944070556,
          1.9170292050567193,
          1.8285814320482991,
          1.67664620337743,
          1.581315371598383,
          1.4920577871848402,
          1.409935200362978,
          1.2768241415105412,
          1.1911541208190792,
          1.1168170635514478,
          1.0462850151897058,
          0.9632588303411923,
          0.904875136661487,
          0.8541193275958588,
          0.8042725273235084,
          0.7611969633115008,
          0.7217542218299198,
          0.6897417592562383,
          0.6635457580819046,
          0.647670850610065,
          0.628363657145899,
          0.6128288389014056,
          0.5988935221787641,
          0.5898175247840765,
          0.5780479801280578,
          0.5676825294116558,
          0.559690432569022,
          0.5494813304632692,
          0.5429417990112825,
          0.5366821620079203,
          0.5321658627392354
        ],
        "population_cv": 0.49221598973047537
      },
      "activation_output_rms_by_block": [
        0.0055193523876369,
        0.009546536952257156,
        0.013977172784507275,
        0.018744563683867455,
        0.00660830270498991,
        0.011765030212700367,
        0.017156247049570084,
        0.022267481312155724,
        0.007449587807059288,
        0.013175037689507008,
        0.019073888659477234,
        0.025712018832564354,
        0.008972279727458954,
        0.01624389924108982,
        0.023743286728858948,
        0.03214457631111145,
        0.00991375558078289,
        0.019281653687357903,
        0.02868141047656536,
        0.03889070823788643,
        0.0127727584913373,
        0.022841427475214005,
        0.034929387271404266,
        0.04634999856352806,
        0.014845813624560833,
        0.029589947313070297,
        0.046351123601198196,
        0.06254568696022034,
        0.01947367936372757,
        0.0370744951069355,
        0.05507569760084152,
        0.07403328269720078
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.013198085885960609,
        "first_to_last_ratio": 0.31146869331458016,
        "imbalance_abs_log_ratio": 1.1664564488083473,
        "last_quartile_mean": 0.04237371578346938,
        "mean": 0.0251484402542701,
        "normalized": [
          0.21947096248642048,
          0.3796075166385794,
          0.5557868656341027,
          0.7453569085933552,
          0.26277187126417706,
          0.46782345520226504,
          0.6821992487847045,
          0.8854418439877121,
          0.2962246458125521,
          0.5238908479530829,
          0.7584521531604159,
          1.0224100808080359,
          0.35677281122575777,
          0.6459207440641045,
          0.9441256192748351,
          1.2781936369057099,
          0.39420956053525336,
          0.7667137004285568,
          1.140484665711838,
          1.5464461352144077,
          0.5078946591595691,
          0.9082641803733982,
          1.3889285744261377,
          1.8430565909811456,
          0.5903274109431131,
          1.1766116313335198,
          1.8431013268637195,
          2.487060283971303,
          0.7743493897368454,
          1.4742264224772512,
          2.190024392923927,
          2.9438518631242045
        ],
        "population_cv": 0.6624204772463126
      },
      "bits_per_byte": 5.18286091123436,
      "branch_output_rms_by_sublayer": [
        0.002723898272961378,
        0.004827107302844524,
        0.0025367254856973886,
        0.004943604115396738,
        0.0027546093333512545,
        0.00549353240057826,
        0.002648916095495224,
        0.005891259294003248,
        0.0027809101156890392,
        0.0059933834709227085,
        0.0025255572982132435,
        0.006044833920896053,
        0.003397067543119192,
        0.0061737713403999805,
        0.0033452848438173532,
        0.0059272525832057,
        0.003559867851436138,
        0.0063678608275949955,
        0.0038245979230850935,
        0.0061998614110052586,
        0.003386941971257329,
        0.006479958072304726,
        0.00461281044408679,
        0.006899545900523663,
        0.004887574352324009,
        0.007097913417965174,
        0.005150536075234413,
        0.006852242164313793,
        0.0054018739610910416,
        0.006816149223595858,
        0.006125702988356352,
        0.007103090640157461,
        0.005625482182949781,
        0.007109561935067177,
        0.006412550341337919,
        0.007626448292285204,
        0.0072620403952896595,
        0.007445307914167643,
        0.008774910122156143,
        0.007442969363182783,
        0.008495661430060863,
        0.007471293676644564,
        0.007638709619641304,
        0.007774693425744772,
        0.009623398073017597,
        0.007449399214237928,
        0.008996927179396152,
        0.007351893465965986,
        0.00961991585791111,
        0.008440978825092316,
        0.011907265521585941,
        0.008385274559259415,
        0.014190610498189926,
        0.008216616697609425,
        0.013635334558784962,
        0.007803319953382015,
        0.014384749345481396,
        0.007909612730145454,
        0.01446358859539032,
        0.007761957123875618,
        0.013604369945824146,
        0.008686874993145466,
        0.015084062702953815,
        0.008338849991559982
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00023786681750428726,
        0.00020576573809540593,
        0.00019740566304489066,
        0.0001813902134209802,
        0.0001696969276866905,
        0.00015923444695871368,
        0.00017314685145041394,
        0.00015953319119459735,
        0.0001633125663305534,
        0.0001601758879769712,
        0.00014581270072009644,
        0.00016837778263142155,
        0.0001603117902727929,
        0.00016340123818181664,
        0.00015138579649161358,
        0.00015756725288262655,
        0.00013845240957374987,
        0.0001515626583813202,
        0.00014303549429743115,
        0.00016115390114098118,
        0.00015144047315466305,
        0.00013355245347473064,
        0.00014934284038847835,
        0.00014161248575260865,
        0.00014194691162247068,
        0.00015146794722523853,
        0.00015520640136475585,
        0.0001429086966261467,
        0.00014288517283478702,
        0.00013975759356060232,
        0.00013469418893153685,
        0.00013332122361590492
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00018550498116949745,
        "first_to_last_ratio": 1.2992954513055368,
        "imbalance_abs_log_ratio": 0.26182215701959494,
        "last_quartile_mean": 0.00014277351697268036,
        "mean": 0.00015833517864966492,
        "normalized": [
          1.5022992333914333,
          1.2995579368416077,
          1.2467580782011416,
          1.1456090489045851,
          1.0717575786626974,
          1.0056795231275704,
          1.0935463169149642,
          1.0075663068381233,
          1.0314357663491922,
          1.0116253971038178,
          0.9209115874541316,
          1.0634262333071103,
          1.0124837173898131,
          1.0319957925671146,
          0.9561096768430238,
          0.9951499990489321,
          0.874426079880151,
          0.9572266862859976,
          0.9033715407863586,
          1.0178022503612605,
          0.9564549991113648,
          0.8434793494011273,
          0.9432069465681839,
          0.894384223141863,
          0.8964963619142705,
          0.9566285175348117,
          0.9802395316594056,
          0.9025707227220104,
          0.9024221531396832,
          0.8826692510944287,
          0.8506902261408595,
          0.8420189673129665
        ],
        "population_cv": 0.13848454603612886
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.6920576691627502,
          "mean_weights": [
            0.5229578018188477,
            0.47704219818115234
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6922386288642883,
          "mean_weights": [
            0.5206724405288696,
            0.479327529668808
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6930604577064514,
          "mean_weights": [
            0.5023424625396729,
            0.4976575970649719
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6924410462379456,
          "mean_weights": [
            0.5176653861999512,
            0.48233455419540405
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6929901838302612,
          "mean_weights": [
            0.5013687014579773,
            0.4986313283443451
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6929076313972473,
          "mean_weights": [
            0.5098068714141846,
            0.49019312858581543
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6927430629730225,
          "mean_weights": [
            0.5127849578857422,
            0.4872150421142578
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6928876638412476,
          "mean_weights": [
            0.5100651383399963,
            0.48993486166000366
          ],
          "sources": 2
        },
        {
          "entropy_mean": 1.098104476928711,
          "mean_weights": [
            0.33572155237197876,
            0.32083213329315186,
            0.34344637393951416
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.098517656326294,
          "mean_weights": [
            0.3379058539867401,
            0.3309776186943054,
            0.33111652731895447
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0984238386154175,
          "mean_weights": [
            0.34072375297546387,
            0.3312225341796875,
            0.3280537724494934
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0985376834869385,
          "mean_weights": [
            0.3372243344783783,
            0.33110499382019043,
            0.3316706418991089
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0983879566192627,
          "mean_weights": [
            0.34103795886039734,
            0.33302855491638184,
            0.3259335160255432
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0984759330749512,
          "mean_weights": [
            0.3378061056137085,
            0.3331226110458374,
            0.3290712833404541
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0982115268707275,
          "mean_weights": [
            0.3427826464176178,
            0.33246177434921265,
            0.32475557923316956
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.098510980606079,
          "mean_weights": [
            0.336602121591568,
            0.33530712127685547,
            0.32809075713157654
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.3860652446746826,
          "mean_weights": [
            0.25563088059425354,
            0.25276094675064087,
            0.24706901609897614,
            0.24453918635845184
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3861669301986694,
          "mean_weights": [
            0.25406360626220703,
            0.25270599126815796,
            0.24658669531345367,
            0.24664369225502014
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.386000394821167,
          "mean_weights": [
            0.2551618814468384,
            0.2541123330593109,
            0.2483193278312683,
            0.24240648746490479
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.386232852935791,
          "mean_weights": [
            0.25183945894241333,
            0.25246676802635193,
            0.24847325682640076,
            0.24722051620483398
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.385960340499878,
          "mean_weights": [
            0.25454509258270264,
            0.25490614771842957,
            0.24972698092460632,
            0.24082177877426147
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.386134147644043,
          "mean_weights": [
            0.25200384855270386,
            0.25437670946121216,
            0.24930089712142944,
            0.24431857466697693
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3859115839004517,
          "mean_weights": [
            0.2541099786758423,
            0.2563633620738983,
            0.2499358206987381,
            0.2395908385515213
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3861732482910156,
          "mean_weights": [
            0.2492598295211792,
            0.25510817766189575,
            0.2504415512084961,
            0.24519044160842896
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.6091704368591309,
          "mean_weights": [
            0.20294126868247986,
            0.20561827719211578,
            0.20076946914196014,
            0.19495637714862823,
            0.19571462273597717
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6093817949295044,
          "mean_weights": [
            0.1999536156654358,
            0.20290037989616394,
            0.19905918836593628,
            0.1980593204498291,
            0.2000274658203125
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6089024543762207,
          "mean_weights": [
            0.20434537529945374,
            0.20689502358436584,
            0.20309439301490784,
            0.19485914707183838,
            0.1908060759305954
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.609398365020752,
          "mean_weights": [
            0.20015040040016174,
            0.2025333195924759,
            0.2003234326839447,
            0.19931097328662872,
            0.19768190383911133
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6085304021835327,
          "mean_weights": [
            0.20286625623703003,
            0.21047379076480865,
            0.20388861000537872,
            0.19463706016540527,
            0.18813428282737732
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.60928213596344,
          "mean_weights": [
            0.20026028156280518,
            0.20464842021465302,
            0.20178082585334778,
            0.1985689103603363,
            0.19474159181118011
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6085617542266846,
          "mean_weights": [
            0.20295967161655426,
            0.2110326737165451,
            0.20332126319408417,
            0.194858580827713,
            0.18782781064510345
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6093013286590576,
          "mean_weights": [
            0.20157885551452637,
            0.20404376089572906,
            0.20028473436832428,
            0.19877153635025024,
            0.19532111287117004
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.79081130027771,
          "mean_weights": [
            0.1695082187652588,
            0.17698298394680023,
            0.17063108086585999,
            0.16265572607517242,
            0.15738935768604279,
            0.16283263266086578
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7917133569717407,
          "mean_weights": [
            0.16738522052764893,
            0.1689736694097519,
            0.16775797307491302,
            0.16576024889945984,
            0.16475272178649902,
            0.16537019610404968
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7902145385742188,
          "mean_weights": [
            0.17057637870311737,
            0.18038460612297058,
            0.17244961857795715,
            0.164292573928833,
            0.15751495957374573,
            0.15478186309337616
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7917354106903076,
          "mean_weights": [
            0.16660897433757782,
            0.16850554943084717,
            0.1666441559791565,
            0.16635674238204956,
            0.16625551879405975,
            0.1656290590763092
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7898752689361572,
          "mean_weights": [
            0.17184747755527496,
            0.180228590965271,
            0.17470473051071167,
            0.16429948806762695,
            0.15632927417755127,
            0.15259043872356415
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7917178869247437,
          "mean_weights": [
            0.1659121811389923,
            0.16851314902305603,
            0.16693055629730225,
            0.16679996252059937,
            0.16575691103935242,
            0.16608723998069763
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7883799076080322,
          "mean_weights": [
            0.1728566288948059,
            0.18512636423110962,
            0.17585304379463196,
            0.16398008167743683,
            0.1537502557039261,
            0.1484336107969284
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7917424440383911,
          "mean_weights": [
            0.1667865514755249,
            0.1680906116962433,
            0.16619735956192017,
            0.16709285974502563,
            0.16617436707019806,
            0.16565825045108795
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.9426109790802002,
          "mean_weights": [
            0.14892546832561493,
            0.16000479459762573,
            0.15354622900485992,
            0.14328595995903015,
            0.13467344641685486,
            0.12974688410758972,
            0.12981724739074707
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9458978176116943,
          "mean_weights": [
            0.1419999748468399,
            0.1434222161769867,
            0.14279979467391968,
            0.14312142133712769,
            0.14279043674468994,
            0.1428302675485611,
            0.143035888671875
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9415357112884521,
          "mean_weights": [
            0.1496729701757431,
            0.16313126683235168,
            0.15414540469646454,
            0.1440294086933136,
            0.1343868225812912,
            0.12868303060531616,
            0.1259511113166809
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9458801746368408,
          "mean_weights": [
            0.1429852843284607,
            0.1429750919342041,
            0.14214137196540833,
            0.1414642632007599,
            0.1421438455581665,
            0.14385518431663513,
            0.14443498849868774
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9424511194229126,
          "mean_weights": [
            0.14928653836250305,
            0.16045087575912476,
            0.15314346551895142,
            0.1440504789352417,
            0.1356543004512787,
            0.13001246750354767,
            0.1274019032716751
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9458882808685303,
          "mean_weights": [
            0.14319148659706116,
            0.14391475915908813,
            0.14284171164035797,
            0.1420421153306961,
            0.14134715497493744,
            0.14342516660690308,
            0.1432376205921173
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9395697116851807,
          "mean_weights": [
            0.15016257762908936,
            0.16719450056552887,
            0.15599417686462402,
            0.14574657380580902,
            0.13377557694911957,
            0.12622776627540588,
            0.12089881300926208
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9458978176116943,
          "mean_weights": [
            0.1428529918193817,
            0.14342239499092102,
            0.14335417747497559,
            0.14274317026138306,
            0.1419316530227661,
            0.14220726490020752,
            0.14348836243152618
          ],
          "sources": 7
        },
        {
          "entropy_mean": 2.074862480163574,
          "mean_weights": [
            0.13159659504890442,
            0.14486080408096313,
            0.13573890924453735,
            0.12842535972595215,
            0.12030278891324997,
            0.11495959758758545,
            0.10915482044219971,
            0.11496111750602722
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079418659210205,
          "mean_weights": [
            0.12459750473499298,
            0.12504151463508606,
            0.12489251047372818,
            0.125042125582695,
            0.12461285293102264,
            0.12601442635059357,
            0.1261361837387085,
            0.12366288155317307
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0741164684295654,
          "mean_weights": [
            0.1323997974395752,
            0.14626389741897583,
            0.1369163542985916,
            0.12836164236068726,
            0.12002508342266083,
            0.11502262949943542,
            0.10908686369657516,
            0.11192375421524048
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0793213844299316,
          "mean_weights": [
            0.12429114431142807,
            0.12243181467056274,
            0.12357296049594879,
            0.12375843524932861,
            0.12469147145748138,
            0.12560917437076569,
            0.12766826152801514,
            0.12797673046588898
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0734808444976807,
          "mean_weights": [
            0.1330580711364746,
            0.14781729876995087,
            0.1374434530735016,
            0.12885303795337677,
            0.11949820816516876,
            0.11495119333267212,
            0.10982058197259903,
            0.10855817794799805
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.07940673828125,
          "mean_weights": [
            0.12457560002803802,
            0.12431882321834564,
            0.12479400634765625,
            0.12437158077955246,
            0.12397614866495132,
            0.12488298863172531,
            0.12619814276695251,
            0.12688270211219788
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.07299542427063,
          "mean_weights": [
            0.13274571299552917,
            0.14826728403568268,
            0.13701492547988892,
            0.12918588519096375,
            0.12125477194786072,
            0.1169566735625267,
            0.10889489948749542,
            0.10567984730005264
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079298734664917,
          "mean_weights": [
            0.12391229718923569,
            0.12222036719322205,
            0.12298265099525452,
            0.12416766583919525,
            0.125657856464386,
            0.12553520500659943,
            0.126623272895813,
            0.1289007067680359
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.189734935760498,
          "mean_weights": [
            0.1203557699918747,
            0.13466420769691467,
            0.12461012601852417,
            0.11655421555042267,
            0.10905998200178146,
            0.10487955063581467,
            0.09900732338428497,
            0.0950026735663414,
            0.0958661437034607
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.197068691253662,
          "mean_weights": [
            0.1099812239408493,
            0.10853179544210434,
            0.10949578881263733,
            0.11001666635274887,
            0.1107073426246643,
            0.11118210852146149,
            0.11218173801898956,
            0.11490923166275024,
            0.11299411952495575
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1907877922058105,
          "mean_weights": [
            0.11948458850383759,
            0.13228623569011688,
            0.12348943203687668,
            0.11669500172138214,
            0.11027230322360992,
            0.1054498627781868,
            0.10031206905841827,
            0.09697622060775757,
            0.09503427147865295
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1971726417541504,
          "mean_weights": [
            0.11100216209888458,
            0.11012113094329834,
            0.1099993959069252,
            0.11000204086303711,
            0.10997181385755539,
            0.11260797083377838,
            0.11264219880104065,
            0.11185945570468903,
            0.11179381608963013
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.190375804901123,
          "mean_weights": [
            0.11983220279216766,
            0.13351142406463623,
            0.12386792153120041,
            0.11615028977394104,
            0.10967587679624557,
            0.10584976524114609,
            0.10062196850776672,
            0.09596225619316101,
            0.09452830255031586
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1969356536865234,
          "mean_weights": [
            0.10960094630718231,
            0.10724944621324539,
            0.10892992466688156,
            0.1096237450838089,
            0.11032412201166153,
            0.11112386733293533,
            0.11386257410049438,
            0.11429048329591751,
            0.11499489098787308
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.188795566558838,
          "mean_weights": [
            0.12045153975486755,
            0.13515985012054443,
            0.12505559623241425,
            0.11722971498966217,
            0.11039333790540695,
            0.10568177700042725,
            0.09951679408550262,
            0.09492656588554382,
            0.09158482402563095
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.028617795556783676,
        0.014952295459806919,
        0.015273182652890682,
        0.014778315089643002,
        0.015790818259119987,
        0.01543855108320713,
        0.01658804342150688,
        0.016689632087945938,
        0.017823126167058945,
        0.01178102195262909,
        0.012884888797998428,
        0.012982882559299469,
        0.013985250145196915,
        0.014135614968836308,
        0.015267650596797466,
        0.015412751585245132,
        0.016450626775622368,
        0.012501097284257412,
        0.013461960479617119,
        0.013585822656750679,
        0.014477741904556751,
        0.014657454565167427,
        0.01562186423689127,
        0.01575874350965023,
        0.016906939446926117,
        0.013761760666966438,
        0.014646334573626518,
        0.014892088249325752,
        0.015772713348269463,
        0.016054842621088028,
        0.016963548958301544,
        0.017220331355929375,
        0.018231164664030075,
        0.015453077852725983,
        0.016293151304125786,
        0.016655873507261276,
        0.017603756859898567,
        0.017766626551747322,
        0.018859725445508957,
        0.018939858302474022,
        0.020210443064570427,
        0.017597980797290802,
        0.018590020015835762,
        0.018625281751155853,
        0.019812360405921936,
        0.01988193951547146,
        0.021170100197196007,
        0.0208931565284729,
        0.022594628855586052,
        0.019704710692167282,
        0.021153371781110764,
        0.021027294918894768,
        0.022732628509402275,
        0.02250491827726364,
        0.024445844814181328,
        0.023921677842736244,
        0.02632913924753666,
        0.022959763184189796,
        0.02505173534154892,
        0.02448389306664467,
        0.026562821120023727,
        0.025734128430485725,
        0.028511708602309227,
        0.027005694806575775
      ],
      "loss_nats": 3.5924854278564453,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 2.1922647953033447,
        "mean_weights": [
          0.09991011023521423,
          0.0929517149925232,
          0.10029369592666626,
          0.10801893472671509,
          0.11530699580907822,
          0.11817377060651779,
          0.12114627659320831,
          0.12246126681566238,
          0.12173722684383392
        ],
        "sources": 9
      },
      "step": 100,
      "stream_state_rms_by_sublayer": [
        0.002723898272961378,
        0.0055193523876369,
        0.006246842443943024,
        0.009546536952257156,
        0.009986688382923603,
        0.013977172784507275,
        0.014312135986983776,
        0.018744563683867455,
        0.0027809101156890392,
        0.00660830270498991,
        0.007167908828705549,
        0.011765030212700367,
        0.012327533215284348,
        0.017156247049570084,
        0.017696699127554893,
        0.022267481312155724,
        0.003559867851436138,
        0.007449587807059288,
        0.008642485365271568,
        0.013175037689507008,
        0.014099182561039925,
        0.019073888659477234,
        0.020258057862520218,
        0.025712018832564354,
        0.004887574352324009,
        0.008972279727458954,
        0.011225050315260887,
        0.01624389924108982,
        0.01857108250260353,
        0.023743286728858948,
        0.026577936485409737,
        0.03214457631111145,
        0.005625482182949781,
        0.00991375558078289,
        0.013686468824744225,
        0.019281653687357903,
        0.023031942546367645,
        0.02868141047656536,
        0.03310495615005493,
        0.03889070823788643,
        0.008495661430060863,
        0.0127727584913373,
        0.017283262684941292,
        0.022841427475214005,
        0.029314789921045303,
        0.034929387271404266,
        0.04071469604969025,
        0.04634999856352806,
        0.00961991585791111,
        0.014845813624560833,
        0.023407738655805588,
        0.029589947313070297,
        0.0400763601064682,
        0.046351123601198196,
        0.05648425966501236,
        0.06254568696022034,
        0.014384749345481396,
        0.01947367936372757,
        0.03149676322937012,
        0.0370744951069355,
        0.04818879812955856,
        0.05507569760084152,
        0.06738673150539398,
        0.07403328269720078
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.0001494410098530352,
        0.00015486938355024904,
        0.00014897517394274473,
        0.0001431136333849281,
        0.0001401594199705869,
        0.00012453079398255795,
        0.00011206007184227929,
        0.00010208225285168737,
        9.998067253036425e-05,
        9.082697215490043e-05,
        8.336143946507946e-05,
        7.669178739888594e-05,
        7.405834912788123e-05,
        6.847811891930178e-05,
        6.22319639660418e-05,
        5.739524931414053e-05,
        5.947135286987759e-05,
        5.3890027629677206e-05,
        4.890635682386346e-05,
        4.4437034375732765e-05,
        4.31341104558669e-05,
        3.886142803821713e-05,
        3.5836677852785215e-05,
        3.340865805512294e-05,
        3.223997919121757e-05,
        3.0436871384154074e-05,
        2.9292581530171447e-05,
        2.8587397537194192e-05,
        2.2913789507583715e-05,
        2.2530648493557237e-05,
        2.244284223706927e-05,
        2.2519136109622195e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00013440396742225857,
        "first_to_last_ratio": 5.096772825661455,
        "imbalance_abs_log_ratio": 1.6286075601589056,
        "last_quartile_mean": 2.6370405748821213e-05,
        "mean": 7.053641201082428e-05,
        "normalized": [
          2.1186363977530145,
          2.195594858531836,
          2.112032207136982,
          2.0289327073081966,
          1.9870506023056362,
          1.7654824002594274,
          1.5886840377574483,
          1.4472277500593902,
          1.4174334883240383,
          1.287660792002894,
          1.1818213754945048,
          1.0872652182409996,
          1.0499307664886113,
          0.9708194245660434,
          0.8822672176250175,
          0.8136967514782698,
          0.8431298271983457,
          0.7640029609304118,
          0.6933490863748281,
          0.6299871670381194,
          0.6115155169679979,
          0.5509413780821966,
          0.5080592679889337,
          0.47363704932987183,
          0.4570686014801849,
          0.43150580695093105,
          0.4152831239229507,
          0.4052856775987878,
          0.32485051130850606,
          0.319418692435047,
          0.31817385655546626,
          0.3192554805051111
        ],
        "population_cv": 0.606636113722199
      },
      "activation_output_rms_by_block": [
        0.021243074908852577,
        0.03624089062213898,
        0.04951142147183418,
        0.05851924046874046,
        0.017167888581752777,
        0.02674153819680214,
        0.03352004662156105,
        0.03946679085493088,
        0.020600929856300354,
        0.027865095064044,
        0.035815127193927765,
        0.04293550178408623,
        0.021564343944191933,
        0.029616132378578186,
        0.035480253398418427,
        0.04259813204407692,
        0.021436138078570366,
        0.03586751967668533,
        0.04331304505467415,
        0.051551222801208496,
        0.025608548894524574,
        0.04091007262468338,
        0.05466190353035927,
        0.07478146255016327,
        0.02793189324438572,
        0.05498877540230751,
        0.08125203847885132,
        0.10916247963905334,
        0.026290444657206535,
        0.05761713907122612,
        0.08842677623033524,
        0.12436132878065109
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.03530136146582663,
        "first_to_last_ratio": 0.495430868506741,
        "imbalance_abs_log_ratio": 0.7023274536001348,
        "last_quartile_mean": 0.07125385943800211,
        "mean": 0.04553272487828508,
        "normalized": [
          0.4665452148018396,
          0.7959306349228079,
          1.0873810342821493,
          1.2852127920121195,
          0.37704505117242126,
          0.5873037088881845,
          0.7361748437231577,
          0.8667785853016872,
          0.452442279950724,
          0.6119795188741951,
          0.7865799222354093,
          0.9429591991003927,
          0.4736009980038805,
          0.6504361963345259,
          0.7792253482140983,
          0.9355498085815704,
          0.47078531179216626,
          0.7877305777891374,
          0.9512508897821419,
          1.1321796123340224,
          0.5624207416309823,
          0.8984762658953829,
          1.2004970859195814,
          1.6423673907077576,
          0.6134465556157976,
          1.2076759198861848,
          1.784475642431869,
          2.397451062523908,
          0.5773966905667149,
          1.2654006371295428,
          1.9420488553389141,
          2.7312516242567333
        ],
        "population_cv": 0.5563959277170655
      },
      "bits_per_byte": 3.0745070201808953,
      "branch_output_rms_by_sublayer": [
        0.01049753837287426,
        0.01706349104642868,
        0.013886291533708572,
        0.012710319831967354,
        0.01245343592017889,
        0.013922877609729767,
        0.01395952794700861,
        0.011416541412472725,
        0.014544809237122536,
        0.008325628004968166,
        0.014345151372253895,
        0.00801781378686428,
        0.015135552734136581,
        0.007913725450634956,
        0.014776745811104774,
        0.008828459307551384,
        0.01795734278857708,
        0.008378393016755581,
        0.013709688559174538,
        0.008136541582643986,
        0.016144096851348877,
        0.008199729025363922,
        0.018440017476677895,
        0.009040302596986294,
        0.018456056714057922,
        0.00936646107584238,
        0.012218938209116459,
        0.009016151539981365,
        0.01701919361948967,
        0.009497544728219509,
        0.01305371057242155,
        0.009566655382514,
        0.017418840900063515,
        0.011108046397566795,
        0.026864025741815567,
        0.0114391278475523,
        0.035265542566776276,
        0.012216469272971153,
        0.027374446392059326,
        0.01392423827201128,
        0.01969025284051895,
        0.015176027081906796,
        0.018602686002850533,
        0.017532862722873688,
        0.0133047541603446,
        0.018514130264520645,
        0.015662170946598053,
        0.022848375141620636,
        0.012774706818163395,
        0.0230264812707901,
        0.016374029219150543,
        0.02426965720951557,
        0.010044055990874767,
        0.025484448298811913,
        0.006929588038474321,
        0.027965376153588295,
        0.005331806372851133,
        0.02452327311038971,
        0.01223068032413721,
        0.02779858186841011,
        0.00525518087670207,
        0.03124845214188099,
        0.010387555696070194,
        0.03395458683371544
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0004975869912568777,
        0.0004115293940014644,
        0.00043817390286628833,
        0.00039988528646951293,
        0.0003968514715139995,
        0.00031053290415162903,
        0.00024154094413850278,
        0.00022869177850848996,
        0.0002476899080691786,
        0.00021028907646624228,
        0.0002010354933691209,
        0.0001845208571901889,
        0.0002273114134138899,
        0.0001715437435350543,
        0.00015668460984383524,
        0.00015894955615086492,
        0.00016556491904516564,
        0.0001408361881569501,
        0.00014198814555526783,
        0.00012254378127881199,
        0.00011450446233020633,
        0.0001100879077446283,
        0.00010549256620556044,
        9.975908855450905e-05,
        0.00010071115476761671,
        9.794352566848755e-05,
        9.525064309078406e-05,
        8.226123370955787e-05,
        6.340888944803813e-05,
        6.739220497758678e-05,
        6.650164980556211e-05,
        7.228417264196778e-05
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.0003655990841133456,
        "first_to_last_ratio": 4.5292712934136725,
        "imbalance_abs_log_ratio": 1.5105610641741143,
        "last_quartile_mean": 8.071918426370012e-05,
        "mean": 0.0001915421207476825,
        "normalized": [
          2.5977941004023166,
          2.148505991241321,
          2.28761121134026,
          2.0877146233348847,
          2.071875731379053,
          1.621225153712757,
          1.2610330469123472,
          1.1939503312158923,
          1.293135458155751,
          1.0978738026152224,
          1.0495628459389565,
          0.963343501000792,
          1.186743743499249,
          0.8955927963282188,
          0.8180164719499744,
          0.8298412669255573,
          0.8643786463201141,
          0.7352752888356756,
          0.7412894093529857,
          0.6397745874404216,
          0.5978030413531993,
          0.574745164744451,
          0.5507538801062211,
          0.5208206328983963,
          0.5257911647552604,
          0.5113419716048152,
          0.4972830138821386,
          0.4294681158820424,
          0.33104410246958865,
          0.3518401316353921,
          0.3471907356250086,
          0.3773800371417385
        ],
        "population_cv": 0.6210714087578444
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.6506600379943848,
          "mean_weights": [
            0.3747701942920685,
            0.6252298355102539
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6806463003158569,
          "mean_weights": [
            0.45339685678482056,
            0.5466030836105347
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6869370937347412,
          "mean_weights": [
            0.4668571949005127,
            0.5331428050994873
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6791321635246277,
          "mean_weights": [
            0.4727538824081421,
            0.5272461175918579
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6804035902023315,
          "mean_weights": [
            0.4487338662147522,
            0.5512661933898926
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.682766318321228,
          "mean_weights": [
            0.4646179676055908,
            0.5353820323944092
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6873732805252075,
          "mean_weights": [
            0.4883750379085541,
            0.5116249322891235
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6835889220237732,
          "mean_weights": [
            0.46665582060813904,
            0.5333442091941833
          ],
          "sources": 2
        },
        {
          "entropy_mean": 1.0856965780258179,
          "mean_weights": [
            0.27463093400001526,
            0.3524433374404907,
            0.372925728559494
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.077350378036499,
          "mean_weights": [
            0.2904614210128784,
            0.3131614923477173,
            0.3963770568370819
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0850789546966553,
          "mean_weights": [
            0.27783387899398804,
            0.34068456292152405,
            0.3814815878868103
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.062077522277832,
          "mean_weights": [
            0.28390786051750183,
            0.29115602374076843,
            0.42493611574172974
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0936429500579834,
          "mean_weights": [
            0.30466002225875854,
            0.3333379328250885,
            0.36200207471847534
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0627415180206299,
          "mean_weights": [
            0.2850727438926697,
            0.29934102296829224,
            0.4155862331390381
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.087097406387329,
          "mean_weights": [
            0.28865373134613037,
            0.3320052921772003,
            0.3793409764766693
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0500720739364624,
          "mean_weights": [
            0.2745887339115143,
            0.286259263753891,
            0.4391520023345947
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.3743348121643066,
          "mean_weights": [
            0.2166871875524521,
            0.23293715715408325,
            0.2495419979095459,
            0.30083364248275757
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3609715700149536,
          "mean_weights": [
            0.23144879937171936,
            0.21839870512485504,
            0.24986028671264648,
            0.3002921938896179
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3810515403747559,
          "mean_weights": [
            0.2385442852973938,
            0.23454180359840393,
            0.24874086678028107,
            0.27817302942276
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.348030686378479,
          "mean_weights": [
            0.23874622583389282,
            0.21300309896469116,
            0.22721335291862488,
            0.32103732228279114
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3825427293777466,
          "mean_weights": [
            0.24534732103347778,
            0.2375141680240631,
            0.25072187185287476,
            0.26641663908958435
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3499886989593506,
          "mean_weights": [
            0.24629786610603333,
            0.22150886058807373,
            0.2235339879989624,
            0.30865928530693054
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3807358741760254,
          "mean_weights": [
            0.24433431029319763,
            0.23110222816467285,
            0.23992134630680084,
            0.2846421003341675
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3399771451950073,
          "mean_weights": [
            0.24593281745910645,
            0.21928201615810394,
            0.21277041733264923,
            0.3220147490501404
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.6075894832611084,
          "mean_weights": [
            0.19650746881961823,
            0.19567455351352692,
            0.18970930576324463,
            0.21128275990486145,
            0.20682592689990997
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5706448554992676,
          "mean_weights": [
            0.1975565403699875,
            0.1772933304309845,
            0.16386017203330994,
            0.21728253364562988,
            0.2440074384212494
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6042816638946533,
          "mean_weights": [
            0.21478912234306335,
            0.18552109599113464,
            0.17930489778518677,
            0.20310312509536743,
            0.2172817587852478
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5764570236206055,
          "mean_weights": [
            0.20157726109027863,
            0.17631688714027405,
            0.1703747808933258,
            0.19828173518180847,
            0.25344932079315186
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6011263132095337,
          "mean_weights": [
            0.20813238620758057,
            0.18490281701087952,
            0.17333751916885376,
            0.20533671975135803,
            0.22829055786132812
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5817476511001587,
          "mean_weights": [
            0.20727968215942383,
            0.18177060782909393,
            0.17236655950546265,
            0.18925222754478455,
            0.24933092296123505
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.603710412979126,
          "mean_weights": [
            0.2016504406929016,
            0.1979413628578186,
            0.17889465391635895,
            0.19759604334831238,
            0.22391751408576965
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5885388851165771,
          "mean_weights": [
            0.20988406240940094,
            0.18461555242538452,
            0.17252585291862488,
            0.19482356309890747,
            0.2381509691476822
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.783994197845459,
          "mean_weights": [
            0.16140900552272797,
            0.16211673617362976,
            0.15125815570354462,
            0.1585075855255127,
            0.17266258597373962,
            0.19404590129852295
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.765566349029541,
          "mean_weights": [
            0.1630604863166809,
            0.1509958952665329,
            0.14303800463676453,
            0.14991191029548645,
            0.16761469841003418,
            0.22537899017333984
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7848252058029175,
          "mean_weights": [
            0.16126418113708496,
            0.1664336919784546,
            0.15361233055591583,
            0.1546509563922882,
            0.16693615913391113,
            0.19710269570350647
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7731435298919678,
          "mean_weights": [
            0.16123875975608826,
            0.15981441736221313,
            0.1509810984134674,
            0.14913737773895264,
            0.16297483444213867,
            0.2158534824848175
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7815005779266357,
          "mean_weights": [
            0.15527218580245972,
            0.17254948616027832,
            0.15193141996860504,
            0.15522462129592896,
            0.1650869995355606,
            0.19993525743484497
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7801456451416016,
          "mean_weights": [
            0.1661754697561264,
            0.16477231681346893,
            0.15300315618515015,
            0.15292593836784363,
            0.1582789272069931,
            0.2048441767692566
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7743862867355347,
          "mean_weights": [
            0.15205824375152588,
            0.1771804541349411,
            0.15399661660194397,
            0.15122173726558685,
            0.15813913941383362,
            0.20740382373332977
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.781898856163025,
          "mean_weights": [
            0.17058727145195007,
            0.1644459068775177,
            0.15141303837299347,
            0.15486885607242584,
            0.15909278392791748,
            0.19959214329719543
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.9389331340789795,
          "mean_weights": [
            0.12895603477954865,
            0.15703952312469482,
            0.13832643628120422,
            0.13419194519519806,
            0.1312965601682663,
            0.15331263840198517,
            0.15687686204910278
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.941878080368042,
          "mean_weights": [
            0.14007402956485748,
            0.15248695015907288,
            0.14026451110839844,
            0.13737857341766357,
            0.1305495798587799,
            0.14631187915802002,
            0.1529344916343689
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.936349630355835,
          "mean_weights": [
            0.12599323689937592,
            0.16029544174671173,
            0.1399340182542801,
            0.13338115811347961,
            0.12978437542915344,
            0.14873966574668884,
            0.16187208890914917
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9385986328125,
          "mean_weights": [
            0.1397915780544281,
            0.15300613641738892,
            0.14345945417881012,
            0.13018865883350372,
            0.1257915198802948,
            0.14249205589294434,
            0.1652706116437912
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.939380407333374,
          "mean_weights": [
            0.12855647504329681,
            0.15473899245262146,
            0.1372847855091095,
            0.13797816634178162,
            0.13512183725833893,
            0.15090687572956085,
            0.15541282296180725
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9388402700424194,
          "mean_weights": [
            0.14282521605491638,
            0.15391701459884644,
            0.1462559849023819,
            0.13105326890945435,
            0.12675246596336365,
            0.13615548610687256,
            0.16304057836532593
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9313902854919434,
          "mean_weights": [
            0.11694864928722382,
            0.17515113949775696,
            0.15024805068969727,
            0.1394161432981491,
            0.13194763660430908,
            0.1451682150363922,
            0.14112016558647156
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9366493225097656,
          "mean_weights": [
            0.14114168286323547,
            0.16076743602752686,
            0.14226865768432617,
            0.12737974524497986,
            0.12711969017982483,
            0.1412842869758606,
            0.16003848612308502
          ],
          "sources": 7
        },
        {
          "entropy_mean": 2.064785957336426,
          "mean_weights": [
            0.106104776263237,
            0.15484434366226196,
            0.13487765192985535,
            0.12357068061828613,
            0.12023378908634186,
            0.12704278528690338,
            0.11963208019733429,
            0.11369390785694122
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.062044143676758,
          "mean_weights": [
            0.11719764769077301,
            0.14862585067749023,
            0.12573552131652832,
            0.12179906666278839,
            0.11081404983997345,
            0.1260838657617569,
            0.12823458015918732,
            0.12150939553976059
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.054962158203125,
          "mean_weights": [
            0.10514847934246063,
            0.16456037759780884,
            0.14191412925720215,
            0.1262327879667282,
            0.1181878000497818,
            0.12522782385349274,
            0.1187765821814537,
            0.09995201230049133
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0610406398773193,
          "mean_weights": [
            0.11198848485946655,
            0.15103809535503387,
            0.13588255643844604,
            0.12388211488723755,
            0.11551118642091751,
            0.12874530255794525,
            0.12429477274417877,
            0.10865749418735504
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0338308811187744,
          "mean_weights": [
            0.0961676836013794,
            0.17635446786880493,
            0.14848342537879944,
            0.13749772310256958,
            0.12027012556791306,
            0.12292921543121338,
            0.10873906314373016,
            0.08955828845500946
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.068699598312378,
          "mean_weights": [
            0.11418519914150238,
            0.14292436838150024,
            0.1359846293926239,
            0.13056153059005737,
            0.12100572884082794,
            0.11985942721366882,
            0.12221725285053253,
            0.1132618635892868
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.036879539489746,
          "mean_weights": [
            0.09746269136667252,
            0.17786666750907898,
            0.15074899792671204,
            0.13200224936008453,
            0.11842608451843262,
            0.12164927273988724,
            0.1129462867975235,
            0.08889776468276978
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.06752347946167,
          "mean_weights": [
            0.11505836993455887,
            0.14705273509025574,
            0.1377231776714325,
            0.12483140081167221,
            0.11869609355926514,
            0.1215740442276001,
            0.12386028468608856,
            0.1112038791179657
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.1517844200134277,
          "mean_weights": [
            0.08786984533071518,
            0.16287443041801453,
            0.13886377215385437,
            0.12039706110954285,
            0.10935483872890472,
            0.1119438111782074,
            0.10067413747310638,
            0.07631799578666687,
            0.0917041003704071
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.177518129348755,
          "mean_weights": [
            0.10418819636106491,
            0.13366681337356567,
            0.12014994025230408,
            0.11398349702358246,
            0.10732515156269073,
            0.12029910087585449,
            0.11035183072090149,
            0.09321349114179611,
            0.09682197868824005
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1535391807556152,
          "mean_weights": [
            0.08976081013679504,
            0.163266122341156,
            0.13892531394958496,
            0.12143255770206451,
            0.10986581444740295,
            0.11029069125652313,
            0.10418520867824554,
            0.08120252937078476,
            0.0810709297657013
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.191593647003174,
          "mean_weights": [
            0.10739251971244812,
            0.12338980287313461,
            0.11759916692972183,
            0.1066875234246254,
            0.1060015857219696,
            0.1124890148639679,
            0.11740898340940475,
            0.10738331079483032,
            0.10164809972047806
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1465134620666504,
          "mean_weights": [
            0.08895281702280045,
            0.1655423492193222,
            0.14156755805015564,
            0.12290289252996445,
            0.11169517785310745,
            0.11284246295690536,
            0.1037452220916748,
            0.0791342705488205,
            0.07361726462841034
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1857833862304688,
          "mean_weights": [
            0.10998444259166718,
            0.12038750946521759,
            0.11211743205785751,
            0.11047163605690002,
            0.10957512259483337,
            0.12139561027288437,
            0.11392621695995331,
            0.10294321179389954,
            0.09919881820678711
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1400556564331055,
          "mean_weights": [
            0.08778931945562363,
            0.16802933812141418,
            0.1466498225927353,
            0.12668846547603607,
            0.11206841468811035,
            0.11208955943584442,
            0.10068599879741669,
            0.07686632871627808,
            0.0691327303647995
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.04377923160791397,
        0.017109708860516548,
        0.023813940584659576,
        0.02606838569045067,
        0.029579024761915207,
        0.030872320756316185,
        0.034570932388305664,
        0.034764163196086884,
        0.037983279675245285,
        0.025188947096467018,
        0.02627667225897312,
        0.026965361088514328,
        0.02788778953254223,
        0.028485212475061417,
        0.03023744560778141,
        0.030349206179380417,
        0.031768180429935455,
        0.022602196782827377,
        0.02424551732838154,
        0.023387571796774864,
        0.025113960728049278,
        0.02450813725590706,
        0.026638666167855263,
        0.025169003754854202,
        0.027698444202542305,
        0.020798001438379288,
        0.022435806691646576,
        0.02118327096104622,
        0.022681118920445442,
        0.021654823794960976,
        0.023235918954014778,
        0.022377775982022285,
        0.023750189691781998,
        0.018887808546423912,
        0.019677624106407166,
        0.01950881816446781,
        0.020757248625159264,
        0.020426612347364426,
        0.02137788012623787,
        0.021234560757875443,
        0.02207154594361782,
        0.01860968954861164,
        0.019271831959486008,
        0.019469313323497772,
        0.020465917885303497,
        0.020844420418143272,
        0.021911408752202988,
        0.02213398739695549,
        0.024193286895751953,
        0.0209572184830904,
        0.02259150519967079,
        0.022487536072731018,
        0.024306390434503555,
        0.023487187922000885,
        0.02611992135643959,
        0.024889562278985977,
        0.02851196378469467,
        0.023824406787753105,
        0.027104467153549194,
        0.025868752971291542,
        0.03068944811820984,
        0.0271493811160326,
        0.03286270797252655,
        0.028410611674189568
      ],
      "loss_nats": 2.1310858726501465,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 2.1613364219665527,
        "mean_weights": [
          0.1255333125591278,
          0.08712290227413177,
          0.09080196917057037,
          0.08838342130184174,
          0.09530800580978394,
          0.09807588160037994,
          0.13817133009433746,
          0.15537576377391815,
          0.12122740596532822
        ],
        "sources": 9
      },
      "step": 500,
      "stream_state_rms_by_sublayer": [
        0.01049753837287426,
        0.021243074908852577,
        0.028129518032073975,
        0.03624089062213898,
        0.040982723236083984,
        0.04951142147183418,
        0.05135732889175415,
        0.05851924046874046,
        0.014544809237122536,
        0.017167888581752777,
        0.023477500304579735,
        0.02674153819680214,
        0.029818039387464523,
        0.03352004662156105,
        0.03540094941854477,
        0.03946679085493088,
        0.01795734278857708,
        0.020600929856300354,
        0.024303091689944267,
        0.027865095064044,
        0.031844362616539,
        0.035815127193927765,
        0.03825921565294266,
        0.04293550178408623,
        0.018456056714057922,
        0.021564343944191933,
        0.02560909278690815,
        0.029616132378578186,
        0.0305727981030941,
        0.035480253398418427,
        0.037264734506607056,
        0.04259813204407692,
        0.017418840900063515,
        0.021436138078570366,
        0.02998829260468483,
        0.03586751967668533,
        0.03708711639046669,
        0.04331304505467415,
        0.04352690279483795,
        0.051551222801208496,
        0.01969025284051895,
        0.025608548894524574,
        0.030230700969696045,
        0.04091007262468338,
        0.041474223136901855,
        0.05466190353035927,
        0.056570712476968765,
        0.07478146255016327,
        0.012774706818163395,
        0.02793189324438572,
        0.03524070605635643,
        0.05498877540230751,
        0.05920406058430672,
        0.08125203847885132,
        0.08430205285549164,
        0.10916247963905334,
        0.005331806372851133,
        0.026290444657206535,
        0.032364487648010254,
        0.05761713907122612,
        0.059356335550546646,
        0.08842677623033524,
        0.09272631257772446,
        0.12436132878065109
      ]
    },
    {
      "activation_grad_rms_by_block": [
        6.294298509601504e-05,
        6.491570093203336e-05,
        6.380436389008537e-05,
        6.04347005719319e-05,
        7.516580808442086e-05,
        7.751413795631379e-05,
        6.965622014831752e-05,
        6.359309190884233e-05,
        6.543538620462641e-05,
        6.0226317145861685e-05,
        5.608194260275923e-05,
        5.3055387979839e-05,
        4.572631951305084e-05,
        4.3727712181862444e-05,
        4.314268517191522e-05,
        4.164052370470017e-05,
        7.694400119362399e-05,
        7.326623017434031e-05,
        6.749018211849034e-05,
        6.173704605316743e-05,
        0.00010233377543045208,
        8.934178913477808e-05,
        7.613583147758618e-05,
        6.664563989033923e-05,
        5.163387322681956e-05,
        4.2875533836195245e-05,
        3.8466609112219885e-05,
        3.59200312232133e-05,
        2.308324837940745e-05,
        2.0679148292401806e-05,
        1.9936938770115376e-05,
        1.9895105651812628e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 6.725337607349502e-05,
        "first_to_last_ratio": 2.130880302861834,
        "imbalance_abs_log_ratio": 0.7565351820700446,
        "last_quartile_mean": 3.1561311061523156e-05,
        "mean": 5.6670258345548064e-05,
        "normalized": [
          1.1106881622493918,
          1.145498588275503,
          1.1258880010928651,
          1.0664271230850946,
          1.3263713679598177,
          1.367809856868302,
          1.2291495077292092,
          1.122159908308202,
          1.154668923611267,
          1.062750001572791,
          0.9896186154790125,
          0.9362122130506743,
          0.8068839078557516,
          0.7716166020495587,
          0.7612932503122151,
          0.7347861986228512,
          1.357749257546284,
          1.2928515294142118,
          1.190927729797305,
          1.089408233799192,
          1.8057756999530474,
          1.5765198843812338,
          1.3434883429213678,
          1.176024987992221,
          0.9111282484717285,
          0.7565791137700626,
          0.678779490957478,
          0.6338427293588494,
          0.4073256246452393,
          0.3649030178459797,
          0.3518060328673549,
          0.35106784815593767
        ],
        "population_cv": 0.34914284904214865
      },
      "activation_output_rms_by_block": [
        0.10241243988275528,
        0.1819411665201187,
        0.24157075583934784,
        0.26444777846336365,
        0.10636615753173828,
        0.13034582138061523,
        0.14029134809970856,
        0.15042904019355774,
        0.04812335968017578,
        0.07954440265893936,
        0.10673297941684723,
        0.14089107513427734,
        0.06885477155447006,
        0.10946234315633774,
        0.13876035809516907,
        0.1687854677438736,
        0.05059739947319031,
        0.07094386219978333,
        0.09166282415390015,
        0.12053482979536057,
        0.048727843910455704,
        0.07109228521585464,
        0.09690473228693008,
        0.11853098124265671,
        0.05259029567241669,
        0.08889466524124146,
        0.12455403059720993,
        0.1728687882423401,
        0.06489895284175873,
        0.14679722487926483,
        0.23576539754867554,
        0.35208994150161743
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.16472556348890066,
        "first_to_last_ratio": 1.0640676779683806,
        "imbalance_abs_log_ratio": 0.0620989960097457,
        "last_quartile_mean": 0.15480741206556559,
        "mean": 0.127700416254811,
        "normalized": [
          0.8019742056157706,
          1.4247499879489567,
          1.8916990478505735,
          2.0708450780276983,
          0.8329350886335192,
          1.020715712629516,
          1.098597422108428,
          1.177983945591802,
          0.3768457542389787,
          0.6228985385624584,
          0.8358076077361742,
          1.1032937813855357,
          0.5391898755997675,
          0.8571807858317286,
          1.086608502655851,
          1.3217299731199175,
          0.396219534415841,
          0.5555491851978249,
          0.7177958133746243,
          0.9438875246486875,
          0.3815793662976406,
          0.5567114603134765,
          0.7588442945524017,
          0.9281957312194052,
          0.411825561848436,
          0.6961188369493074,
          0.9753611949758815,
          1.3537057537651325,
          0.5082125395132666,
          1.1495438244018596,
          1.8462382853806345,
          2.7571557856089037
        ],
        "population_cv": 0.532159231456062
      },
      "bits_per_byte": 2.0533380169036586,
      "branch_output_rms_by_sublayer": [
        0.05045261234045029,
        0.0860176682472229,
        0.09517527371644974,
        0.031032679602503777,
        0.08140470087528229,
        0.011454286053776741,
        0.035610511898994446,
        0.03601690009236336,
        0.1042611226439476,
        0.01171922218054533,
        0.050615254789590836,
        0.008982792496681213,
        0.04264022409915924,
        0.01206296682357788,
        0.03225839138031006,
        0.026524480432271957,
        0.040291935205459595,
        0.024220434948801994,
        0.036216676235198975,
        0.026388581842184067,
        0.02871432714164257,
        0.03031591884791851,
        0.03194425627589226,
        0.032942064106464386,
        0.05663999170064926,
        0.029961559921503067,
        0.03894488140940666,
        0.02937687374651432,
        0.03595346212387085,
        0.031781863421201706,
        0.03414257988333702,
        0.035546254366636276,
        0.03680795058608055,
        0.030760737136006355,
        0.022694969549775124,
        0.03042135201394558,
        0.022243596613407135,
        0.03508196026086807,
        0.02950950525701046,
        0.04756028205156326,
        0.035728659480810165,
        0.0295625701546669,
        0.024367405101656914,
        0.03350908309221268,
        0.0301151555031538,
        0.038334235548973083,
        0.022455131635069847,
        0.043223608285188675,
        0.025910304859280586,
        0.04338310658931732,
        0.025968369096517563,
        0.046161677688360214,
        0.01597527042031288,
        0.050398919731378555,
        0.015805471688508987,
        0.062427736818790436,
        0.013562481850385666,
        0.0627857968211174,
        0.02184087596833706,
        0.08928166329860687,
        0.012465676292777061,
        0.09919292479753494,
        0.01876053772866726,
        0.13455425202846527
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00034319970120439075,
        0.0002207216513437987,
        0.00019628105547348242,
        0.00016658832628724645,
        0.00021517433675904697,
        0.00019244240293185932,
        0.0001402257608174488,
        0.00015224450647613049,
        0.00018347185792801386,
        0.00016427150761043105,
        0.00015319316983403424,
        0.00015483500137268905,
        0.00013970482137555046,
        0.00012521709576159806,
        0.00014171058058500498,
        0.00014304258084243227,
        0.00024403444477986432,
        0.0002443248488541129,
        0.0002486173108247806,
        0.0002822915928542698,
        0.00039666535280576766,
        0.0003607137256748422,
        0.0003593407909791253,
        0.00030791631238161,
        0.0002432000733486066,
        0.00020051667741228585,
        0.0001766506293272091,
        0.00017705575210645458,
        0.00010859706957547057,
        0.0001081942080930384,
        0.00010514694748959741,
        0.00011321633644894032
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.0002033597176616755,
        "first_to_last_ratio": 1.3198987369921267,
        "imbalance_abs_log_ratio": 0.2775550192556754,
        "last_quartile_mean": 0.00015407221172520036,
        "mean": 0.00020340020092372293,
        "normalized": [
          1.6873124984428802,
          1.085159455799021,
          0.9649993194799731,
          0.8190175109498476,
          1.0578865496781857,
          0.9461269067478802,
          0.6894081725614166,
          0.7484973258862397,
          0.9020239758603661,
          0.8076270665633928,
          0.7531613495872759,
          0.7612332764152662,
          0.6868470175599342,
          0.6156193317063423,
          0.6967081642075066,
          0.703256831570558,
          1.1997748461978148,
          1.2012025934317396,
          1.2223061220967748,
          1.3878629006867693,
          1.9501718828415566,
          1.7734187283822467,
          1.7666688102922847,
          1.5138446814862372,
          1.1956727291523621,
          0.985823398903532,
          0.8684879785023163,
          0.8704797306117324,
          0.5339083692262209,
          0.5319277345926137,
          0.516946133839015,
          0.5566186067406962
        ],
        "population_cv": 0.38898494230064556
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.39672940969467163,
          "mean_weights": [
            0.854300856590271,
            0.1456991732120514
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6258258819580078,
          "mean_weights": [
            0.5948886871337891,
            0.40511131286621094
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.5053832530975342,
          "mean_weights": [
            0.778947651386261,
            0.221052348613739
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6737473011016846,
          "mean_weights": [
            0.5406605005264282,
            0.4593394994735718
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6529731750488281,
          "mean_weights": [
            0.5619670748710632,
            0.43803298473358154
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6891955733299255,
          "mean_weights": [
            0.5099825263023376,
            0.49001747369766235
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.48565131425857544,
          "mean_weights": [
            0.7992967367172241,
            0.20070326328277588
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6875144243240356,
          "mean_weights": [
            0.5191382169723511,
            0.4808618128299713
          ],
          "sources": 2
        },
        {
          "entropy_mean": 1.048142671585083,
          "mean_weights": [
            0.3059995770454407,
            0.2586204409599304,
            0.4353799819946289
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.078475832939148,
          "mean_weights": [
            0.2919672429561615,
            0.28997206687927246,
            0.4180607199668884
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0040717124938965,
          "mean_weights": [
            0.27878478169441223,
            0.21860124170780182,
            0.5026139616966248
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0727062225341797,
          "mean_weights": [
            0.3306647837162018,
            0.2443503737449646,
            0.42498481273651123
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.036304235458374,
          "mean_weights": [
            0.3365095853805542,
            0.20819777250289917,
            0.455292671918869
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.071068525314331,
          "mean_weights": [
            0.3291535973548889,
            0.24366843700408936,
            0.42717796564102173
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0272725820541382,
          "mean_weights": [
            0.29237720370292664,
            0.2188395857810974,
            0.48878318071365356
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0856674909591675,
          "mean_weights": [
            0.3748946189880371,
            0.26831528544425964,
            0.35679009556770325
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.3326265811920166,
          "mean_weights": [
            0.26391589641571045,
            0.14978256821632385,
            0.2545396089553833,
            0.3317619264125824
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3771977424621582,
          "mean_weights": [
            0.27525436878204346,
            0.2067744880914688,
            0.23979949951171875,
            0.2781716585159302
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3576545715332031,
          "mean_weights": [
            0.2848266065120697,
            0.17341844737529755,
            0.23106196522712708,
            0.3106929659843445
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3772028684616089,
          "mean_weights": [
            0.2702856659889221,
            0.21318203210830688,
            0.23082655668258667,
            0.2857057452201843
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3277652263641357,
          "mean_weights": [
            0.3119839131832123,
            0.16646917164325714,
            0.1912841796875,
            0.3302627205848694
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3750126361846924,
          "mean_weights": [
            0.2621780037879944,
            0.21051618456840515,
            0.23754313588142395,
            0.2897626757621765
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3568847179412842,
          "mean_weights": [
            0.2628105878829956,
            0.1954234540462494,
            0.21842113137245178,
            0.3233448266983032
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3733241558074951,
          "mean_weights": [
            0.25842776894569397,
            0.22964225709438324,
            0.21348324418067932,
            0.2984467148780823
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.5855276584625244,
          "mean_weights": [
            0.18060272932052612,
            0.1563272476196289,
            0.18859915435314178,
            0.24991817772388458,
            0.2245527058839798
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5906484127044678,
          "mean_weights": [
            0.19773373007774353,
            0.18790484964847565,
            0.15935488045215607,
            0.18813443183898926,
            0.2668721079826355
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5916568040847778,
          "mean_weights": [
            0.2148856818675995,
            0.14808836579322815,
            0.18413493037223816,
            0.22754491865634918,
            0.22534608840942383
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5474352836608887,
          "mean_weights": [
            0.19825294613838196,
            0.1729484647512436,
            0.11353601515293121,
            0.19613605737686157,
            0.3191264867782593
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5919160842895508,
          "mean_weights": [
            0.21136541664600372,
            0.1572093367576599,
            0.18091553449630737,
            0.2257765829563141,
            0.2247331142425537
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.542572021484375,
          "mean_weights": [
            0.2053474485874176,
            0.19789552688598633,
            0.10896679759025574,
            0.1649853140115738,
            0.32280486822128296
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5774884223937988,
          "mean_weights": [
            0.1874420940876007,
            0.14777487516403198,
            0.18392466008663177,
            0.23523849248886108,
            0.24561989307403564
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.585869312286377,
          "mean_weights": [
            0.2123720645904541,
            0.19153764843940735,
            0.14032581448554993,
            0.1918477714061737,
            0.26391667127609253
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.747511625289917,
          "mean_weights": [
            0.13209694623947144,
            0.11971477419137955,
            0.14703533053398132,
            0.17972922325134277,
            0.15732964873313904,
            0.2640940546989441
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.6791677474975586,
          "mean_weights": [
            0.1429731547832489,
            0.128341943025589,
            0.08682706952095032,
            0.12974600493907928,
            0.16802877187728882,
            0.3440830111503601
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7079570293426514,
          "mean_weights": [
            0.12207645177841187,
            0.11454501748085022,
            0.14774824678897858,
            0.1675407588481903,
            0.1342848241329193,
            0.3138047158718109
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.689455270767212,
          "mean_weights": [
            0.14791087806224823,
            0.15263086557388306,
            0.0942135751247406,
            0.12123478949069977,
            0.14674627780914307,
            0.3372636139392853
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.6673545837402344,
          "mean_weights": [
            0.119766965508461,
            0.10584045946598053,
            0.1391725242137909,
            0.15867865085601807,
            0.1261359602212906,
            0.3504054546356201
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7276721000671387,
          "mean_weights": [
            0.16197484731674194,
            0.16130125522613525,
            0.10335841774940491,
            0.12991490960121155,
            0.15279719233512878,
            0.2906533479690552
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.655259609222412,
          "mean_weights": [
            0.12291313707828522,
            0.10535868257284164,
            0.14431551098823547,
            0.1644165962934494,
            0.11736815422773361,
            0.34562790393829346
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7456457614898682,
          "mean_weights": [
            0.17144063115119934,
            0.17272350192070007,
            0.09372985363006592,
            0.13482090830802917,
            0.1837429702281952,
            0.2435421198606491
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.8196223974227905,
          "mean_weights": [
            0.097611203789711,
            0.08662202209234238,
            0.10460364818572998,
            0.12231925129890442,
            0.09176328778266907,
            0.22007866203784943,
            0.27700191736221313
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.8201665878295898,
          "mean_weights": [
            0.12032124400138855,
            0.12977322936058044,
            0.07435770332813263,
            0.09230788052082062,
            0.1133352667093277,
            0.15118572115898132,
            0.31871896982192993
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.7483060359954834,
          "mean_weights": [
            0.09602338075637817,
            0.07951871305704117,
            0.09892410784959793,
            0.11023538559675217,
            0.08284324407577515,
            0.17450374364852905,
            0.35795143246650696
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.851926565170288,
          "mean_weights": [
            0.14006076753139496,
            0.14811867475509644,
            0.08932580053806305,
            0.09198777377605438,
            0.11175547540187836,
            0.12843111157417297,
            0.29032039642333984
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.7523608207702637,
          "mean_weights": [
            0.10785886645317078,
            0.08607444912195206,
            0.09595336019992828,
            0.10649852454662323,
            0.0838143527507782,
            0.14955362677574158,
            0.3702468276023865
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.8693451881408691,
          "mean_weights": [
            0.1378144919872284,
            0.1244172528386116,
            0.09968579560518265,
            0.10974414646625519,
            0.11655184626579285,
            0.130243182182312,
            0.2815432548522949
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.7898914813995361,
          "mean_weights": [
            0.10115306824445724,
            0.09615937620401382,
            0.11218561232089996,
            0.1163020133972168,
            0.09238721430301666,
            0.14564841985702515,
            0.33616429567337036
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9094455242156982,
          "mean_weights": [
            0.14588291943073273,
            0.14179158210754395,
            0.1023377850651741,
            0.11511939018964767,
            0.13270339369773865,
            0.13753530383110046,
            0.22462961077690125
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9082129001617432,
          "mean_weights": [
            0.09145341068506241,
            0.07698438316583633,
            0.08525429666042328,
            0.08438773453235626,
            0.06885559856891632,
            0.11482041329145432,
            0.2519546449184418,
            0.2262895107269287
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.025789499282837,
          "mean_weights": [
            0.11047068238258362,
            0.12069687247276306,
            0.09073689579963684,
            0.09690164029598236,
            0.10987945646047592,
            0.11353722959756851,
            0.15697816014289856,
            0.20079906284809113
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.9302079677581787,
          "mean_weights": [
            0.0971018448472023,
            0.08240298926830292,
            0.0830029845237732,
            0.08926018327474594,
            0.07480385154485703,
            0.10335960984230042,
            0.22211775183677673,
            0.24795079231262207
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.010532855987549,
          "mean_weights": [
            0.10700424760580063,
            0.09611141681671143,
            0.09769981354475021,
            0.10596007108688354,
            0.09634587168693542,
            0.11705268919467926,
            0.17551329731941223,
            0.20431260764598846
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.9408204555511475,
          "mean_weights": [
            0.08590779453516006,
            0.09530991315841675,
            0.09476494789123535,
            0.1030702069401741,
            0.08048167079687119,
            0.12287819385528564,
            0.22555866837501526,
            0.19202858209609985
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0024678707122803,
          "mean_weights": [
            0.10720442980527878,
            0.09982497990131378,
            0.10896851122379303,
            0.1345594823360443,
            0.09260416775941849,
            0.1016254797577858,
            0.1643495261669159,
            0.19086343050003052
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.9389225244522095,
          "mean_weights": [
            0.08632777631282806,
            0.09041796624660492,
            0.09941554069519043,
            0.10805264860391617,
            0.07865813374519348,
            0.11747924983501434,
            0.2661469578742981,
            0.1535017192363739
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.008256673812866,
          "mean_weights": [
            0.1016545370221138,
            0.10481878370046616,
            0.10823093354701996,
            0.1270347535610199,
            0.0919865146279335,
            0.14368976652622223,
            0.20522482693195343,
            0.11735989153385162
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.094825267791748,
          "mean_weights": [
            0.08519881963729858,
            0.09903685003519058,
            0.08900875598192215,
            0.09697237610816956,
            0.071251779794693,
            0.09201781451702118,
            0.19253811240196228,
            0.1091965064406395,
            0.16477900743484497
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.140371322631836,
          "mean_weights": [
            0.10001853853464127,
            0.10924500226974487,
            0.10232416540384293,
            0.11669869720935822,
            0.10123330354690552,
            0.1287832260131836,
            0.14476822316646576,
            0.09069216251373291,
            0.10623672604560852
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.117004632949829,
          "mean_weights": [
            0.09184888750314713,
            0.09620077908039093,
            0.09620209038257599,
            0.10222946852445602,
            0.06924223154783249,
            0.0821654349565506,
            0.17978665232658386,
            0.14873988926410675,
            0.13358455896377563
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1423683166503906,
          "mean_weights": [
            0.09596475213766098,
            0.07681103050708771,
            0.11225531250238419,
            0.11592596024274826,
            0.0810609757900238,
            0.11778091639280319,
            0.1649385243654251,
            0.12366649508476257,
            0.11159604042768478
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.093705415725708,
          "mean_weights": [
            0.08838655799627304,
            0.09557235240936279,
            0.09992480278015137,
            0.10721968859434128,
            0.07486072182655334,
            0.09290309250354767,
            0.21980023384094238,
            0.12490785866975784,
            0.09642469882965088
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1627297401428223,
          "mean_weights": [
            0.0994885265827179,
            0.09216944873332977,
            0.11588040739297867,
            0.12231601774692535,
            0.09873615205287933,
            0.13436099886894226,
            0.13960200548171997,
            0.1002899631857872,
            0.09715649485588074
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1300058364868164,
          "mean_weights": [
            0.089942067861557,
            0.1065598577260971,
            0.11013364791870117,
            0.12377548217773438,
            0.08726659417152405,
            0.11411209404468536,
            0.1931532621383667,
            0.10132836550474167,
            0.07372863590717316
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.05571148172020912,
        0.04887383058667183,
        0.061161335557699203,
        0.06501953303813934,
        0.09595561027526855,
        0.11691275984048843,
        0.12880638241767883,
        0.07614059001207352,
        0.13726800680160522,
        0.09499973803758621,
        0.10176635533571243,
        0.09968593716621399,
        0.09925490617752075,
        0.09547581523656845,
        0.10055620223283768,
        0.09975490719079971,
        0.0990053340792656,
        0.06202486902475357,
        0.07309409230947495,
        0.06455004960298538,
        0.07328634709119797,
        0.062386661767959595,
        0.07434380799531937,
        0.07036563754081726,
        0.07861478626728058,
        0.05747222527861595,
        0.06236656382679939,
        0.057312749326229095,
        0.06600815057754517,
        0.06149744614958763,
        0.07462523877620697,
        0.06584152579307556,
        0.07498960196971893,
        0.04952141270041466,
        0.05233807489275932,
        0.04880477488040924,
        0.05626988410949707,
        0.0513896569609642,
        0.061378348618745804,
        0.057161398231983185,
        0.06856362521648407,
        0.04507989436388016,
        0.04801172763109207,
        0.04335150495171547,
        0.0518198125064373,
        0.047289349138736725,
        0.05105668306350708,
        0.05114771053195,
        0.054886870086193085,
        0.042588189244270325,
        0.04603559896349907,
        0.044108107686042786,
        0.04718422889709473,
        0.05026337504386902,
        0.05210597813129425,
        0.05349528789520264,
        0.05379685014486313,
        0.04645879194140434,
        0.05137190222740173,
        0.05323651432991028,
        0.05757243186235428,
        0.05820684880018234,
        0.06228902190923691,
        0.058245208114385605
      ],
      "loss_nats": 1.4232654571533203,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 1.9258099794387817,
        "mean_weights": [
          0.11242504417896271,
          0.0684729665517807,
          0.07014584541320801,
          0.059657223522663116,
          0.05741160362958908,
          0.05127572640776634,
          0.12547241151332855,
          0.2706507742404938,
          0.18448840081691742
        ],
        "sources": 9
      },
      "step": 2000,
      "stream_state_rms_by_sublayer": [
        0.05045261234045029,
        0.10241243988275528,
        0.16565440595149994,
        0.1819411665201187,
        0.23719695210456848,
        0.24157075583934784,
        0.24811406433582306,
        0.26444777846336365,
        0.1042611226439476,
        0.10636615753173828,
        0.12873972952365875,
        0.13034582138061523,
        0.1382850706577301,
        0.14029134809970856,
        0.14404986798763275,
        0.15042904019355774,
        0.040291935205459595,
        0.04812335968017578,
        0.06678888201713562,
        0.07954440265893936,
        0.09121427685022354,
        0.10673297941684723,
        0.12122578918933868,
        0.14089107513427734,
        0.05663999170064926,
        0.06885477155447006,
        0.09547740966081619,
        0.10946234315633774,
        0.12204843014478683,
        0.13876035809516907,
        0.14941293001174927,
        0.1687854677438736,
        0.03680795058608055,
        0.05059739947319031,
        0.055156365036964417,
        0.07094386219978333,
        0.07373107224702835,
        0.09166282415390015,
        0.09760652482509613,
        0.12053482979536057,
        0.035728659480810165,
        0.048727843910455704,
        0.05559300258755684,
        0.07109228521585464,
        0.08069392293691635,
        0.09690473228693008,
        0.10087756812572479,
        0.11853098124265671,
        0.025910304859280586,
        0.05259029567241669,
        0.060516636818647385,
        0.08889466524124146,
        0.09090615063905716,
        0.12455403059720993,
        0.1273101419210434,
        0.1728687882423401,
        0.013562481850385666,
        0.06489895284175873,
        0.07113923132419586,
        0.14679722487926483,
        0.14901472628116608,
        0.23576539754867554,
        0.23766301572322845,
        0.35208994150161743
      ]
    },
    {
      "activation_grad_rms_by_block": [
        4.705962055595592e-05,
        4.8540012357989326e-05,
        4.8112680815393105e-05,
        4.5881148253101856e-05,
        6.218662019819021e-05,
        6.19510465185158e-05,
        5.5873108067316934e-05,
        5.11586076754611e-05,
        5.377902198233642e-05,
        4.923442247672938e-05,
        4.591418110067025e-05,
        4.340316809248179e-05,
        3.853498856187798e-05,
        3.637185727711767e-05,
        3.548145105014555e-05,
        3.440591899561696e-05,
        7.517015910707414e-05,
        7.005458610365167e-05,
        6.350460171233863e-05,
        5.674128988175653e-05,
        0.00010789759107865393,
        9.314908675150946e-05,
        7.818391168257222e-05,
        6.73785325489007e-05,
        5.735654485761188e-05,
        4.582547626341693e-05,
        3.993835343862884e-05,
        3.59240839316044e-05,
        2.40640074480325e-05,
        2.1051846488262527e-05,
        1.9940822312491946e-05,
        1.993459591176361e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 5.259535555524053e-05,
        "first_to_last_ratio": 1.59358297228639,
        "imbalance_abs_log_ratio": 0.46598492272985975,
        "last_quartile_mean": 3.300446633147658e-05,
        "mean": 5.106260448428657e-05,
        "normalized": [
          0.921606350307445,
          0.9505980521014451,
          0.9422292752458163,
          0.8985273805847644,
          1.2178505351666271,
          1.2132371065714032,
          1.0942079557361923,
          1.0018801076079868,
          1.0531977858451338,
          0.9641972432463806,
          0.8991742893725557,
          0.8499991046449304,
          0.7546616344988103,
          0.712299296999438,
          0.6948617566317875,
          0.6737987484795189,
          1.4721176067352022,
          1.3719352314903852,
          1.2436616258358075,
          1.111210257581308,
          2.1130451955668876,
          1.8242133884920444,
          1.531138344238427,
          1.319527925169485,
          1.1232592899812257,
          0.8974371112918603,
          0.7821448561426008,
          0.7035301918972675,
          0.4712647874324093,
          0.41227521981846393,
          0.3905171409466258,
          0.3903952043397641
        ],
        "population_cv": 0.3859968358313239
      },
      "activation_output_rms_by_block": [
        0.16142085194587708,
        0.2773628532886505,
        0.3768865168094635,
        0.40868353843688965,
        0.14338424801826477,
        0.16827327013015747,
        0.18227216601371765,
        0.19825012981891632,
        0.06985136866569519,
        0.1086883395910263,
        0.14860449731349945,
        0.21192558109760284,
        0.08672910183668137,
        0.13778631389141083,
        0.18114051222801208,
        0.23228809237480164,
        0.062035780400037766,
        0.0909910649061203,
        0.12021689862012863,
        0.1606491208076477,
        0.061457011848688126,
        0.092374786734581,
        0.12395651638507843,
        0.15162576735019684,
        0.06645520776510239,
        0.10725662857294083,
        0.14660172164440155,
        0.20186874270439148,
        0.07922796159982681,
        0.17203646898269653,
        0.2743971645832062,
        0.41855114698410034
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.23956669680774212,
        "first_to_last_ratio": 1.306969485354029,
        "imbalance_abs_log_ratio": 0.26771108728004156,
        "last_quartile_mean": 0.18329938035458326,
        "mean": 0.1694765428546816,
        "normalized": [
          0.9524672218755847,
          1.6365855039093904,
          2.223827029163714,
          2.4114460417529116,
          0.846041851002414,
          0.9929000633110867,
          1.0755008506988941,
          1.1697791710850909,
          0.41215950885657104,
          0.6413178942658843,
          0.8768440446706599,
          1.250471466598887,
          0.5117469378110197,
          0.8130111198312341,
          1.068823503104685,
          1.370620904003087,
          0.36604346156170187,
          0.5368947429151949,
          0.7093424057110318,
          0.9479136056332998,
          0.36262842522923483,
          0.5450594234376621,
          0.7314081010689808,
          0.8946711137494054,
          0.392120388141766,
          0.6328700524939813,
          0.8650266235965518,
          1.1911308745392848,
          0.46748629789882584,
          1.0151048993854561,
          1.6190863936756676,
          2.4696700790208403
        ],
        "population_cv": 0.5508327874312803
      },
      "bits_per_byte": 1.9147359946947653,
      "branch_output_rms_by_sublayer": [
        0.08130212873220444,
        0.13576549291610718,
        0.1432749330997467,
        0.041234951466321945,
        0.12958019971847534,
        0.023600688204169273,
        0.04409155994653702,
        0.0554339736700058,
        0.14100632071495056,
        0.01666521653532982,
        0.0481393076479435,
        0.008341255597770214,
        0.04208913818001747,
        0.01377889234572649,
        0.04236427694559097,
        0.03695881366729736,
        0.05848926678299904,
        0.033706482499837875,
        0.04504085332155228,
        0.0386466309428215,
        0.04397343844175339,
        0.04499480128288269,
        0.05665678158402443,
        0.057638294994831085,
        0.06478945910930634,
        0.04709118604660034,
        0.04678606986999512,
        0.044345736503601074,
        0.04817425087094307,
        0.04982399195432663,
        0.049439266324043274,
        0.060589101165533066,
        0.04376237466931343,
        0.040811967104673386,
        0.02850259095430374,
        0.04198655113577843,
        0.02922668866813183,
        0.05064959079027176,
        0.04021619260311127,
        0.07217346876859665,
        0.04409762844443321,
        0.039372652769088745,
        0.030135832726955414,
        0.04702961444854736,
        0.03803896903991699,
        0.054020222276449203,
        0.030100898817181587,
        0.06186771020293236,
        0.03295852243900299,
        0.054687224328517914,
        0.0306381918489933,
        0.05866995081305504,
        0.02206242084503174,
        0.06297128647565842,
        0.02074088715016842,
        0.07938925921916962,
        0.021799447014927864,
        0.07573848962783813,
        0.026608876883983612,
        0.1105475053191185,
        0.01902451552450657,
        0.12047550827264786,
        0.02519184537231922,
        0.16982334852218628
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00023209204329478853,
        0.00014705811075659293,
        0.0001443553349415038,
        0.0001409677954299832,
        0.0001547598070491459,
        0.00010679063100428474,
        0.00010173131038502872,
        0.00012885752112210694,
        0.00018269174364617834,
        0.00015061935227176197,
        0.00015101151039961197,
        0.0001756105532961918,
        0.00014068815971577485,
        0.0001326364304799064,
        0.00015121016067746042,
        0.00016411832907359437,
        0.0002779928432077,
        0.00028472923376674154,
        0.0003007215693300081,
        0.0003372879732191828,
        0.0005019822813973469,
        0.0004655562424498461,
        0.00043909856222474334,
        0.00038874448014086744,
        0.0003187708239967095,
        0.00026126643200496464,
        0.0002213996139509037,
        0.00021927053392099255,
        0.00013957865743180746,
        0.00013181726813679266,
        0.00012444014246083002,
        0.00012714374135915476
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00014457656924792933,
        "first_to_last_ratio": 0.74925318033778,
        "imbalance_abs_log_ratio": 0.28867832809930477,
        "last_quartile_mean": 0.0001929609016577694,
        "mean": 0.00021703122476695333,
        "normalized": [
          1.069394708268395,
          0.6775896459806783,
          0.6651362498484334,
          0.6495277146472977,
          0.7130762276963877,
          0.49205192072687143,
          0.46874043352179906,
          0.5937280281234798,
          0.8417763104933472,
          0.6939985360792947,
          0.6958054563889003,
          0.8091487917683791,
          0.6482392562030871,
          0.6111398515228881,
          0.6967207637510633,
          0.7561968525488604,
          1.2808886993390323,
          1.311927507539442,
          1.3856143034391522,
          1.554098833388423,
          2.3129495856477433,
          2.145111805684907,
          2.0232045536131698,
          1.7911914774397033,
          1.468778625467388,
          1.2038195530873992,
          1.0201279294656385,
          1.0103179123485286,
          0.6431270780584042,
          0.6073654529588411,
          0.573374373178113,
          0.5858315617749513
        ],
        "population_cv": 0.5013169471957857
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.22879935801029205,
          "mean_weights": [
            0.9326952695846558,
            0.06730473041534424
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.5610548257827759,
          "mean_weights": [
            0.6759846210479736,
            0.3240153193473816
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.43457120656967163,
          "mean_weights": [
            0.8295697569847107,
            0.1704302579164505
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6696063280105591,
          "mean_weights": [
            0.545992374420166,
            0.45400765538215637
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6690812110900879,
          "mean_weights": [
            0.5202739238739014,
            0.47972607612609863
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6865538358688354,
          "mean_weights": [
            0.5361902713775635,
            0.4638097584247589
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.4152516722679138,
          "mean_weights": [
            0.8452726602554321,
            0.15472736954689026
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6853918433189392,
          "mean_weights": [
            0.5220736265182495,
            0.4779263138771057
          ],
          "sources": 2
        },
        {
          "entropy_mean": 1.0265861749649048,
          "mean_weights": [
            0.3228098750114441,
            0.23137259483337402,
            0.4458175599575043
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.066486120223999,
          "mean_weights": [
            0.2909930944442749,
            0.26484501361846924,
            0.44416186213493347
          ],
          "sources": 3
        },
        {
          "entropy_mean": 0.8915231227874756,
          "mean_weights": [
            0.21873247623443604,
            0.16816270351409912,
            0.6131048202514648
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0651111602783203,
          "mean_weights": [
            0.3639928698539734,
            0.22285741567611694,
            0.41314971446990967
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0005524158477783,
          "mean_weights": [
            0.3166183531284332,
            0.18387392163276672,
            0.49950775504112244
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0648009777069092,
          "mean_weights": [
            0.35599708557128906,
            0.22703218460083008,
            0.41697072982788086
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0006463527679443,
          "mean_weights": [
            0.28650256991386414,
            0.19727063179016113,
            0.5162267684936523
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.072684407234192,
          "mean_weights": [
            0.41332390904426575,
            0.2456190586090088,
            0.34105703234672546
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.304518699645996,
          "mean_weights": [
            0.2721603810787201,
            0.12663133442401886,
            0.2481587529182434,
            0.35304954648017883
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3646502494812012,
          "mean_weights": [
            0.29890745878219604,
            0.18950369954109192,
            0.2219487726688385,
            0.2896400988101959
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.331923484802246,
          "mean_weights": [
            0.31035515666007996,
            0.14580613374710083,
            0.22454026341438293,
            0.31929847598075867
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3639233112335205,
          "mean_weights": [
            0.2952677309513092,
            0.19548878073692322,
            0.21280673146247864,
            0.29643672704696655
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.2764989137649536,
          "mean_weights": [
            0.3656603693962097,
            0.14041098952293396,
            0.17602922022342682,
            0.3178994059562683
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.364103078842163,
          "mean_weights": [
            0.27470695972442627,
            0.18989428877830505,
            0.23374854028224945,
            0.30165016651153564
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3429455757141113,
          "mean_weights": [
            0.2830207943916321,
            0.17677831649780273,
            0.23265977203845978,
            0.3075411319732666
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3732160329818726,
          "mean_weights": [
            0.2642192244529724,
            0.22012627124786377,
            0.22637531161308289,
            0.28927919268608093
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.5599325895309448,
          "mean_weights": [
            0.18314200639724731,
            0.13039523363113403,
            0.18218961358070374,
            0.26679670810699463,
            0.2374764084815979
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5772080421447754,
          "mean_weights": [
            0.19838765263557434,
            0.18937477469444275,
            0.15325719118118286,
            0.16343195736408234,
            0.2955484390258789
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5662866830825806,
          "mean_weights": [
            0.22973918914794922,
            0.12454323470592499,
            0.16623356938362122,
            0.21617932617664337,
            0.2633046507835388
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5421115159988403,
          "mean_weights": [
            0.2021060585975647,
            0.1677371859550476,
            0.11891372501850128,
            0.17452098429203033,
            0.3367220163345337
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.564535140991211,
          "mean_weights": [
            0.22419840097427368,
            0.12987476587295532,
            0.1717071682214737,
            0.21590615808963776,
            0.25831347703933716
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5267750024795532,
          "mean_weights": [
            0.215010404586792,
            0.19819971919059753,
            0.10917140543460846,
            0.13804513216018677,
            0.33957335352897644
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.556459903717041,
          "mean_weights": [
            0.2037975788116455,
            0.12750469148159027,
            0.1812792420387268,
            0.22193393111228943,
            0.2654845714569092
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5851528644561768,
          "mean_weights": [
            0.21840542554855347,
            0.1923592984676361,
            0.1442098617553711,
            0.1790001392364502,
            0.26602524518966675
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.7158764600753784,
          "mean_weights": [
            0.13390135765075684,
            0.10310791432857513,
            0.14190614223480225,
            0.16341811418533325,
            0.15331929922103882,
            0.3043471872806549
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.6410574913024902,
          "mean_weights": [
            0.14665581285953522,
            0.12231245636940002,
            0.08100371062755585,
            0.11105543375015259,
            0.16015371680259705,
            0.3788188695907593
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.6537573337554932,
          "mean_weights": [
            0.12217442691326141,
            0.09171081334352493,
            0.1348741054534912,
            0.16064059734344482,
            0.1268058568239212,
            0.363794207572937
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.6571812629699707,
          "mean_weights": [
            0.15173450112342834,
            0.14253422617912292,
            0.09320533275604248,
            0.10436245799064636,
            0.14149092137813568,
            0.3666725754737854
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.6046829223632812,
          "mean_weights": [
            0.12253664433956146,
            0.08343374729156494,
            0.1302124410867691,
            0.14439499378204346,
            0.11938183009624481,
            0.4000403583049774
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7041404247283936,
          "mean_weights": [
            0.1659325659275055,
            0.15505465865135193,
            0.10219550132751465,
            0.11347830295562744,
            0.1465306580066681,
            0.3168083429336548
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.5808616876602173,
          "mean_weights": [
            0.12713032960891724,
            0.08526259660720825,
            0.1294248104095459,
            0.1489948332309723,
            0.10688205063343048,
            0.40230536460876465
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7305450439453125,
          "mean_weights": [
            0.17891491949558258,
            0.17673785984516144,
            0.09075747430324554,
            0.11627361923456192,
            0.17774534225463867,
            0.25957077741622925
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7513642311096191,
          "mean_weights": [
            0.09819091856479645,
            0.06520214676856995,
            0.091708242893219,
            0.11103156208992004,
            0.08388154953718185,
            0.22660189867019653,
            0.3233836889266968
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.766061782836914,
          "mean_weights": [
            0.12103402614593506,
            0.1159343495965004,
            0.06956179440021515,
            0.07680170238018036,
            0.10650800168514252,
            0.1487376093864441,
            0.36142250895500183
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.6586570739746094,
          "mean_weights": [
            0.09605097770690918,
            0.06012085825204849,
            0.08562932163476944,
            0.10102558881044388,
            0.07161714881658554,
            0.17364460229873657,
            0.4119114875793457
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.7821578979492188,
          "mean_weights": [
            0.13399386405944824,
            0.12986589968204498,
            0.0802837461233139,
            0.07629825174808502,
            0.10134232044219971,
            0.12680892646312714,
            0.351406991481781
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.655133843421936,
          "mean_weights": [
            0.10968073457479477,
            0.0657510757446289,
            0.08727138489484787,
            0.09466101229190826,
            0.0703636109828949,
            0.14186590909957886,
            0.4304063022136688
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.8160101175308228,
          "mean_weights": [
            0.13769462704658508,
            0.11569900065660477,
            0.08787183463573456,
            0.0929793268442154,
            0.10649852454662323,
            0.1315194070339203,
            0.3277372717857361
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.6999847888946533,
          "mean_weights": [
            0.10220035165548325,
            0.07546016573905945,
            0.1028895378112793,
            0.10639216750860214,
            0.07665040343999863,
            0.13976991176605225,
            0.3966374397277832
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.8706753253936768,
          "mean_weights": [
            0.14694823324680328,
            0.12422746419906616,
            0.09348422288894653,
            0.1027352511882782,
            0.12104703485965729,
            0.14027650654315948,
            0.27128130197525024
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.7911043167114258,
          "mean_weights": [
            0.09078209102153778,
            0.05932023376226425,
            0.06744872033596039,
            0.0717800185084343,
            0.05460817739367485,
            0.10924961417913437,
            0.271850049495697,
            0.27496111392974854
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.9753689765930176,
          "mean_weights": [
            0.10981250554323196,
            0.10515635460615158,
            0.07847881317138672,
            0.0806870236992836,
            0.10027817636728287,
            0.1099884957075119,
            0.1757785975933075,
            0.23982004821300507
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.81705904006958,
          "mean_weights": [
            0.0980612188577652,
            0.060419049113988876,
            0.06808654963970184,
            0.07581339031457901,
            0.06074263155460358,
            0.09064903855323792,
            0.23993313312530518,
            0.3062949776649475
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.9814136028289795,
          "mean_weights": [
            0.11045937240123749,
            0.08757394552230835,
            0.08714327216148376,
            0.09700852632522583,
            0.09435248374938965,
            0.114869624376297,
            0.1777447760105133,
            0.23084799945354462
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.8373757600784302,
          "mean_weights": [
            0.08649781346321106,
            0.07379771769046783,
            0.0769633948802948,
            0.0880836620926857,
            0.066677525639534,
            0.1187550276517868,
            0.23661984503269196,
            0.25260502099990845
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.968585729598999,
          "mean_weights": [
            0.10324696451425552,
            0.0860632136464119,
            0.09797552973031998,
            0.1271316111087799,
            0.08284065127372742,
            0.10640716552734375,
            0.17243653535842896,
            0.22389833629131317
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.8529754877090454,
          "mean_weights": [
            0.08385773003101349,
            0.07188580930233002,
            0.08174896240234375,
            0.09558320790529251,
            0.06307300925254822,
            0.108722984790802,
            0.2897399365901947,
            0.20538833737373352
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.9637342691421509,
          "mean_weights": [
            0.10303820669651031,
            0.08717727661132812,
            0.09220535308122635,
            0.11336663365364075,
            0.08865514397621155,
            0.14618384838104248,
            0.2349606156349182,
            0.13441291451454163
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.962258219718933,
          "mean_weights": [
            0.07841141521930695,
            0.07437640428543091,
            0.07400830090045929,
            0.07925958186388016,
            0.05560165271162987,
            0.08937598764896393,
            0.1956467628479004,
            0.1118645966053009,
            0.24145527184009552
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.0993216037750244,
          "mean_weights": [
            0.1043272316455841,
            0.09323035180568695,
            0.08778752386569977,
            0.10311444103717804,
            0.09694567322731018,
            0.13869863748550415,
            0.1680138111114502,
            0.0874539315700531,
            0.12042839825153351
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.0489909648895264,
          "mean_weights": [
            0.0919051468372345,
            0.07878251373767853,
            0.07574650645256042,
            0.09311886876821518,
            0.05773407965898514,
            0.07325181365013123,
            0.17863741517066956,
            0.18801960349082947,
            0.1628040373325348
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.126072406768799,
          "mean_weights": [
            0.09415344893932343,
            0.07414553314447403,
            0.10936354845762253,
            0.11765885353088379,
            0.07923045754432678,
            0.12059831619262695,
            0.17183929681777954,
            0.12057285010814667,
            0.11243770271539688
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.028747081756592,
          "mean_weights": [
            0.08796507865190506,
            0.0757753849029541,
            0.0847562700510025,
            0.09340266138315201,
            0.06450396031141281,
            0.08635804057121277,
            0.24226921796798706,
            0.14922890067100525,
            0.11574049293994904
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1297550201416016,
          "mean_weights": [
            0.10085826367139816,
            0.08009681105613708,
            0.0982624888420105,
            0.11403533071279526,
            0.09301646053791046,
            0.12771156430244446,
            0.17309343814849854,
            0.10872717201709747,
            0.10419845581054688
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1051812171936035,
          "mean_weights": [
            0.09203394502401352,
            0.08722756803035736,
            0.10295312106609344,
            0.11847955733537674,
            0.07401065528392792,
            0.11382973194122314,
            0.21416787803173065,
            0.11479790508747101,
            0.08249962329864502
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.059127259999513626,
        0.055979277938604355,
        0.0761958509683609,
        0.07538121938705444,
        0.13674014806747437,
        0.18601374328136444,
        0.18700072169303894,
        0.08912894874811172,
        0.2049511820077896,
        0.12973569333553314,
        0.14231130480766296,
        0.13784728944301605,
        0.13109618425369263,
        0.1306019276380539,
        0.13412337005138397,
        0.13534265756607056,
        0.13132649660110474,
        0.08038423955440521,
        0.09715579450130463,
        0.0852823480963707,
        0.09982985258102417,
        0.08248664438724518,
        0.10234102606773376,
        0.09743349254131317,
        0.11231652647256851,
        0.07765212655067444,
        0.08827291429042816,
        0.07436380535364151,
        0.08991213887929916,
        0.08191535621881485,
        0.1065397635102272,
        0.09093168377876282,
        0.10877490043640137,
        0.06605636328458786,
        0.07123783230781555,
        0.06527329236268997,
        0.07851143926382065,
        0.0699063166975975,
        0.08676032721996307,
        0.08022409677505493,
        0.10016339272260666,
        0.05980068817734718,
        0.06517573446035385,
        0.05788872763514519,
        0.07041674107313156,
        0.0643942728638649,
        0.07159053534269333,
        0.07148924469947815,
        0.07631570100784302,
        0.056988880038261414,
        0.06180137023329735,
        0.05852367728948593,
        0.06370948255062103,
        0.06811124831438065,
        0.07006713002920151,
        0.07283915579319,
        0.0730312243103981,
        0.057786133140325546,
        0.06770572811365128,
        0.07090020179748535,
        0.07336394488811493,
        0.07763758301734924,
        0.08089767396450043,
        0.07564623653888702
      ],
      "loss_nats": 1.3271938562393188,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 1.8321013450622559,
        "mean_weights": [
          0.09867185354232788,
          0.05898427963256836,
          0.057430200278759,
          0.05067210644483566,
          0.04944504797458649,
          0.047245003283023834,
          0.1162109300494194,
          0.30178403854370117,
          0.219556525349617
        ],
        "sources": 9
      },
      "step": 4000,
      "stream_state_rms_by_sublayer": [
        0.08130212873220444,
        0.16142085194587708,
        0.2561342120170593,
        0.2773628532886505,
        0.3685380816459656,
        0.3768865168094635,
        0.38441672921180725,
        0.40868353843688965,
        0.14100632071495056,
        0.14338424801826477,
        0.16686734557151794,
        0.16827327013015747,
        0.18023288249969482,
        0.18227216601371765,
        0.18965557217597961,
        0.19825012981891632,
        0.05848926678299904,
        0.06985136866569519,
        0.09072534739971161,
        0.1086883395910263,
        0.12624414265155792,
        0.14860449731349945,
        0.17554737627506256,
        0.21192558109760284,
        0.06478945910930634,
        0.08672910183668137,
        0.11512516438961029,
        0.13778631389141083,
        0.15369470417499542,
        0.18114051222801208,
        0.19735252857208252,
        0.23228809237480164,
        0.04376237466931343,
        0.062035780400037766,
        0.07002050429582596,
        0.0909910649061203,
        0.09636416286230087,
        0.12021689862012863,
        0.12901709973812103,
        0.1606491208076477,
        0.04409762844443321,
        0.061457011848688126,
        0.07137759029865265,
        0.092374786734581,
        0.10297419875860214,
        0.12395651638507843,
        0.1298016905784607,
        0.15162576735019684,
        0.03295852243900299,
        0.06645520776510239,
        0.07529281079769135,
        0.10725662857294083,
        0.11068660020828247,
        0.14660172164440155,
        0.1495535671710968,
        0.20186874270439148,
        0.021799447014927864,
        0.07922796159982681,
        0.08539005368947983,
        0.17203646898269653,
        0.17447474598884583,
        0.2743971645832062,
        0.27695441246032715,
        0.41855114698410034
      ]
    },
    {
      "activation_grad_rms_by_block": [
        6.761295662727207e-05,
        7.037314207991585e-05,
        6.971564289415255e-05,
        6.589700205950066e-05,
        0.00010337811545468867,
        0.00010314521205145866,
        9.018617129186168e-05,
        8.218877337640151e-05,
        8.93865289981477e-05,
        8.14282102510333e-05,
        7.567049033241346e-05,
        7.11992834112607e-05,
        6.774777284590527e-05,
        6.376176315825433e-05,
        6.184441008372232e-05,
        5.9780148149002343e-05,
        0.00014381110668182373,
        0.00013238933752290905,
        0.0001167725640698336,
        0.00010191192995989695,
        0.0002003924601012841,
        0.0001659920671954751,
        0.0001361987233394757,
        0.00011443908442743123,
        0.00010709096386563033,
        8.16169849713333e-05,
        6.831773498561233e-05,
        5.791887815576047e-05,
        4.921185245621018e-05,
        3.8056739867897704e-05,
        3.282592660980299e-05,
        3.210229624528438e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 8.156212697940646e-05,
        "first_to_last_ratio": 1.3967870279562358,
        "imbalance_abs_log_ratio": 0.33417461908716395,
        "last_quartile_mean": 5.839267214469146e-05,
        "mean": 8.757388354752038e-05,
        "normalized": [
          0.7720675832605176,
          0.803585946279625,
          0.7960780094481321,
          0.752473219070419,
          1.180467409539882,
          1.1778079020041263,
          1.029829529518624,
          0.9385078067458692,
          1.0206984705622164,
          0.9298229900567075,
          0.8640759923745102,
          0.8130195958778703,
          0.7736070401530517,
          0.7280910766467854,
          0.7061969571118035,
          0.6826252956632183,
          1.642168884788434,
          1.5117445082936212,
          1.3334176736203447,
          1.1637251407789435,
          2.2882673690329693,
          1.8954517085610634,
          1.5552436162725376,
          1.306771834154562,
          1.2228641636923565,
          0.9319785952743014,
          0.7801153976292595,
          0.661371584878177,
          0.561946672486053,
          0.4345672285647456,
          0.37483694087850516,
          0.36657385678076787
        ],
        "population_cv": 0.42684708327825066
      },
      "activation_output_rms_by_block": [
        0.16424456238746643,
        0.29588937759399414,
        0.42337876558303833,
        0.45154979825019836,
        0.13891471922397614,
        0.1555756777524948,
        0.16914354264736176,
        0.1874145120382309,
        0.06211879849433899,
        0.09842939674854279,
        0.137006014585495,
        0.20118924975395203,
        0.08399304747581482,
        0.12138351052999496,
        0.1587214469909668,
        0.2071460783481598,
        0.0581100657582283,
        0.08358953893184662,
        0.10947012156248093,
        0.1472771316766739,
        0.05584336072206497,
        0.08318652212619781,
        0.11238116025924683,
        0.1379220336675644,
        0.05995604768395424,
        0.09392217546701431,
        0.1259489357471466,
        0.17086055874824524,
        0.06700707972049713,
        0.13971327245235443,
        0.2211977243423462,
        0.32473331689834595
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.2482638694345951,
        "first_to_last_ratio": 1.6504997944655762,
        "imbalance_abs_log_ratio": 0.5010781477825214,
        "last_quartile_mean": 0.150417388882488,
        "mean": 0.1577255482552573,
        "normalized": [
          1.0413313772202522,
          1.8759762186094133,
          2.6842751238870792,
          2.86288304745295,
          0.8807369558111258,
          0.9863695480754758,
          1.0723915340184837,
          1.1882317995492147,
          0.3938410687519575,
          0.6240548714989927,
          0.8686355260833801,
          1.2755653854399962,
          0.532526584341831,
          0.7695868670150526,
          1.006314124418815,
          1.3133324349770026,
          0.3684251942759779,
          0.5299682889456079,
          0.6940544684956076,
          0.9337557124120812,
          0.3540539965769534,
          0.5274131112327584,
          0.7125108234042923,
          0.874443203356988,
          0.3801289540418877,
          0.5954785163594047,
          0.7985322424957776,
          1.083277634082024,
          0.4248333923179985,
          0.8857998846594436,
          1.4024216545081702,
          2.0588504556840044
        ],
        "population_cv": 0.6067899489376947
      },
      "bits_per_byte": 1.8565786854442996,
      "branch_output_rms_by_sublayer": [
        0.08626964688301086,
        0.13547764718532562,
        0.16337208449840546,
        0.039667773991823196,
        0.15584443509578705,
        0.026703152805566788,
        0.04305505007505417,
        0.05304057523608208,
        0.1360139101743698,
        0.016798952594399452,
        0.03393339365720749,
        0.00753376679494977,
        0.03730965405702591,
        0.01294450368732214,
        0.037494905292987823,
        0.03526800498366356,
        0.05234000086784363,
        0.030294200405478477,
        0.04223497211933136,
        0.034149833023548126,
        0.041619230061769485,
        0.04367009550333023,
        0.05722156912088394,
        0.05930330231785774,
        0.062187470495700836,
        0.04527241364121437,
        0.0369485504925251,
        0.04281846433877945,
        0.04293901473283768,
        0.04812675341963768,
        0.045369699597358704,
        0.06094835698604584,
        0.04149825870990753,
        0.038031257688999176,
        0.02440357394516468,
        0.040106989443302155,
        0.026009807363152504,
        0.04961257055401802,
        0.03634416684508324,
        0.07085621356964111,
        0.038355328142642975,
        0.03753058984875679,
        0.02631118707358837,
        0.04544757679104805,
        0.033921729773283005,
        0.05368982255458832,
        0.02731483057141304,
        0.06174425780773163,
        0.027557097375392914,
        0.05051252990961075,
        0.024315863847732544,
        0.054937634617090225,
        0.02051021158695221,
        0.05777662619948387,
        0.018631720915436745,
        0.07133159041404724,
        0.019497457891702652,
        0.0636468306183815,
        0.021715763956308365,
        0.09261580556631088,
        0.017114246264100075,
        0.1020733118057251,
        0.020576251670718193,
        0.12878532707691193
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0003637111411518744,
        0.0002328437345071155,
        0.00026754486121072013,
        0.00020961054010746675,
        0.00023324217449373482,
        0.00013226952769200274,
        0.0001490970084679736,
        0.0001891597343986376,
        0.0002918746014559555,
        0.00023307226647352922,
        0.00024024223572494834,
        0.00029360747277183056,
        0.00025626387819923106,
        0.00024114885933974834,
        0.0002716298155375107,
        0.0003086967264992385,
        0.0005417594428612147,
        0.0005350465304351142,
        0.0005646380221663597,
        0.0006164043223392351,
        0.0009244274084097353,
        0.0008484364345428017,
        0.0007892136159088089,
        0.0006791712867767833,
        0.0005903268079560613,
        0.00047055631494833115,
        0.0003773406829883918,
        0.0003589161190616393,
        0.00026391753197443733,
        0.00022775345188039026,
        0.00019334088037647755,
        0.0001778215115139631
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00022218484025369068,
        "first_to_last_ratio": 0.6682317907333766,
        "imbalance_abs_log_ratio": 0.4031201735174996,
        "last_quartile_mean": 0.00033249666258746146,
        "mean": 0.00037728390444285195,
        "normalized": [
          0.9640250667172751,
          0.6171578796900011,
          0.7091340448403511,
          0.5555777430182339,
          0.6182139543911144,
          0.3505835423520907,
          0.3951851820664077,
          0.5013723940277184,
          0.7736206024664017,
          0.6177636091253748,
          0.6367677838780912,
          0.7782136192780627,
          0.6792335299266601,
          0.6391708114234587,
          0.7199613138509996,
          0.81820804668331,
          1.4359463430099129,
          1.4181536082893218,
          1.496586563903859,
          1.633794378929312,
          2.4502169255666204,
          2.2488010343184843,
          2.0918295390158974,
          1.8001597173347186,
          1.5646753042057733,
          1.2472207518187604,
          1.000150492891086,
          0.9513157461399342,
          0.6995197220622846,
          0.6036659640085139,
          0.5124546213069721,
          0.471320163462998
        ],
        "population_cv": 0.5587385684613161
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.21598131954669952,
          "mean_weights": [
            0.9387255907058716,
            0.06127442046999931
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.5377621054649353,
          "mean_weights": [
            0.7082071900367737,
            0.2917928099632263
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.43774232268333435,
          "mean_weights": [
            0.829714834690094,
            0.1702851504087448
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6749181151390076,
          "mean_weights": [
            0.5407299399375916,
            0.45927006006240845
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6663151383399963,
          "mean_weights": [
            0.5011438131332397,
            0.49885618686676025
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6865031719207764,
          "mean_weights": [
            0.5308656692504883,
            0.4691343307495117
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.4133813679218292,
          "mean_weights": [
            0.8479139804840088,
            0.15208598971366882
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6850745677947998,
          "mean_weights": [
            0.5398513674736023,
            0.4601486027240753
          ],
          "sources": 2
        },
        {
          "entropy_mean": 1.0074119567871094,
          "mean_weights": [
            0.32744842767715454,
            0.20981493592262268,
            0.4627366065979004
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0489801168441772,
          "mean_weights": [
            0.28095993399620056,
            0.24264614284038544,
            0.4763939380645752
          ],
          "sources": 3
        },
        {
          "entropy_mean": 0.8527575731277466,
          "mean_weights": [
            0.20522087812423706,
            0.1493145227432251,
            0.6454645991325378
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.049942135810852,
          "mean_weights": [
            0.35788866877555847,
            0.20253834128379822,
            0.4395729899406433
          ],
          "sources": 3
        },
        {
          "entropy_mean": 0.9791408181190491,
          "mean_weights": [
            0.31465935707092285,
            0.1638563871383667,
            0.5214842557907104
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0485806465148926,
          "mean_weights": [
            0.346283882856369,
            0.20800183713436127,
            0.4457142949104309
          ],
          "sources": 3
        },
        {
          "entropy_mean": 0.9801348447799683,
          "mean_weights": [
            0.2748759686946869,
            0.1817527413368225,
            0.5433712601661682
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0688585042953491,
          "mean_weights": [
            0.4051227569580078,
            0.23231396079063416,
            0.36256328225135803
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.2915732860565186,
          "mean_weights": [
            0.26621368527412415,
            0.1108347550034523,
            0.2563389539718628,
            0.36661261320114136
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3568191528320312,
          "mean_weights": [
            0.294538676738739,
            0.1710844486951828,
            0.2252884805202484,
            0.30908840894699097
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.320021152496338,
          "mean_weights": [
            0.304654598236084,
            0.12911874055862427,
            0.23493197560310364,
            0.3312946557998657
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3578683137893677,
          "mean_weights": [
            0.29558902978897095,
            0.17812995612621307,
            0.21904321014881134,
            0.30723777413368225
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.258887767791748,
          "mean_weights": [
            0.35744374990463257,
            0.12067858874797821,
            0.17883820831775665,
            0.34303945302963257
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3539760112762451,
          "mean_weights": [
            0.2736307978630066,
            0.1722734123468399,
            0.2365075945854187,
            0.3175881803035736
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3203014135360718,
          "mean_weights": [
            0.28207293152809143,
            0.15420743823051453,
            0.22419029474258423,
            0.3395293354988098
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3734662532806396,
          "mean_weights": [
            0.2766023576259613,
            0.21176086366176605,
            0.22839435935020447,
            0.283242404460907
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.5325813293457031,
          "mean_weights": [
            0.17451074719429016,
            0.10643941164016724,
            0.1691833734512329,
            0.27394938468933105,
            0.27591708302497864
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.560360074043274,
          "mean_weights": [
            0.19377154111862183,
            0.17211592197418213,
            0.14605137705802917,
            0.16254930198192596,
            0.3255118727684021
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.540909767150879,
          "mean_weights": [
            0.21921303868293762,
            0.10205383598804474,
            0.16381604969501495,
            0.215368390083313,
            0.2995486855506897
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5142419338226318,
          "mean_weights": [
            0.2000826597213745,
            0.15480534732341766,
            0.10827909409999847,
            0.16766759753227234,
            0.3691653311252594
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5333527326583862,
          "mean_weights": [
            0.21770668029785156,
            0.10525082051753998,
            0.15970060229301453,
            0.217601016163826,
            0.29974091053009033
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.512608289718628,
          "mean_weights": [
            0.21392285823822021,
            0.1860550343990326,
            0.10291537642478943,
            0.1385909616947174,
            0.35851576924324036
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5262645483016968,
          "mean_weights": [
            0.20246261358261108,
            0.10530520975589752,
            0.16650918126106262,
            0.21146778762340546,
            0.3142552077770233
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5794764757156372,
          "mean_weights": [
            0.2210189402103424,
            0.18839940428733826,
            0.14016607403755188,
            0.17440536618232727,
            0.2760102152824402
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6602650880813599,
          "mean_weights": [
            0.12695768475532532,
            0.08036886155605316,
            0.12672865390777588,
            0.15391480922698975,
            0.1544383019208908,
            0.3575916886329651
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.5925545692443848,
          "mean_weights": [
            0.14075040817260742,
            0.11028306186199188,
            0.07537610828876495,
            0.10244421660900116,
            0.1520608514547348,
            0.41908538341522217
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.5918848514556885,
          "mean_weights": [
            0.11773697286844254,
            0.07344591617584229,
            0.11976177245378494,
            0.14735516905784607,
            0.1265447735786438,
            0.4151553511619568
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.6208878755569458,
          "mean_weights": [
            0.14785674214363098,
            0.12492769956588745,
            0.08781817555427551,
            0.10019385814666748,
            0.14005999267101288,
            0.3991435170173645
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.5470075607299805,
          "mean_weights": [
            0.12043031305074692,
            0.06855648756027222,
            0.11753211170434952,
            0.13396470248699188,
            0.1162077933549881,
            0.44330859184265137
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.683492660522461,
          "mean_weights": [
            0.1658952832221985,
            0.1435280442237854,
            0.09461991488933563,
            0.10784181207418442,
            0.15157341957092285,
            0.3365415334701538
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.5540567636489868,
          "mean_weights": [
            0.12987136840820312,
            0.07178180664777756,
            0.12132491171360016,
            0.1441602110862732,
            0.11020088195800781,
            0.42266082763671875
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7208337783813477,
          "mean_weights": [
            0.18048501014709473,
            0.1685236394405365,
            0.08726631104946136,
            0.11160479485988617,
            0.18141084909439087,
            0.27070939540863037
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.6729483604431152,
          "mean_weights": [
            0.09229261428117752,
            0.051420144736766815,
            0.08079522103071213,
            0.09550285339355469,
            0.07590493559837341,
            0.22372649610042572,
            0.3803577423095703
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.710235595703125,
          "mean_weights": [
            0.11657393723726273,
            0.10617929697036743,
            0.061112720519304276,
            0.06796726584434509,
            0.10333786904811859,
            0.14365722239017487,
            0.4011717140674591
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.5640671253204346,
          "mean_weights": [
            0.0904226303100586,
            0.04561381787061691,
            0.07565715163946152,
            0.08878682553768158,
            0.06555216014385223,
            0.16504976153373718,
            0.4689176678657532
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.7556228637695312,
          "mean_weights": [
            0.13281463086605072,
            0.12175846099853516,
            0.07235272228717804,
            0.0707377940416336,
            0.10551950335502625,
            0.12620802223682404,
            0.3706088662147522
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.594892978668213,
          "mean_weights": [
            0.10721130669116974,
            0.053925663232803345,
            0.07673212885856628,
            0.08865121006965637,
            0.06888366490602493,
            0.14094024896621704,
            0.4636557698249817
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.7775511741638184,
          "mean_weights": [
            0.13637228310108185,
            0.10478455573320389,
            0.07710614800453186,
            0.08548048138618469,
            0.11000139266252518,
            0.12931162118911743,
            0.3569435477256775
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.654036045074463,
          "mean_weights": [
            0.10232305526733398,
            0.061968062072992325,
            0.08900615572929382,
            0.1005433052778244,
            0.07864195853471756,
            0.14099234342575073,
            0.4265251159667969
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.84956693649292,
          "mean_weights": [
            0.14851760864257812,
            0.11630287766456604,
            0.08512166142463684,
            0.09215595573186874,
            0.1263951063156128,
            0.14375528693199158,
            0.28775155544281006
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.7161309719085693,
          "mean_weights": [
            0.0837959572672844,
            0.04434927552938461,
            0.0567089319229126,
            0.06495873630046844,
            0.04885464161634445,
            0.09997115284204483,
            0.2760006785392761,
            0.32536062598228455
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.9285151958465576,
          "mean_weights": [
            0.10564712435007095,
            0.09173665940761566,
            0.07135222852230072,
            0.07045470178127289,
            0.09785081446170807,
            0.10425601899623871,
            0.1782122403383255,
            0.2804902195930481
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.7219383716583252,
          "mean_weights": [
            0.0923302173614502,
            0.04646994173526764,
            0.05750556290149689,
            0.06549686193466187,
            0.054404959082603455,
            0.08427833020687103,
            0.2333412766456604,
            0.3661728501319885
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.9470824003219604,
          "mean_weights": [
            0.10804469138383865,
            0.07850410044193268,
            0.07298290729522705,
            0.08511960506439209,
            0.09968686103820801,
            0.11502514779567719,
            0.18013443052768707,
            0.26050227880477905
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.7855088710784912,
          "mean_weights": [
            0.08493785560131073,
            0.05972691997885704,
            0.07017223536968231,
            0.07963316142559052,
            0.0645163506269455,
            0.11586237698793411,
            0.23710276186466217,
            0.28804832696914673
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.973182201385498,
          "mean_weights": [
            0.1049819216132164,
            0.0840599462389946,
            0.09310521930456161,
            0.11566175520420074,
            0.0963074192404747,
            0.1096896231174469,
            0.16087493300437927,
            0.23531918227672577
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.802741289138794,
          "mean_weights": [
            0.08190082013607025,
            0.05632811039686203,
            0.07102417945861816,
            0.08421063423156738,
            0.06341388821601868,
            0.10201875865459442,
            0.2913346588611603,
            0.24976897239685059
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.945815086364746,
          "mean_weights": [
            0.1064736619591713,
            0.07925166189670563,
            0.07823627442121506,
            0.09447507560253143,
            0.09724453091621399,
            0.1459730863571167,
            0.2451319396495819,
            0.153213769197464
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.8717057704925537,
          "mean_weights": [
            0.07336360216140747,
            0.05720400810241699,
            0.057025715708732605,
            0.06655597686767578,
            0.05032068490982056,
            0.08329878002405167,
            0.1901058554649353,
            0.11727118492126465,
            0.3048541843891144
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.09096622467041,
          "mean_weights": [
            0.10326676070690155,
            0.08232800662517548,
            0.07164021581411362,
            0.08822821080684662,
            0.10803373903036118,
            0.13578416407108307,
            0.17320571839809418,
            0.09955950826406479,
            0.13795366883277893
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.96881103515625,
          "mean_weights": [
            0.08843819797039032,
            0.06052382290363312,
            0.0642135739326477,
            0.07744920998811722,
            0.05161681026220322,
            0.06631232053041458,
            0.16417551040649414,
            0.2115624099969864,
            0.21570813655853271
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.124063014984131,
          "mean_weights": [
            0.09220977127552032,
            0.06839151680469513,
            0.10039852559566498,
            0.10902463644742966,
            0.0849665105342865,
            0.12137697637081146,
            0.17017139494419098,
            0.12761826813220978,
            0.1258423924446106
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.9633058309555054,
          "mean_weights": [
            0.0846441313624382,
            0.05872901901602745,
            0.07210009545087814,
            0.07792744785547256,
            0.05918677896261215,
            0.07445863634347916,
            0.23511825501918793,
            0.18020221590995789,
            0.15763342380523682
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1238560676574707,
          "mean_weights": [
            0.10231245309114456,
            0.07575537264347076,
            0.08897507935762405,
            0.0967075377702713,
            0.09801244735717773,
            0.12438583374023438,
            0.1766587197780609,
            0.12203006446361542,
            0.11516247689723969
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.0835556983947754,
          "mean_weights": [
            0.09133295714855194,
            0.07054752856492996,
            0.09000501036643982,
            0.10956106334924698,
            0.07114207744598389,
            0.11565886437892914,
            0.22511571645736694,
            0.13003571331501007,
            0.09660108387470245
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.062413111329078674,
        0.05931045487523079,
        0.07552354782819748,
        0.07934419065713882,
        0.14653873443603516,
        0.214645653963089,
        0.2105521410703659,
        0.09514089673757553,
        0.2186809629201889,
        0.12821291387081146,
        0.14272794127464294,
        0.1321839988231659,
        0.1287408471107483,
        0.12568901479244232,
        0.13253600895404816,
        0.1335759311914444,
        0.13371841609477997,
        0.07788944244384766,
        0.09585829079151154,
        0.08352623134851456,
        0.09883332997560501,
        0.07934392243623734,
        0.10020580142736435,
        0.09343641996383667,
        0.11237040907144547,
        0.07063088566064835,
        0.08538161963224411,
        0.06870647519826889,
        0.08663177490234375,
        0.07622843235731125,
        0.10358677804470062,
        0.08666495233774185,
        0.10874549299478531,
        0.057347409427165985,
        0.06467291712760925,
        0.05777324363589287,
        0.07278503477573395,
        0.06351501494646072,
        0.08346924930810928,
        0.07469023764133453,
        0.09857872873544693,
        0.051319729536771774,
        0.06093711405992508,
        0.05102822557091713,
        0.06834813207387924,
        0.05902284011244774,
        0.0680762529373169,
        0.06656526774168015,
        0.07369214296340942,
        0.050236333161592484,
        0.05633483827114105,
        0.05184732377529144,
        0.05850831791758537,
        0.061248794198036194,
        0.06479237228631973,
        0.06608261168003082,
        0.06705968081951141,
        0.048912134021520615,
        0.061747051775455475,
        0.062179163098335266,
        0.0654391348361969,
        0.07131517678499222,
        0.07205834239721298,
        0.06802479177713394
      ],
      "loss_nats": 1.2868822813034058,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 1.679484486579895,
        "mean_weights": [
          0.08993907272815704,
          0.043493133038282394,
          0.04922422021627426,
          0.037924446165561676,
          0.03939434140920639,
          0.03898489475250244,
          0.09112586826086044,
          0.32717907428741455,
          0.2827349305152893
        ],
        "sources": 9
      },
      "step": 8000,
      "stream_state_rms_by_sublayer": [
        0.08626964688301086,
        0.16424456238746643,
        0.27676042914390564,
        0.29588937759399414,
        0.41335058212280273,
        0.42337876558303833,
        0.43042075634002686,
        0.45154979825019836,
        0.1360139101743698,
        0.13891471922397614,
        0.15445689857006073,
        0.1555756777524948,
        0.1670466512441635,
        0.16914354264736176,
        0.17834730446338654,
        0.1874145120382309,
        0.05234000086784363,
        0.06211879849433899,
        0.08269349485635757,
        0.09842939674854279,
        0.11548293381929398,
        0.137006014585495,
        0.16410532593727112,
        0.20118924975395203,
        0.062187470495700836,
        0.08399304747581482,
        0.0988159254193306,
        0.12138351052999496,
        0.13199056684970856,
        0.1587214469909668,
        0.17199495434761047,
        0.2071460783481598,
        0.04149825870990753,
        0.0581100657582283,
        0.06434677541255951,
        0.08358953893184662,
        0.08728200942277908,
        0.10947012156248093,
        0.11647984385490417,
        0.1472771316766739,
        0.038355328142642975,
        0.05584336072206497,
        0.06353354454040527,
        0.08318652212619781,
        0.09196189045906067,
        0.11238116025924683,
        0.1169828325510025,
        0.1379220336675644,
        0.027557097375392914,
        0.05995604768395424,
        0.06579430401325226,
        0.09392217546701431,
        0.09665275365114212,
        0.1259489357471466,
        0.12799666821956635,
        0.17086055874824524,
        0.019497457891702652,
        0.06700707972049713,
        0.07176875323057175,
        0.13971327245235443,
        0.14131556451320648,
        0.2211977243423462,
        0.22250843048095703,
        0.32473331689834595
      ]
    }
  ],
  "environment": {
    "autocast": "cuda-bfloat16-forward-fp32-cross-entropy",
    "compile": false,
    "compute_capability": [
      12,
      0
    ],
    "cublas_workspace_config": ":4096:8",
    "cuda": "12.8",
    "deterministic_algorithms": true,
    "gpu": "NVIDIA GeForce RTX 5090",
    "python": "3.10.14",
    "torch": "2.11.0+cu128"
  },
  "evaluations": [
    {
      "bits_per_byte": 8.061194200882085,
      "cross_entropy_nats": 5.587594032287598,
      "step": 0
    },
    {
      "bits_per_byte": 5.191908918496252,
      "cross_entropy_nats": 3.598757028579712,
      "step": 100
    },
    {
      "bits_per_byte": 3.0881952483500545,
      "cross_entropy_nats": 2.1405738294124603,
      "step": 500
    },
    {
      "bits_per_byte": 1.948972795980338,
      "cross_entropy_nats": 1.3509249985218048,
      "step": 2000
    },
    {
      "bits_per_byte": 1.7872753095784035,
      "cross_entropy_nats": 1.2388448417186737,
      "step": 4000
    },
    {
      "bits_per_byte": 1.7030966812870931,
      "cross_entropy_nats": 1.1804966628551483,
      "step": 8000
    }
  ],
  "gradient_gate": null,
  "hashes": {
    "final_mixer_parameters": "94814c85fe1adce99ef9f8a23b6b40ed1a12e214f38d376a95be44bd8a14d14a",
    "final_model_state": "638568aede21890773b6932a19ec4e112f5ac0a4770ba3402fcd82980a9ecf76",
    "final_optimizer_state": "83947fd743ec8e3e31ca7788fd201981846f0f1c7e9e38c88afcef95cfc6ec4e",
    "final_public_parameters": "73e15de1ee917017dc95607378bdbf84b7b54f65be48f3497fdb5654f3f0479e",
    "initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d",
    "initial_public_parameter_elements": 18985152,
    "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670",
    "initial_public_parameter_tensors": 227,
    "initial_public_parameters": "3f93c11463780b386fff00c53b0424f687e1136b1b80f7b626f058adef96c764"
  },
  "manifest": {
    "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716",
    "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371",
    "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
    "input_gate_tensor_hashes": {
      "0": "3eff5c07fa0cfbd5ceed5922cbd09525ad514f801cc0a440c03fd2fa15df4c84",
      "1": "e83e4001c5811b19c69a54c7ea7a802af996a34aa57072cde43a76bc8369e2d0",
      "7999": "d04c4ca879f0209183a48ac54a0c4ffe3a0f62a78888479d4641f6c304426442"
    },
    "path": "experiments/k3/attnres_gradient/manifest.json",
    "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338"
  },
  "model": {
    "attnres_aggregation_groups": 8,
    "context": 256,
    "d_ff": 768,
    "d_head": 32,
    "d_model": 192,
    "heads": 6,
    "layers": 32,
    "parameters": {
      "core": 18985152,
      "embedding": 98304,
      "mixer": 24960,
      "total": 19010112
    },
    "sublayers": 64,
    "sublayers_per_attnres_group": 8,
    "transformer_blocks_per_attnres_group": 4,
    "vocabulary": 256
  },
  "optimizer": {
    "betas": [
      0.9,
      0.95
    ],
    "epsilon": 1e-08,
    "grad_clip": 1.0,
    "min_lr": 3e-05,
    "name": "AdamW",
    "peak_lr": 0.0003,
    "warmup_steps": 400,
    "weight_decay_ndim_ge_2": 0.1
  },
  "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
  "run_kind": "formal",
  "schema_version": 1,
  "seed": 2026073003,
  "steps": 8000,
  "target_bytes_seen": 65536000,
  "timing": {
    "mean_ms": 109.96596761305909,
    "measured_steps": 7980,
    "median_ms": 109.86474549281411,
    "p95_ms": 110.51533734425902,
    "peak_allocated_bytes": 12815843328,
    "peak_reserved_bytes": 13337886720,
    "warmup_steps_excluded": 20
  },
  "training_history": [
    {
      "bits_per_byte": 8.082576459898554,
      "learning_rate": 7.499999999999999e-07,
      "loss_nats": 5.602415084838867,
      "step": 1,
      "unclipped_grad_norm": 19.083253860473633
    },
    {
      "bits_per_byte": 7.337116283872504,
      "learning_rate": 7.499999999999999e-06,
      "loss_nats": 5.0857014656066895,
      "step": 10,
      "unclipped_grad_norm": 11.73559856414795
    },
    {
      "bits_per_byte": 6.794594947933964,
      "learning_rate": 1.4999999999999999e-05,
      "loss_nats": 4.709654331207275,
      "step": 20,
      "unclipped_grad_norm": 3.3705036640167236
    },
    {
      "bits_per_byte": 6.669727976170927,
      "learning_rate": 2.2499999999999998e-05,
      "loss_nats": 4.623103141784668,
      "step": 30,
      "unclipped_grad_norm": 3.037728786468506
    },
    {
      "bits_per_byte": 6.488515360401613,
      "learning_rate": 2.9999999999999997e-05,
      "loss_nats": 4.497496128082275,
      "step": 40,
      "unclipped_grad_norm": 3.071707010269165
    },
    {
      "bits_per_byte": 6.170372909737745,
      "learning_rate": 3.75e-05,
      "loss_nats": 4.276976585388184,
      "step": 50,
      "unclipped_grad_norm": 2.8251144886016846
    },
    {
      "bits_per_byte": 5.9458358004242795,
      "learning_rate": 4.4999999999999996e-05,
      "loss_nats": 4.121339321136475,
      "step": 60,
      "unclipped_grad_norm": 2.6051926612854004
    },
    {
      "bits_per_byte": 5.763860269615738,
      "learning_rate": 5.2499999999999995e-05,
      "loss_nats": 3.9952034950256348,
      "step": 70,
      "unclipped_grad_norm": 2.513765573501587
    },
    {
      "bits_per_byte": 5.595572150058764,
      "learning_rate": 5.9999999999999995e-05,
      "loss_nats": 3.8785550594329834,
      "step": 80,
      "unclipped_grad_norm": 2.595860719680786
    },
    {
      "bits_per_byte": 5.4051244099034195,
      "learning_rate": 6.749999999999999e-05,
      "loss_nats": 3.746546745300293,
      "step": 90,
      "unclipped_grad_norm": 2.317351818084717
    },
    {
      "bits_per_byte": 5.1779332643218385,
      "learning_rate": 7.5e-05,
      "loss_nats": 3.5890698432922363,
      "step": 100,
      "unclipped_grad_norm": 2.214414358139038
    },
    {
      "bits_per_byte": 4.995566300999934,
      "learning_rate": 8.249999999999999e-05,
      "loss_nats": 3.462662696838379,
      "step": 110,
      "unclipped_grad_norm": 2.0333311557769775
    },
    {
      "bits_per_byte": 4.799066491360726,
      "learning_rate": 8.999999999999999e-05,
      "loss_nats": 3.3264594078063965,
      "step": 120,
      "unclipped_grad_norm": 1.8019887208938599
    },
    {
      "bits_per_byte": 4.633962458601826,
      "learning_rate": 9.75e-05,
      "loss_nats": 3.2120180130004883,
      "step": 130,
      "unclipped_grad_norm": 1.6018544435501099
    },
    {
      "bits_per_byte": 4.476944362155892,
      "learning_rate": 0.00010499999999999999,
      "loss_nats": 3.1031813621520996,
      "step": 140,
      "unclipped_grad_norm": 1.3790134191513062
    },
    {
      "bits_per_byte": 4.275779695094682,
      "learning_rate": 0.0001125,
      "loss_nats": 2.963744640350342,
      "step": 150,
      "unclipped_grad_norm": 1.2061023712158203
    },
    {
      "bits_per_byte": 4.128819144071206,
      "learning_rate": 0.00011999999999999999,
      "loss_nats": 2.861879348754883,
      "step": 160,
      "unclipped_grad_norm": 1.0463906526565552
    },
    {
      "bits_per_byte": 4.021535678532382,
      "learning_rate": 0.00012749999999999998,
      "loss_nats": 2.7875161170959473,
      "step": 170,
      "unclipped_grad_norm": 0.9023508429527283
    },
    {
      "bits_per_byte": 3.919193618518149,
      "learning_rate": 0.00013499999999999997,
      "loss_nats": 2.7165780067443848,
      "step": 180,
      "unclipped_grad_norm": 0.9033810496330261
    },
    {
      "bits_per_byte": 3.8348498868820102,
      "learning_rate": 0.0001425,
      "loss_nats": 2.6581153869628906,
      "step": 190,
      "unclipped_grad_norm": 0.8499347567558289
    },
    {
      "bits_per_byte": 3.7665400977858114,
      "learning_rate": 0.00015,
      "loss_nats": 2.610766649246216,
      "step": 200,
      "unclipped_grad_norm": 0.8223105072975159
    },
    {
      "bits_per_byte": 3.6822345462981643,
      "learning_rate": 0.0001575,
      "loss_nats": 2.552330493927002,
      "step": 210,
      "unclipped_grad_norm": 0.8248683214187622
    },
    {
      "bits_per_byte": 3.6564082555833983,
      "learning_rate": 0.00016499999999999997,
      "loss_nats": 2.5344290733337402,
      "step": 220,
      "unclipped_grad_norm": 1.0457000732421875
    },
    {
      "bits_per_byte": 3.6121584953776043,
      "learning_rate": 0.0001725,
      "loss_nats": 2.5037574768066406,
      "step": 230,
      "unclipped_grad_norm": 1.2628414630889893
    },
    {
      "bits_per_byte": 3.5855720413460244,
      "learning_rate": 0.00017999999999999998,
      "loss_nats": 2.4853291511535645,
      "step": 240,
      "unclipped_grad_norm": 0.9464337229728699
    },
    {
      "bits_per_byte": 3.5024318443006237,
      "learning_rate": 0.0001875,
      "loss_nats": 2.4277007579803467,
      "step": 250,
      "unclipped_grad_norm": 0.8258991837501526
    },
    {
      "bits_per_byte": 3.520142273721174,
      "learning_rate": 0.000195,
      "loss_nats": 2.439976692199707,
      "step": 260,
      "unclipped_grad_norm": 1.1166419982910156
    },
    {
      "bits_per_byte": 3.497664141253405,
      "learning_rate": 0.00020249999999999996,
      "loss_nats": 2.42439603805542,
      "step": 270,
      "unclipped_grad_norm": 0.9347699880599976
    },
    {
      "bits_per_byte": 3.454170760746106,
      "learning_rate": 0.00020999999999999998,
      "loss_nats": 2.3942487239837646,
      "step": 280,
      "unclipped_grad_norm": 1.2552868127822876
    },
    {
      "bits_per_byte": 3.4220065653817433,
      "learning_rate": 0.00021749999999999997,
      "loss_nats": 2.3719542026519775,
      "step": 290,
      "unclipped_grad_norm": 0.8032089471817017
    },
    {
      "bits_per_byte": 3.435558454304431,
      "learning_rate": 0.000225,
      "loss_nats": 2.38134765625,
      "step": 300,
      "unclipped_grad_norm": 1.544488787651062
    },
    {
      "bits_per_byte": 3.330305071976392,
      "learning_rate": 0.00023249999999999996,
      "loss_nats": 2.308391571044922,
      "step": 310,
      "unclipped_grad_norm": 0.8757423162460327
    },
    {
      "bits_per_byte": 3.390824390953808,
      "learning_rate": 0.00023999999999999998,
      "loss_nats": 2.3503403663635254,
      "step": 320,
      "unclipped_grad_norm": 0.8823365569114685
    },
    {
      "bits_per_byte": 3.3979087003081485,
      "learning_rate": 0.0002475,
      "loss_nats": 2.355250835418701,
      "step": 330,
      "unclipped_grad_norm": 0.898759663105011
    },
    {
      "bits_per_byte": 3.325636774901372,
      "learning_rate": 0.00025499999999999996,
      "loss_nats": 2.3051557540893555,
      "step": 340,
      "unclipped_grad_norm": 0.9536095261573792
    },
    {
      "bits_per_byte": 3.3170438337336625,
      "learning_rate": 0.0002625,
      "loss_nats": 2.2991995811462402,
      "step": 350,
      "unclipped_grad_norm": 1.2654696702957153
    },
    {
      "bits_per_byte": 3.313786482326321,
      "learning_rate": 0.00026999999999999995,
      "loss_nats": 2.2969417572021484,
      "step": 360,
      "unclipped_grad_norm": 1.3773932456970215
    },
    {
      "bits_per_byte": 3.3236555347634327,
      "learning_rate": 0.00027749999999999997,
      "loss_nats": 2.3037824630737305,
      "step": 370,
      "unclipped_grad_norm": 1.2408338785171509
    },
    {
      "bits_per_byte": 3.208911117864879,
      "learning_rate": 0.000285,
      "loss_nats": 2.224247694015503,
      "step": 380,
      "unclipped_grad_norm": 1.0960922241210938
    },
    {
      "bits_per_byte": 3.2804789462990227,
      "learning_rate": 0.0002925,
      "loss_nats": 2.2738547325134277,
      "step": 390,
      "unclipped_grad_norm": 0.9866538643836975
    },
    {
      "bits_per_byte": 3.27038631641579,
      "learning_rate": 0.0003,
      "loss_nats": 2.2668590545654297,
      "step": 400,
      "unclipped_grad_norm": 1.2531954050064087
    },
    {
      "bits_per_byte": 3.2889521875417316,
      "learning_rate": 0.0002999988466112388,
      "loss_nats": 2.2797279357910156,
      "step": 410,
      "unclipped_grad_norm": 1.448500633239746
    },
    {
      "bits_per_byte": 3.205672684549132,
      "learning_rate": 0.00029999538646466355,
      "loss_nats": 2.2220029830932617,
      "step": 420,
      "unclipped_grad_norm": 0.9654834866523743
    },
    {
      "bits_per_byte": 3.218386673996816,
      "learning_rate": 0.0002999896196193985,
      "loss_nats": 2.2308156490325928,
      "step": 430,
      "unclipped_grad_norm": 0.974204421043396
    },
    {
      "bits_per_byte": 3.1822768526563587,
      "learning_rate": 0.00029998154617398326,
      "loss_nats": 2.2057862281799316,
      "step": 440,
      "unclipped_grad_norm": 1.4095590114593506
    },
    {
      "bits_per_byte": 3.237966210866943,
      "learning_rate": 0.0002999711662663707,
      "loss_nats": 2.244387149810791,
      "step": 450,
      "unclipped_grad_norm": 0.9198341369628906
    },
    {
      "bits_per_byte": 3.1857660366770637,
      "learning_rate": 0.0002999584800739247,
      "loss_nats": 2.208204746246338,
      "step": 460,
      "unclipped_grad_norm": 1.4268778562545776
    },
    {
      "bits_per_byte": 3.1576792059993566,
      "learning_rate": 0.0002999434878134173,
      "loss_nats": 2.1887364387512207,
      "step": 470,
      "unclipped_grad_norm": 1.2876354455947876
    },
    {
      "bits_per_byte": 3.131057323541699,
      "learning_rate": 0.000299926189741025,
      "loss_nats": 2.170283555984497,
      "step": 480,
      "unclipped_grad_norm": 1.0718865394592285
    },
    {
      "bits_per_byte": 3.1147361699748184,
      "learning_rate": 0.000299906586152324,
      "loss_nats": 2.158970594406128,
      "step": 490,
      "unclipped_grad_norm": 0.8680974841117859
    },
    {
      "bits_per_byte": 3.059730958749366,
      "learning_rate": 0.0002998846773822856,
      "loss_nats": 2.1208438873291016,
      "step": 500,
      "unclipped_grad_norm": 1.296195387840271
    },
    {
      "bits_per_byte": 3.12068642572937,
      "learning_rate": 0.0002998604638052702,
      "loss_nats": 2.163094997406006,
      "step": 510,
      "unclipped_grad_norm": 1.0469720363616943
    },
    {
      "bits_per_byte": 3.1397613655018657,
      "learning_rate": 0.0002998339458350211,
      "loss_nats": 2.176316738128662,
      "step": 520,
      "unclipped_grad_norm": 1.2089059352874756
    },
    {
      "bits_per_byte": 2.9844827015867583,
      "learning_rate": 0.0002998051239246572,
      "loss_nats": 2.06868577003479,
      "step": 530,
      "unclipped_grad_norm": 1.1353814601898193
    },
    {
      "bits_per_byte": 3.0015299659055814,
      "learning_rate": 0.00029977399856666537,
      "loss_nats": 2.0805020332336426,
      "step": 540,
      "unclipped_grad_norm": 0.919367253780365
    },
    {
      "bits_per_byte": 2.987599371185694,
      "learning_rate": 0.0002997405702928922,
      "loss_nats": 2.0708460807800293,
      "step": 550,
      "unclipped_grad_norm": 0.8667266368865967
    },
    {
      "bits_per_byte": 2.995183462123442,
      "learning_rate": 0.0002997048396745345,
      "loss_nats": 2.0761029720306396,
      "step": 560,
      "unclipped_grad_norm": 0.9804732203483582
    },
    {
      "bits_per_byte": 2.962294187903042,
      "learning_rate": 0.0002996668073221302,
      "loss_nats": 2.0533058643341064,
      "step": 570,
      "unclipped_grad_norm": 0.9414609670639038
    },
    {
      "bits_per_byte": 2.956716790535561,
      "learning_rate": 0.000299626473885547,
      "loss_nats": 2.0494399070739746,
      "step": 580,
      "unclipped_grad_norm": 0.8727462291717529
    },
    {
      "bits_per_byte": 2.9113505189603663,
      "learning_rate": 0.00029958384005397226,
      "loss_nats": 2.0179944038391113,
      "step": 590,
      "unclipped_grad_norm": 1.1642946004867554
    },
    {
      "bits_per_byte": 2.8665478345320485,
      "learning_rate": 0.0002995389065559004,
      "loss_nats": 1.986939549446106,
      "step": 600,
      "unclipped_grad_norm": 0.9213871359825134
    },
    {
      "bits_per_byte": 2.875505550902238,
      "learning_rate": 0.000299491674159121,
      "loss_nats": 1.9931485652923584,
      "step": 610,
      "unclipped_grad_norm": 0.7122263312339783
    },
    {
      "bits_per_byte": 2.878308696128651,
      "learning_rate": 0.00029944214367070535,
      "loss_nats": 1.9950915575027466,
      "step": 620,
      "unclipped_grad_norm": 1.4622896909713745
    },
    {
      "bits_per_byte": 2.790202156074285,
      "learning_rate": 0.00029939031593699284,
      "loss_nats": 1.934020757675171,
      "step": 630,
      "unclipped_grad_norm": 1.1472660303115845
    },
    {
      "bits_per_byte": 2.7890029210498186,
      "learning_rate": 0.00029933619184357654,
      "loss_nats": 1.9331895112991333,
      "step": 640,
      "unclipped_grad_norm": 1.1472764015197754
    },
    {
      "bits_per_byte": 2.7734051265124666,
      "learning_rate": 0.00029927977231528774,
      "loss_nats": 1.9223779439926147,
      "step": 650,
      "unclipped_grad_norm": 0.9182148575782776
    },
    {
      "bits_per_byte": 2.795219749913207,
      "learning_rate": 0.00029922105831618057,
      "loss_nats": 1.937498688697815,
      "step": 660,
      "unclipped_grad_norm": 1.083983302116394
    },
    {
      "bits_per_byte": 2.7702638633944576,
      "learning_rate": 0.0002991600508495153,
      "loss_nats": 1.9202005863189697,
      "step": 670,
      "unclipped_grad_norm": 1.1721765995025635
    },
    {
      "bits_per_byte": 2.755497089026075,
      "learning_rate": 0.00029909675095774114,
      "loss_nats": 1.9099650382995605,
      "step": 680,
      "unclipped_grad_norm": 1.280714750289917
    },
    {
      "bits_per_byte": 2.7128121989604823,
      "learning_rate": 0.0002990311597224787,
      "loss_nats": 1.8803781270980835,
      "step": 690,
      "unclipped_grad_norm": 1.0380759239196777
    },
    {
      "bits_per_byte": 2.7091302223881604,
      "learning_rate": 0.00029896327826450103,
      "loss_nats": 1.8778259754180908,
      "step": 700,
      "unclipped_grad_norm": 1.1401211023330688
    },
    {
      "bits_per_byte": 2.6870259802033667,
      "learning_rate": 0.0002988931077437152,
      "loss_nats": 1.862504482269287,
      "step": 710,
      "unclipped_grad_norm": 1.6450653076171875
    },
    {
      "bits_per_byte": 2.688485424978589,
      "learning_rate": 0.00029882064935914164,
      "loss_nats": 1.863516092300415,
      "step": 720,
      "unclipped_grad_norm": 1.0721391439437866
    },
    {
      "bits_per_byte": 2.65192277331841,
      "learning_rate": 0.0002987459043488943,
      "loss_nats": 1.8381727933883667,
      "step": 730,
      "unclipped_grad_norm": 1.0645538568496704
    },
    {
      "bits_per_byte": 2.6143492076364594,
      "learning_rate": 0.0002986688739901591,
      "loss_nats": 1.8121287822723389,
      "step": 740,
      "unclipped_grad_norm": 1.1877458095550537
    },
    {
      "bits_per_byte": 2.6277774410331767,
      "learning_rate": 0.0002985895595991723,
      "loss_nats": 1.8214365243911743,
      "step": 750,
      "unclipped_grad_norm": 1.1091431379318237
    },
    {
      "bits_per_byte": 2.540277655771055,
      "learning_rate": 0.00029850796253119824,
      "loss_nats": 1.7607862949371338,
      "step": 760,
      "unclipped_grad_norm": 1.222191333770752
    },
    {
      "bits_per_byte": 2.5790566478529025,
      "learning_rate": 0.00029842408418050544,
      "loss_nats": 1.787665843963623,
      "step": 770,
      "unclipped_grad_norm": 1.254854679107666
    },
    {
      "bits_per_byte": 2.5138998166949715,
      "learning_rate": 0.0002983379259803436,
      "loss_nats": 1.7425025701522827,
      "step": 780,
      "unclipped_grad_norm": 0.9573470950126648
    },
    {
      "bits_per_byte": 2.5258618979930834,
      "learning_rate": 0.0002982494894029186,
      "loss_nats": 1.7507940530776978,
      "step": 790,
      "unclipped_grad_norm": 1.108724594116211
    },
    {
      "bits_per_byte": 2.430956195819507,
      "learning_rate": 0.0002981587759593675,
      "loss_nats": 1.6850104331970215,
      "step": 800,
      "unclipped_grad_norm": 0.9832104444503784
    },
    {
      "bits_per_byte": 2.468076415233783,
      "learning_rate": 0.00029806578719973283,
      "loss_nats": 1.7107402086257935,
      "step": 810,
      "unclipped_grad_norm": 0.9492170214653015
    },
    {
      "bits_per_byte": 2.4602348662676934,
      "learning_rate": 0.00029797052471293596,
      "loss_nats": 1.7053048610687256,
      "step": 820,
      "unclipped_grad_norm": 1.192505121231079
    },
    {
      "bits_per_byte": 2.4583616312344954,
      "learning_rate": 0.0002978729901267499,
      "loss_nats": 1.7040064334869385,
      "step": 830,
      "unclipped_grad_norm": 0.9113606214523315
    },
    {
      "bits_per_byte": 2.4751598646748696,
      "learning_rate": 0.0002977731851077716,
      "loss_nats": 1.7156500816345215,
      "step": 840,
      "unclipped_grad_norm": 1.3671480417251587
    },
    {
      "bits_per_byte": 2.386899400147981,
      "learning_rate": 0.0002976711113613935,
      "loss_nats": 1.6544725894927979,
      "step": 850,
      "unclipped_grad_norm": 1.0324512720108032
    },
    {
      "bits_per_byte": 2.472668523994471,
      "learning_rate": 0.0002975667706317743,
      "loss_nats": 1.7139232158660889,
      "step": 860,
      "unclipped_grad_norm": 1.503003478050232
    },
    {
      "bits_per_byte": 2.4348980381772827,
      "learning_rate": 0.00029746016470180915,
      "loss_nats": 1.6877427101135254,
      "step": 870,
      "unclipped_grad_norm": 1.1144746541976929
    },
    {
      "bits_per_byte": 2.3709108610278693,
      "learning_rate": 0.00029735129539309926,
      "loss_nats": 1.64339017868042,
      "step": 880,
      "unclipped_grad_norm": 1.0134795904159546
    },
    {
      "bits_per_byte": 2.3377405671559597,
      "learning_rate": 0.0002972401645659207,
      "loss_nats": 1.6203982830047607,
      "step": 890,
      "unclipped_grad_norm": 1.0625330209732056
    },
    {
      "bits_per_byte": 2.389408283058768,
      "learning_rate": 0.00029712677411919283,
      "loss_nats": 1.6562116146087646,
      "step": 900,
      "unclipped_grad_norm": 0.9719302654266357
    },
    {
      "bits_per_byte": 2.3782030974071033,
      "learning_rate": 0.0002970111259904454,
      "loss_nats": 1.6484447717666626,
      "step": 910,
      "unclipped_grad_norm": 1.0672563314437866
    },
    {
      "bits_per_byte": 2.3530695528100125,
      "learning_rate": 0.0002968932221557859,
      "loss_nats": 1.6310235261917114,
      "step": 920,
      "unclipped_grad_norm": 1.1197935342788696
    },
    {
      "bits_per_byte": 2.297453115243394,
      "learning_rate": 0.0002967730646298655,
      "loss_nats": 1.5924731492996216,
      "step": 930,
      "unclipped_grad_norm": 1.0027272701263428
    },
    {
      "bits_per_byte": 2.344061961471073,
      "learning_rate": 0.0002966506554658449,
      "loss_nats": 1.6247799396514893,
      "step": 940,
      "unclipped_grad_norm": 1.0651428699493408
    },
    {
      "bits_per_byte": 2.367935217202641,
      "learning_rate": 0.00029652599675535876,
      "loss_nats": 1.6413276195526123,
      "step": 950,
      "unclipped_grad_norm": 0.9822106957435608
    },
    {
      "bits_per_byte": 2.309206925551311,
      "learning_rate": 0.0002963990906284808,
      "loss_nats": 1.6006202697753906,
      "step": 960,
      "unclipped_grad_norm": 0.8860561847686768
    },
    {
      "bits_per_byte": 2.2419830349126597,
      "learning_rate": 0.0002962699392536863,
      "loss_nats": 1.5540242195129395,
      "step": 970,
      "unclipped_grad_norm": 1.1816905736923218
    },
    {
      "bits_per_byte": 2.3851068249780383,
      "learning_rate": 0.0002961385448378161,
      "loss_nats": 1.65323007106781,
      "step": 980,
      "unclipped_grad_norm": 0.9257189035415649
    },
    {
      "bits_per_byte": 2.232579711494085,
      "learning_rate": 0.00029600490962603817,
      "loss_nats": 1.547506332397461,
      "step": 990,
      "unclipped_grad_norm": 0.9634203910827637
    },
    {
      "bits_per_byte": 2.2247954327507324,
      "learning_rate": 0.00029586903590180956,
      "loss_nats": 1.5421106815338135,
      "step": 1000,
      "unclipped_grad_norm": 0.809658944606781
    },
    {
      "bits_per_byte": 2.2354428786656486,
      "learning_rate": 0.00029573092598683735,
      "loss_nats": 1.5494909286499023,
      "step": 1010,
      "unclipped_grad_norm": 1.3530596494674683
    },
    {
      "bits_per_byte": 2.2067197121797495,
      "learning_rate": 0.00029559058224103895,
      "loss_nats": 1.5295815467834473,
      "step": 1020,
      "unclipped_grad_norm": 1.0303069353103638
    },
    {
      "bits_per_byte": 2.1954008459958647,
      "learning_rate": 0.0002954480070625017,
      "loss_nats": 1.5217359066009521,
      "step": 1030,
      "unclipped_grad_norm": 0.9071640968322754
    },
    {
      "bits_per_byte": 2.291673982226451,
      "learning_rate": 0.00029530320288744205,
      "loss_nats": 1.5884673595428467,
      "step": 1040,
      "unclipped_grad_norm": 1.3239798545837402
    },
    {
      "bits_per_byte": 2.196653911590051,
      "learning_rate": 0.0002951561721901638,
      "loss_nats": 1.5226044654846191,
      "step": 1050,
      "unclipped_grad_norm": 0.9085557460784912
    },
    {
      "bits_per_byte": 2.225028985190604,
      "learning_rate": 0.00029500691748301594,
      "loss_nats": 1.5422725677490234,
      "step": 1060,
      "unclipped_grad_norm": 0.9488101005554199
    },
    {
      "bits_per_byte": 2.275690258620563,
      "learning_rate": 0.00029485544131634963,
      "loss_nats": 1.5773882865905762,
      "step": 1070,
      "unclipped_grad_norm": 1.047879695892334
    },
    {
      "bits_per_byte": 2.235951603346901,
      "learning_rate": 0.0002947017462784745,
      "loss_nats": 1.5498435497283936,
      "step": 1080,
      "unclipped_grad_norm": 0.8919407725334167
    },
    {
      "bits_per_byte": 2.1733449309359636,
      "learning_rate": 0.00029454583499561476,
      "loss_nats": 1.5064479112625122,
      "step": 1090,
      "unclipped_grad_norm": 1.0036922693252563
    },
    {
      "bits_per_byte": 2.169043472855234,
      "learning_rate": 0.00029438771013186413,
      "loss_nats": 1.5034663677215576,
      "step": 1100,
      "unclipped_grad_norm": 0.9904995560646057
    },
    {
      "bits_per_byte": 2.124962083664635,
      "learning_rate": 0.0002942273743891401,
      "loss_nats": 1.4729114770889282,
      "step": 1110,
      "unclipped_grad_norm": 0.9176430702209473
    },
    {
      "bits_per_byte": 2.176794214981668,
      "learning_rate": 0.00029406483050713824,
      "loss_nats": 1.5088387727737427,
      "step": 1120,
      "unclipped_grad_norm": 0.9229135513305664
    },
    {
      "bits_per_byte": 2.210271497885368,
      "learning_rate": 0.00029390008126328504,
      "loss_nats": 1.53204345703125,
      "step": 1130,
      "unclipped_grad_norm": 0.871993899345398
    },
    {
      "bits_per_byte": 2.123456375556331,
      "learning_rate": 0.00029373312947269045,
      "loss_nats": 1.4718677997589111,
      "step": 1140,
      "unclipped_grad_norm": 0.8673478960990906
    },
    {
      "bits_per_byte": 2.1911007637763418,
      "learning_rate": 0.00029356397798809995,
      "loss_nats": 1.518755316734314,
      "step": 1150,
      "unclipped_grad_norm": 0.8959671258926392
    },
    {
      "bits_per_byte": 2.0953000638876755,
      "learning_rate": 0.0002933926296998457,
      "loss_nats": 1.4523513317108154,
      "step": 1160,
      "unclipped_grad_norm": 0.9113589525222778
    },
    {
      "bits_per_byte": 2.1570418355474237,
      "learning_rate": 0.0002932190875357972,
      "loss_nats": 1.495147466659546,
      "step": 1170,
      "unclipped_grad_norm": 0.8980047702789307
    },
    {
      "bits_per_byte": 2.1555593450969863,
      "learning_rate": 0.00029304335446131126,
      "loss_nats": 1.4941198825836182,
      "step": 1180,
      "unclipped_grad_norm": 0.948503315448761
    },
    {
      "bits_per_byte": 2.1334348089593913,
      "learning_rate": 0.00029286543347918116,
      "loss_nats": 1.4787843227386475,
      "step": 1190,
      "unclipped_grad_norm": 1.1094794273376465
    },
    {
      "bits_per_byte": 2.1317098229712372,
      "learning_rate": 0.0002926853276295856,
      "loss_nats": 1.4775886535644531,
      "step": 1200,
      "unclipped_grad_norm": 0.8304017782211304
    },
    {
      "bits_per_byte": 2.1385674591917407,
      "learning_rate": 0.0002925030399900368,
      "loss_nats": 1.482342004776001,
      "step": 1210,
      "unclipped_grad_norm": 0.8620252013206482
    },
    {
      "bits_per_byte": 2.0980762078379027,
      "learning_rate": 0.00029231857367532726,
      "loss_nats": 1.4542756080627441,
      "step": 1220,
      "unclipped_grad_norm": 0.88591068983078
    },
    {
      "bits_per_byte": 2.108927281211441,
      "learning_rate": 0.0002921319318374776,
      "loss_nats": 1.4617969989776611,
      "step": 1230,
      "unclipped_grad_norm": 1.0479243993759155
    },
    {
      "bits_per_byte": 2.081558994049039,
      "learning_rate": 0.00029194311766568174,
      "loss_nats": 1.442826747894287,
      "step": 1240,
      "unclipped_grad_norm": 0.9365729689598083
    },
    {
      "bits_per_byte": 2.0172010223043655,
      "learning_rate": 0.000291752134386253,
      "loss_nats": 1.3982172012329102,
      "step": 1250,
      "unclipped_grad_norm": 0.8966301083564758
    },
    {
      "bits_per_byte": 2.0690183631134245,
      "learning_rate": 0.00029155898526256856,
      "loss_nats": 1.4341342449188232,
      "step": 1260,
      "unclipped_grad_norm": 0.8974369168281555
    },
    {
      "bits_per_byte": 2.101840220134686,
      "learning_rate": 0.0002913636735950141,
      "loss_nats": 1.4568846225738525,
      "step": 1270,
      "unclipped_grad_norm": 0.9220007061958313
    },
    {
      "bits_per_byte": 2.1256708241688407,
      "learning_rate": 0.0002911662027209272,
      "loss_nats": 1.473402738571167,
      "step": 1280,
      "unclipped_grad_norm": 0.9313441514968872
    },
    {
      "bits_per_byte": 2.076867651298803,
      "learning_rate": 0.00029096657601454013,
      "loss_nats": 1.439574956893921,
      "step": 1290,
      "unclipped_grad_norm": 0.8451681137084961
    },
    {
      "bits_per_byte": 2.032476865319302,
      "learning_rate": 0.00029076479688692266,
      "loss_nats": 1.4088056087493896,
      "step": 1300,
      "unclipped_grad_norm": 0.9314314126968384
    },
    {
      "bits_per_byte": 2.058903719450881,
      "learning_rate": 0.00029056086878592323,
      "loss_nats": 1.4271233081817627,
      "step": 1310,
      "unclipped_grad_norm": 0.8796194195747375
    },
    {
      "bits_per_byte": 2.0275063947267156,
      "learning_rate": 0.00029035479519611045,
      "loss_nats": 1.4053603410720825,
      "step": 1320,
      "unclipped_grad_norm": 0.9140937924385071
    },
    {
      "bits_per_byte": 2.037489299678698,
      "learning_rate": 0.0002901465796387134,
      "loss_nats": 1.4122799634933472,
      "step": 1330,
      "unclipped_grad_norm": 0.821657657623291
    },
    {
      "bits_per_byte": 2.0199990080512524,
      "learning_rate": 0.00028993622567156154,
      "loss_nats": 1.4001566171646118,
      "step": 1340,
      "unclipped_grad_norm": 0.8321346044540405
    },
    {
      "bits_per_byte": 2.048903616233812,
      "learning_rate": 0.0002897237368890237,
      "loss_nats": 1.420191764831543,
      "step": 1350,
      "unclipped_grad_norm": 0.9421875476837158
    },
    {
      "bits_per_byte": 2.00944598061166,
      "learning_rate": 0.0002895091169219469,
      "loss_nats": 1.3928418159484863,
      "step": 1360,
      "unclipped_grad_norm": 0.8682821393013
    },
    {
      "bits_per_byte": 2.0435301902902543,
      "learning_rate": 0.00028929236943759425,
      "loss_nats": 1.4164671897888184,
      "step": 1370,
      "unclipped_grad_norm": 0.8776147961616516
    },
    {
      "bits_per_byte": 2.0135448431296727,
      "learning_rate": 0.00028907349813958225,
      "loss_nats": 1.39568293094635,
      "step": 1380,
      "unclipped_grad_norm": 0.87440025806427
    },
    {
      "bits_per_byte": 2.0533221944997795,
      "learning_rate": 0.0002888525067678175,
      "loss_nats": 1.4232544898986816,
      "step": 1390,
      "unclipped_grad_norm": 0.7801422476768494
    },
    {
      "bits_per_byte": 2.1564656936670334,
      "learning_rate": 0.00028862939909843273,
      "loss_nats": 1.4947481155395508,
      "step": 1400,
      "unclipped_grad_norm": 0.8561526536941528
    },
    {
      "bits_per_byte": 2.0373921294809607,
      "learning_rate": 0.0002884041789437225,
      "loss_nats": 1.412212610244751,
      "step": 1410,
      "unclipped_grad_norm": 0.8376067876815796
    },
    {
      "bits_per_byte": 2.0325960492963495,
      "learning_rate": 0.0002881768501520778,
      "loss_nats": 1.4088882207870483,
      "step": 1420,
      "unclipped_grad_norm": 0.751315712928772
    },
    {
      "bits_per_byte": 1.9860154082234123,
      "learning_rate": 0.0002879474166079205,
      "loss_nats": 1.376600980758667,
      "step": 1430,
      "unclipped_grad_norm": 0.9057046175003052
    },
    {
      "bits_per_byte": 2.015529006972677,
      "learning_rate": 0.00028771588223163694,
      "loss_nats": 1.3970582485198975,
      "step": 1440,
      "unclipped_grad_norm": 0.8342951536178589
    },
    {
      "bits_per_byte": 1.9818307263626185,
      "learning_rate": 0.00028748225097951054,
      "loss_nats": 1.3737003803253174,
      "step": 1450,
      "unclipped_grad_norm": 0.8142686486244202
    },
    {
      "bits_per_byte": 2.1060192265680016,
      "learning_rate": 0.000287246526843655,
      "loss_nats": 1.459781289100647,
      "step": 1460,
      "unclipped_grad_norm": 0.8357279896736145
    },
    {
      "bits_per_byte": 2.079546797033944,
      "learning_rate": 0.00028700871385194533,
      "loss_nats": 1.441431999206543,
      "step": 1470,
      "unclipped_grad_norm": 0.7720605731010437
    },
    {
      "bits_per_byte": 2.025708316111945,
      "learning_rate": 0.00028676881606794945,
      "loss_nats": 1.404114007949829,
      "step": 1480,
      "unclipped_grad_norm": 0.8328851461410522
    },
    {
      "bits_per_byte": 2.0422327531721454,
      "learning_rate": 0.00028652683759085854,
      "loss_nats": 1.4155678749084473,
      "step": 1490,
      "unclipped_grad_norm": 0.9698525071144104
    },
    {
      "bits_per_byte": 1.9217496191796013,
      "learning_rate": 0.0002862827825554171,
      "loss_nats": 1.3320553302764893,
      "step": 1500,
      "unclipped_grad_norm": 0.7396577000617981
    },
    {
      "bits_per_byte": 1.9436662282923034,
      "learning_rate": 0.00028603665513185245,
      "loss_nats": 1.347246766090393,
      "step": 1510,
      "unclipped_grad_norm": 0.809975266456604
    },
    {
      "bits_per_byte": 1.9993110429963143,
      "learning_rate": 0.000285788459525803,
      "loss_nats": 1.3858168125152588,
      "step": 1520,
      "unclipped_grad_norm": 0.8165730834007263
    },
    {
      "bits_per_byte": 2.03089187320895,
      "learning_rate": 0.00028553819997824707,
      "loss_nats": 1.4077069759368896,
      "step": 1530,
      "unclipped_grad_norm": 0.7620540261268616
    },
    {
      "bits_per_byte": 2.002577509484152,
      "learning_rate": 0.00028528588076542964,
      "loss_nats": 1.3880809545516968,
      "step": 1540,
      "unclipped_grad_norm": 0.8928612470626831
    },
    {
      "bits_per_byte": 2.0629018001354886,
      "learning_rate": 0.00028503150619878996,
      "loss_nats": 1.4298945665359497,
      "step": 1550,
      "unclipped_grad_norm": 0.7963082194328308
    },
    {
      "bits_per_byte": 1.9429003895480113,
      "learning_rate": 0.0002847750806248876,
      "loss_nats": 1.3467159271240234,
      "step": 1560,
      "unclipped_grad_norm": 0.7710158228874207
    },
    {
      "bits_per_byte": 1.9502758655302628,
      "learning_rate": 0.000284516608425328,
      "loss_nats": 1.3518282175064087,
      "step": 1570,
      "unclipped_grad_norm": 0.8057935833930969
    },
    {
      "bits_per_byte": 1.9693647358974786,
      "learning_rate": 0.000284256094016688,
      "loss_nats": 1.3650596141815186,
      "step": 1580,
      "unclipped_grad_norm": 0.8588574528694153
    },
    {
      "bits_per_byte": 1.972771368245769,
      "learning_rate": 0.0002839935418504399,
      "loss_nats": 1.3674209117889404,
      "step": 1590,
      "unclipped_grad_norm": 0.8608299493789673
    },
    {
      "bits_per_byte": 1.9610373359427014,
      "learning_rate": 0.000283728956412876,
      "loss_nats": 1.3592875003814697,
      "step": 1600,
      "unclipped_grad_norm": 0.8587453365325928
    },
    {
      "bits_per_byte": 1.9257736692444893,
      "learning_rate": 0.00028346234222503136,
      "loss_nats": 1.3348445892333984,
      "step": 1610,
      "unclipped_grad_norm": 0.8028180003166199
    },
    {
      "bits_per_byte": 1.929578785394829,
      "learning_rate": 0.00028319370384260675,
      "loss_nats": 1.3374820947647095,
      "step": 1620,
      "unclipped_grad_norm": 0.7991352677345276
    },
    {
      "bits_per_byte": 1.9716548568763677,
      "learning_rate": 0.00028292304585589105,
      "loss_nats": 1.3666470050811768,
      "step": 1630,
      "unclipped_grad_norm": 0.7573540806770325
    },
    {
      "bits_per_byte": 1.9236465878186182,
      "learning_rate": 0.0002826503728896824,
      "loss_nats": 1.3333702087402344,
      "step": 1640,
      "unclipped_grad_norm": 0.8524133563041687
    },
    {
      "bits_per_byte": 1.9335809936630575,
      "learning_rate": 0.0002823756896032095,
      "loss_nats": 1.3402562141418457,
      "step": 1650,
      "unclipped_grad_norm": 0.7908568978309631
    },
    {
      "bits_per_byte": 1.9524230689262851,
      "learning_rate": 0.0002820990006900518,
      "loss_nats": 1.3533165454864502,
      "step": 1660,
      "unclipped_grad_norm": 0.8385109901428223
    },
    {
      "bits_per_byte": 1.8864807930018634,
      "learning_rate": 0.00028182031087805947,
      "loss_nats": 1.3076088428497314,
      "step": 1670,
      "unclipped_grad_norm": 0.8104477524757385
    },
    {
      "bits_per_byte": 1.918672677573015,
      "learning_rate": 0.0002815396249292725,
      "loss_nats": 1.3299225568771362,
      "step": 1680,
      "unclipped_grad_norm": 0.7970330119132996
    },
    {
      "bits_per_byte": 1.9517956762159374,
      "learning_rate": 0.0002812569476398391,
      "loss_nats": 1.352881669998169,
      "step": 1690,
      "unclipped_grad_norm": 0.7619603276252747
    },
    {
      "bits_per_byte": 1.8990867773448057,
      "learning_rate": 0.00028097228383993446,
      "loss_nats": 1.3163466453552246,
      "step": 1700,
      "unclipped_grad_norm": 0.8769286274909973
    },
    {
      "bits_per_byte": 1.938959579086141,
      "learning_rate": 0.0002806856383936774,
      "loss_nats": 1.3439843654632568,
      "step": 1710,
      "unclipped_grad_norm": 0.7246651649475098
    },
    {
      "bits_per_byte": 1.847509524316351,
      "learning_rate": 0.00028039701619904774,
      "loss_nats": 1.2805960178375244,
      "step": 1720,
      "unclipped_grad_norm": 0.7415109276771545
    },
    {
      "bits_per_byte": 1.944612132872049,
      "learning_rate": 0.0002801064221878024,
      "loss_nats": 1.3479024171829224,
      "step": 1730,
      "unclipped_grad_norm": 0.8057509660720825
    },
    {
      "bits_per_byte": 1.956270664791388,
      "learning_rate": 0.00027981386132539134,
      "loss_nats": 1.3559834957122803,
      "step": 1740,
      "unclipped_grad_norm": 0.8188740611076355
    },
    {
      "bits_per_byte": 1.908994697861012,
      "learning_rate": 0.0002795193386108724,
      "loss_nats": 1.3232142925262451,
      "step": 1750,
      "unclipped_grad_norm": 0.7595267295837402
    },
    {
      "bits_per_byte": 1.9438418225788343,
      "learning_rate": 0.00027922285907682625,
      "loss_nats": 1.3473684787750244,
      "step": 1760,
      "unclipped_grad_norm": 0.7832275629043579
    },
    {
      "bits_per_byte": 1.9277638524802738,
      "learning_rate": 0.00027892442778927005,
      "loss_nats": 1.33622407913208,
      "step": 1770,
      "unclipped_grad_norm": 0.7276267409324646
    },
    {
      "bits_per_byte": 1.987644083927083,
      "learning_rate": 0.00027862404984757116,
      "loss_nats": 1.377729892730713,
      "step": 1780,
      "unclipped_grad_norm": 0.7699368000030518
    },
    {
      "bits_per_byte": 1.8704368754681737,
      "learning_rate": 0.0002783217303843597,
      "loss_nats": 1.2964880466461182,
      "step": 1790,
      "unclipped_grad_norm": 0.7781511545181274
    },
    {
      "bits_per_byte": 2.006532250540742,
      "learning_rate": 0.00027801747456544134,
      "loss_nats": 1.390822172164917,
      "step": 1800,
      "unclipped_grad_norm": 0.8113785982131958
    },
    {
      "bits_per_byte": 1.8832052114335305,
      "learning_rate": 0.00027771128758970843,
      "loss_nats": 1.3053383827209473,
      "step": 1810,
      "unclipped_grad_norm": 0.8088275790214539
    },
    {
      "bits_per_byte": 1.9263209180395349,
      "learning_rate": 0.00027740317468905165,
      "loss_nats": 1.335223913192749,
      "step": 1820,
      "unclipped_grad_norm": 0.7881855964660645
    },
    {
      "bits_per_byte": 1.968589438107389,
      "learning_rate": 0.0002770931411282704,
      "loss_nats": 1.3645222187042236,
      "step": 1830,
      "unclipped_grad_norm": 0.7948311567306519
    },
    {
      "bits_per_byte": 1.9398172565659937,
      "learning_rate": 0.0002767811922049827,
      "loss_nats": 1.3445788621902466,
      "step": 1840,
      "unclipped_grad_norm": 0.8343047499656677
    },
    {
      "bits_per_byte": 1.8747907882574166,
      "learning_rate": 0.00027646733324953503,
      "loss_nats": 1.2995059490203857,
      "step": 1850,
      "unclipped_grad_norm": 0.7942258715629578
    },
    {
      "bits_per_byte": 1.9135037390013332,
      "learning_rate": 0.00027615156962491097,
      "loss_nats": 1.3263397216796875,
      "step": 1860,
      "unclipped_grad_norm": 0.769598126411438
    },
    {
      "bits_per_byte": 1.987024774401326,
      "learning_rate": 0.0002758339067266396,
      "loss_nats": 1.3773006200790405,
      "step": 1870,
      "unclipped_grad_norm": 0.7647593021392822
    },
    {
      "bits_per_byte": 1.8769319722606586,
      "learning_rate": 0.0002755143499827035,
      "loss_nats": 1.300990104675293,
      "step": 1880,
      "unclipped_grad_norm": 0.8446611166000366
    },
    {
      "bits_per_byte": 1.8953207012562119,
      "learning_rate": 0.0002751929048534456,
      "loss_nats": 1.3137362003326416,
      "step": 1890,
      "unclipped_grad_norm": 0.7048609852790833
    },
    {
      "bits_per_byte": 1.915848206497895,
      "learning_rate": 0.0002748695768314763,
      "loss_nats": 1.3279647827148438,
      "step": 1900,
      "unclipped_grad_norm": 0.7416327595710754
    },
    {
      "bits_per_byte": 1.9362319342434569,
      "learning_rate": 0.00027454437144157947,
      "loss_nats": 1.3420937061309814,
      "step": 1910,
      "unclipped_grad_norm": 0.8084686398506165
    },
    {
      "bits_per_byte": 1.8860243510464736,
      "learning_rate": 0.00027421729424061787,
      "loss_nats": 1.3072924613952637,
      "step": 1920,
      "unclipped_grad_norm": 0.8338462710380554
    },
    {
      "bits_per_byte": 1.930916638435891,
      "learning_rate": 0.0002738883508174385,
      "loss_nats": 1.338409423828125,
      "step": 1930,
      "unclipped_grad_norm": 0.6878981590270996
    },
    {
      "bits_per_byte": 1.9257588787365152,
      "learning_rate": 0.0002735575467927768,
      "loss_nats": 1.334834337234497,
      "step": 1940,
      "unclipped_grad_norm": 0.6981199383735657
    },
    {
      "bits_per_byte": 1.8699291826828268,
      "learning_rate": 0.0002732248878191609,
      "loss_nats": 1.2961361408233643,
      "step": 1950,
      "unclipped_grad_norm": 0.76193767786026
    },
    {
      "bits_per_byte": 1.859464038377825,
      "learning_rate": 0.0002728903795808148,
      "loss_nats": 1.2888822555541992,
      "step": 1960,
      "unclipped_grad_norm": 0.7367894053459167
    },
    {
      "bits_per_byte": 1.8691335909399354,
      "learning_rate": 0.0002725540277935614,
      "loss_nats": 1.2955846786499023,
      "step": 1970,
      "unclipped_grad_norm": 0.8052698373794556
    },
    {
      "bits_per_byte": 1.8850715671606937,
      "learning_rate": 0.00027221583820472456,
      "loss_nats": 1.3066320419311523,
      "step": 1980,
      "unclipped_grad_norm": 0.7624203562736511
    },
    {
      "bits_per_byte": 1.8715028239382214,
      "learning_rate": 0.0002718758165930314,
      "loss_nats": 1.297226905822754,
      "step": 1990,
      "unclipped_grad_norm": 0.779162585735321
    },
    {
      "bits_per_byte": 1.9325518494802945,
      "learning_rate": 0.0002715339687685131,
      "loss_nats": 1.3395428657531738,
      "step": 2000,
      "unclipped_grad_norm": 0.7498615384101868
    },
    {
      "bits_per_byte": 1.9335421255839624,
      "learning_rate": 0.00027119030057240576,
      "loss_nats": 1.3402292728424072,
      "step": 2010,
      "unclipped_grad_norm": 0.7343811392784119
    },
    {
      "bits_per_byte": 1.838237939608337,
      "learning_rate": 0.0002708448178770506,
      "loss_nats": 1.2741694450378418,
      "step": 2020,
      "unclipped_grad_norm": 0.7684847116470337
    },
    {
      "bits_per_byte": 1.8631659649376469,
      "learning_rate": 0.00027049752658579376,
      "loss_nats": 1.2914482355117798,
      "step": 2030,
      "unclipped_grad_norm": 0.6878870129585266
    },
    {
      "bits_per_byte": 1.8841837927169398,
      "learning_rate": 0.00027014843263288514,
      "loss_nats": 1.3060166835784912,
      "step": 2040,
      "unclipped_grad_norm": 0.853155255317688
    },
    {
      "bits_per_byte": 1.8339112000779305,
      "learning_rate": 0.00026979754198337724,
      "loss_nats": 1.2711703777313232,
      "step": 2050,
      "unclipped_grad_norm": 0.8042898774147034
    },
    {
      "bits_per_byte": 1.8136427007084468,
      "learning_rate": 0.00026944486063302305,
      "loss_nats": 1.2571213245391846,
      "step": 2060,
      "unclipped_grad_norm": 0.7331950664520264
    },
    {
      "bits_per_byte": 1.9306452498128293,
      "learning_rate": 0.0002690903946081738,
      "loss_nats": 1.3382213115692139,
      "step": 2070,
      "unclipped_grad_norm": 0.7544640898704529
    },
    {
      "bits_per_byte": 1.9159075405124426,
      "learning_rate": 0.00026873414996567596,
      "loss_nats": 1.3280059099197388,
      "step": 2080,
      "unclipped_grad_norm": 0.7521310448646545
    },
    {
      "bits_per_byte": 1.8394872215842044,
      "learning_rate": 0.00026837613279276736,
      "loss_nats": 1.2750353813171387,
      "step": 2090,
      "unclipped_grad_norm": 0.7430872321128845
    },
    {
      "bits_per_byte": 1.7890550290451945,
      "learning_rate": 0.0002680163492069739,
      "loss_nats": 1.2400784492492676,
      "step": 2100,
      "unclipped_grad_norm": 0.7162277698516846
    },
    {
      "bits_per_byte": 1.8155914361253733,
      "learning_rate": 0.00026765480535600414,
      "loss_nats": 1.2584720849990845,
      "step": 2110,
      "unclipped_grad_norm": 0.7056337594985962
    },
    {
      "bits_per_byte": 1.834290249840432,
      "learning_rate": 0.0002672915074176452,
      "loss_nats": 1.2714331150054932,
      "step": 2120,
      "unclipped_grad_norm": 0.8666519522666931
    },
    {
      "bits_per_byte": 1.8865812308699674,
      "learning_rate": 0.00026692646159965624,
      "loss_nats": 1.307678461074829,
      "step": 2130,
      "unclipped_grad_norm": 0.762921154499054
    },
    {
      "bits_per_byte": 1.9030090136803828,
      "learning_rate": 0.0002665596741396633,
      "loss_nats": 1.3190653324127197,
      "step": 2140,
      "unclipped_grad_norm": 0.7508745789527893
    },
    {
      "bits_per_byte": 1.8090877402003454,
      "learning_rate": 0.000266191151305052,
      "loss_nats": 1.2539640665054321,
      "step": 2150,
      "unclipped_grad_norm": 0.7472740411758423
    },
    {
      "bits_per_byte": 1.898260228724759,
      "learning_rate": 0.00026582089939286075,
      "loss_nats": 1.3157737255096436,
      "step": 2160,
      "unclipped_grad_norm": 0.7288866639137268
    },
    {
      "bits_per_byte": 1.8098697453138184,
      "learning_rate": 0.0002654489247296732,
      "loss_nats": 1.2545061111450195,
      "step": 2170,
      "unclipped_grad_norm": 0.7270416617393494
    },
    {
      "bits_per_byte": 1.896411071262682,
      "learning_rate": 0.00026507523367151007,
      "loss_nats": 1.3144919872283936,
      "step": 2180,
      "unclipped_grad_norm": 0.7397056221961975
    },
    {
      "bits_per_byte": 1.8517294626205718,
      "learning_rate": 0.00026469983260372037,
      "loss_nats": 1.283521056175232,
      "step": 2190,
      "unclipped_grad_norm": 0.7186560034751892
    },
    {
      "bits_per_byte": 1.8299633383273748,
      "learning_rate": 0.0002643227279408727,
      "loss_nats": 1.268433928489685,
      "step": 2200,
      "unclipped_grad_norm": 0.6916610598564148
    },
    {
      "bits_per_byte": 1.8274437924922347,
      "learning_rate": 0.0002639439261266453,
      "loss_nats": 1.2666875123977661,
      "step": 2210,
      "unclipped_grad_norm": 0.7494430541992188
    },
    {
      "bits_per_byte": 1.8262679471082857,
      "learning_rate": 0.000263563433633716,
      "loss_nats": 1.2658724784851074,
      "step": 2220,
      "unclipped_grad_norm": 0.782909095287323
    },
    {
      "bits_per_byte": 1.782249847533204,
      "learning_rate": 0.00026318125696365156,
      "loss_nats": 1.2353614568710327,
      "step": 2230,
      "unclipped_grad_norm": 0.7370295524597168
    },
    {
      "bits_per_byte": 1.8783687153259665,
      "learning_rate": 0.000262797402646797,
      "loss_nats": 1.3019859790802002,
      "step": 2240,
      "unclipped_grad_norm": 0.7077252268791199
    },
    {
      "bits_per_byte": 1.844123529886164,
      "learning_rate": 0.00026241187724216334,
      "loss_nats": 1.2782490253448486,
      "step": 2250,
      "unclipped_grad_norm": 0.7420696020126343
    },
    {
      "bits_per_byte": 1.897104505310961,
      "learning_rate": 0.0002620246873373162,
      "loss_nats": 1.3149726390838623,
      "step": 2260,
      "unclipped_grad_norm": 0.7444695234298706
    },
    {
      "bits_per_byte": 1.7929542197055561,
      "learning_rate": 0.00026163583954826264,
      "loss_nats": 1.242781162261963,
      "step": 2270,
      "unclipped_grad_norm": 0.7164454460144043
    },
    {
      "bits_per_byte": 1.8643507534194408,
      "learning_rate": 0.0002612453405193386,
      "loss_nats": 1.2922694683074951,
      "step": 2280,
      "unclipped_grad_norm": 0.7152382135391235
    },
    {
      "bits_per_byte": 1.8035947863144381,
      "learning_rate": 0.000260853196923095,
      "loss_nats": 1.2501566410064697,
      "step": 2290,
      "unclipped_grad_norm": 0.7178454399108887
    },
    {
      "bits_per_byte": 1.7935200426268947,
      "learning_rate": 0.0002604594154601839,
      "loss_nats": 1.243173360824585,
      "step": 2300,
      "unclipped_grad_norm": 0.7068437337875366
    },
    {
      "bits_per_byte": 1.8020768674379228,
      "learning_rate": 0.0002600640028592441,
      "loss_nats": 1.2491044998168945,
      "step": 2310,
      "unclipped_grad_norm": 0.7017168998718262
    },
    {
      "bits_per_byte": 1.8564474626816911,
      "learning_rate": 0.0002596669658767859,
      "loss_nats": 1.2867913246154785,
      "step": 2320,
      "unclipped_grad_norm": 0.6947462558746338
    },
    {
      "bits_per_byte": 1.7936961528613782,
      "learning_rate": 0.000259268311297076,
      "loss_nats": 1.243295431137085,
      "step": 2330,
      "unclipped_grad_norm": 0.6949398517608643
    },
    {
      "bits_per_byte": 1.8129625093242843,
      "learning_rate": 0.00025886804593202114,
      "loss_nats": 1.2566498517990112,
      "step": 2340,
      "unclipped_grad_norm": 0.7047421336174011
    },
    {
      "bits_per_byte": 1.871945851246844,
      "learning_rate": 0.00025846617662105193,
      "loss_nats": 1.2975339889526367,
      "step": 2350,
      "unclipped_grad_norm": 0.7291849851608276
    },
    {
      "bits_per_byte": 1.7987307729827362,
      "learning_rate": 0.00025806271023100637,
      "loss_nats": 1.2467851638793945,
      "step": 2360,
      "unclipped_grad_norm": 0.7353813052177429
    },
    {
      "bits_per_byte": 1.7311453748119714,
      "learning_rate": 0.00025765765365601165,
      "loss_nats": 1.1999385356903076,
      "step": 2370,
      "unclipped_grad_norm": 0.6746760010719299
    },
    {
      "bits_per_byte": 1.834468595849377,
      "learning_rate": 0.00025725101381736707,
      "loss_nats": 1.2715567350387573,
      "step": 2380,
      "unclipped_grad_norm": 0.7166123390197754
    },
    {
      "bits_per_byte": 1.8260691351638882,
      "learning_rate": 0.00025684279766342567,
      "loss_nats": 1.2657346725463867,
      "step": 2390,
      "unclipped_grad_norm": 0.6613916158676147
    },
    {
      "bits_per_byte": 1.8641103216735346,
      "learning_rate": 0.00025643301216947504,
      "loss_nats": 1.2921028137207031,
      "step": 2400,
      "unclipped_grad_norm": 0.824955403804779
    },
    {
      "bits_per_byte": 1.793840446305452,
      "learning_rate": 0.00025602166433761866,
      "loss_nats": 1.243395447731018,
      "step": 2410,
      "unclipped_grad_norm": 0.7929511070251465
    },
    {
      "bits_per_byte": 1.8233349549804712,
      "learning_rate": 0.0002556087611966561,
      "loss_nats": 1.2638394832611084,
      "step": 2420,
      "unclipped_grad_norm": 0.725745677947998
    },
    {
      "bits_per_byte": 1.825773496987055,
      "learning_rate": 0.00025519430980196277,
      "loss_nats": 1.265529751777649,
      "step": 2430,
      "unclipped_grad_norm": 0.7284960746765137
    },
    {
      "bits_per_byte": 1.7725216488871491,
      "learning_rate": 0.0002547783172353695,
      "loss_nats": 1.2286183834075928,
      "step": 2440,
      "unclipped_grad_norm": 0.7458904981613159
    },
    {
      "bits_per_byte": 1.9001444706476118,
      "learning_rate": 0.0002543607906050415,
      "loss_nats": 1.317079782485962,
      "step": 2450,
      "unclipped_grad_norm": 0.8321223258972168
    },
    {
      "bits_per_byte": 1.7654523020234334,
      "learning_rate": 0.0002539417370453568,
      "loss_nats": 1.223718285560608,
      "step": 2460,
      "unclipped_grad_norm": 0.7338651418685913
    },
    {
      "bits_per_byte": 1.7862824967306352,
      "learning_rate": 0.00025352116371678464,
      "loss_nats": 1.2381566762924194,
      "step": 2470,
      "unclipped_grad_norm": 0.7279908061027527
    },
    {
      "bits_per_byte": 1.8666427662074894,
      "learning_rate": 0.0002530990778057627,
      "loss_nats": 1.2938581705093384,
      "step": 2480,
      "unclipped_grad_norm": 0.7701565027236938
    },
    {
      "bits_per_byte": 1.850673661126925,
      "learning_rate": 0.00025267548652457475,
      "loss_nats": 1.2827892303466797,
      "step": 2490,
      "unclipped_grad_norm": 0.6776601672172546
    },
    {
      "bits_per_byte": 1.8000481600883451,
      "learning_rate": 0.00025225039711122696,
      "loss_nats": 1.2476983070373535,
      "step": 2500,
      "unclipped_grad_norm": 0.7444605231285095
    },
    {
      "bits_per_byte": 1.7603095048146828,
      "learning_rate": 0.00025182381682932464,
      "loss_nats": 1.220153570175171,
      "step": 2510,
      "unclipped_grad_norm": 0.7512273788452148
    },
    {
      "bits_per_byte": 1.7805241736144466,
      "learning_rate": 0.0002513957529679477,
      "loss_nats": 1.2341653108596802,
      "step": 2520,
      "unclipped_grad_norm": 0.6968159079551697
    },
    {
      "bits_per_byte": 1.803689032807111,
      "learning_rate": 0.00025096621284152656,
      "loss_nats": 1.2502219676971436,
      "step": 2530,
      "unclipped_grad_norm": 0.7150991559028625
    },
    {
      "bits_per_byte": 1.8049077018711248,
      "learning_rate": 0.0002505352037897168,
      "loss_nats": 1.2510666847229004,
      "step": 2540,
      "unclipped_grad_norm": 0.7343016862869263
    },
    {
      "bits_per_byte": 1.742795479581418,
      "learning_rate": 0.0002501027331772739,
      "loss_nats": 1.2080137729644775,
      "step": 2550,
      "unclipped_grad_norm": 0.7049802541732788
    },
    {
      "bits_per_byte": 1.7179068382791558,
      "learning_rate": 0.00024966880839392746,
      "loss_nats": 1.1907622814178467,
      "step": 2560,
      "unclipped_grad_norm": 0.7055768370628357
    },
    {
      "bits_per_byte": 1.7731971967397384,
      "learning_rate": 0.0002492334368542548,
      "loss_nats": 1.2290866374969482,
      "step": 2570,
      "unclipped_grad_norm": 0.6996883749961853
    },
    {
      "bits_per_byte": 1.7458676056737803,
      "learning_rate": 0.00024879662599755416,
      "loss_nats": 1.2101432085037231,
      "step": 2580,
      "unclipped_grad_norm": 0.6933735609054565
    },
    {
      "bits_per_byte": 1.7458146350173145,
      "learning_rate": 0.00024835838328771803,
      "loss_nats": 1.2101064920425415,
      "step": 2590,
      "unclipped_grad_norm": 0.7094681859016418
    },
    {
      "bits_per_byte": 1.7637799427264436,
      "learning_rate": 0.0002479187162131051,
      "loss_nats": 1.2225590944290161,
      "step": 2600,
      "unclipped_grad_norm": 0.7199745178222656
    },
    {
      "bits_per_byte": 1.786311561798631,
      "learning_rate": 0.00024747763228641264,
      "loss_nats": 1.2381768226623535,
      "step": 2610,
      "unclipped_grad_norm": 0.7421150207519531
    },
    {
      "bits_per_byte": 1.7252271078304804,
      "learning_rate": 0.00024703513904454795,
      "loss_nats": 1.1958363056182861,
      "step": 2620,
      "unclipped_grad_norm": 0.7062536478042603
    },
    {
      "bits_per_byte": 1.7243496523457782,
      "learning_rate": 0.00024659124404849956,
      "loss_nats": 1.195228099822998,
      "step": 2630,
      "unclipped_grad_norm": 0.6937817931175232
    },
    {
      "bits_per_byte": 1.7124126805146924,
      "learning_rate": 0.00024614595488320833,
      "loss_nats": 1.1869540214538574,
      "step": 2640,
      "unclipped_grad_norm": 0.7252171039581299
    },
    {
      "bits_per_byte": 1.781446344588373,
      "learning_rate": 0.00024569927915743745,
      "loss_nats": 1.2348045110702515,
      "step": 2650,
      "unclipped_grad_norm": 0.6726356744766235
    },
    {
      "bits_per_byte": 1.6915262474806587,
      "learning_rate": 0.0002452512245036427,
      "loss_nats": 1.1724766492843628,
      "step": 2660,
      "unclipped_grad_norm": 0.6618354320526123
    },
    {
      "bits_per_byte": 1.7563157236963467,
      "learning_rate": 0.00024480179857784177,
      "loss_nats": 1.2173852920532227,
      "step": 2670,
      "unclipped_grad_norm": 0.7113842368125916
    },
    {
      "bits_per_byte": 1.858153874543552,
      "learning_rate": 0.00024435100905948387,
      "loss_nats": 1.2879741191864014,
      "step": 2680,
      "unclipped_grad_norm": 0.7770810127258301
    },
    {
      "bits_per_byte": 1.784997614346039,
      "learning_rate": 0.00024389886365131793,
      "loss_nats": 1.2372660636901855,
      "step": 2690,
      "unclipped_grad_norm": 0.7249491810798645
    },
    {
      "bits_per_byte": 1.7001362149308035,
      "learning_rate": 0.00024344537007926163,
      "loss_nats": 1.1784446239471436,
      "step": 2700,
      "unclipped_grad_norm": 0.6978740096092224
    },
    {
      "bits_per_byte": 1.7671420315681605,
      "learning_rate": 0.00024299053609226878,
      "loss_nats": 1.2248895168304443,
      "step": 2710,
      "unclipped_grad_norm": 0.6759880185127258
    },
    {
      "bits_per_byte": 1.8404900524213854,
      "learning_rate": 0.00024253436946219733,
      "loss_nats": 1.2757304906845093,
      "step": 2720,
      "unclipped_grad_norm": 0.7600791454315186
    },
    {
      "bits_per_byte": 1.708491820040322,
      "learning_rate": 0.0002420768779836764,
      "loss_nats": 1.1842362880706787,
      "step": 2730,
      "unclipped_grad_norm": 0.6715761423110962
    },
    {
      "bits_per_byte": 1.7849604660934528,
      "learning_rate": 0.00024161806947397305,
      "loss_nats": 1.2372403144836426,
      "step": 2740,
      "unclipped_grad_norm": 0.7194302082061768
    },
    {
      "bits_per_byte": 1.7485555225241152,
      "learning_rate": 0.00024115795177285886,
      "loss_nats": 1.2120063304901123,
      "step": 2750,
      "unclipped_grad_norm": 0.6991336941719055
    },
    {
      "bits_per_byte": 1.8629487508496072,
      "learning_rate": 0.0002406965327424758,
      "loss_nats": 1.2912976741790771,
      "step": 2760,
      "unclipped_grad_norm": 0.8377771973609924
    },
    {
      "bits_per_byte": 1.7856594035865592,
      "learning_rate": 0.00024023382026720197,
      "loss_nats": 1.237724781036377,
      "step": 2770,
      "unclipped_grad_norm": 0.7150954604148865
    },
    {
      "bits_per_byte": 1.7901013714930438,
      "learning_rate": 0.00023976982225351698,
      "loss_nats": 1.2408037185668945,
      "step": 2780,
      "unclipped_grad_norm": 0.6968044638633728
    },
    {
      "bits_per_byte": 1.752669863480706,
      "learning_rate": 0.0002393045466298666,
      "loss_nats": 1.2148581743240356,
      "step": 2790,
      "unclipped_grad_norm": 0.7223197817802429
    },
    {
      "bits_per_byte": 1.7567675221201633,
      "learning_rate": 0.00023883800134652763,
      "loss_nats": 1.2176984548568726,
      "step": 2800,
      "unclipped_grad_norm": 0.7307369709014893
    },
    {
      "bits_per_byte": 1.7124591158304254,
      "learning_rate": 0.0002383701943754715,
      "loss_nats": 1.1869862079620361,
      "step": 2810,
      "unclipped_grad_norm": 0.7025272250175476
    },
    {
      "bits_per_byte": 1.7803284573577647,
      "learning_rate": 0.00023790113371022884,
      "loss_nats": 1.2340296506881714,
      "step": 2820,
      "unclipped_grad_norm": 0.7275472283363342
    },
    {
      "bits_per_byte": 1.7225665362216327,
      "learning_rate": 0.0002374308273657521,
      "loss_nats": 1.1939921379089355,
      "step": 2830,
      "unclipped_grad_norm": 0.6614095568656921
    },
    {
      "bits_per_byte": 1.7602923065495966,
      "learning_rate": 0.00023695928337827911,
      "loss_nats": 1.2201416492462158,
      "step": 2840,
      "unclipped_grad_norm": 0.7144313454627991
    },
    {
      "bits_per_byte": 1.7021394688480533,
      "learning_rate": 0.00023648650980519564,
      "loss_nats": 1.179833173751831,
      "step": 2850,
      "unclipped_grad_norm": 0.7220690846443176
    },
    {
      "bits_per_byte": 1.744293620453084,
      "learning_rate": 0.00023601251472489757,
      "loss_nats": 1.2090522050857544,
      "step": 2860,
      "unclipped_grad_norm": 0.7174125909805298
    },
    {
      "bits_per_byte": 1.7433115995166573,
      "learning_rate": 0.00023553730623665305,
      "loss_nats": 1.2083715200424194,
      "step": 2870,
      "unclipped_grad_norm": 0.730305552482605
    },
    {
      "bits_per_byte": 1.7421880368585705,
      "learning_rate": 0.00023506089246046403,
      "loss_nats": 1.2075927257537842,
      "step": 2880,
      "unclipped_grad_norm": 0.7075303792953491
    },
    {
      "bits_per_byte": 1.7085926018737276,
      "learning_rate": 0.00023458328153692736,
      "loss_nats": 1.1843061447143555,
      "step": 2890,
      "unclipped_grad_norm": 0.6744402050971985
    },
    {
      "bits_per_byte": 1.7378320602775172,
      "learning_rate": 0.0002341044816270961,
      "loss_nats": 1.204573392868042,
      "step": 2900,
      "unclipped_grad_norm": 0.6889722347259521
    },
    {
      "bits_per_byte": 1.7031551983840492,
      "learning_rate": 0.00023362450091233953,
      "loss_nats": 1.180537223815918,
      "step": 2910,
      "unclipped_grad_norm": 0.7552688717842102
    },
    {
      "bits_per_byte": 1.7508614659068837,
      "learning_rate": 0.00023314334759420388,
      "loss_nats": 1.2136046886444092,
      "step": 2920,
      "unclipped_grad_norm": 0.6863148808479309
    },
    {
      "bits_per_byte": 1.701510700276499,
      "learning_rate": 0.0002326610298942717,
      "loss_nats": 1.1793973445892334,
      "step": 2930,
      "unclipped_grad_norm": 0.6827563047409058
    },
    {
      "bits_per_byte": 1.7106488264474435,
      "learning_rate": 0.00023217755605402175,
      "loss_nats": 1.1857314109802246,
      "step": 2940,
      "unclipped_grad_norm": 0.6807736158370972
    },
    {
      "bits_per_byte": 1.7703209588867088,
      "learning_rate": 0.00023169293433468802,
      "loss_nats": 1.227092981338501,
      "step": 2950,
      "unclipped_grad_norm": 0.6866815686225891
    },
    {
      "bits_per_byte": 1.7651767858167513,
      "learning_rate": 0.00023120717301711857,
      "loss_nats": 1.2235273122787476,
      "step": 2960,
      "unclipped_grad_norm": 0.741482138633728
    },
    {
      "bits_per_byte": 1.7465264712092357,
      "learning_rate": 0.00023072028040163408,
      "loss_nats": 1.2105998992919922,
      "step": 2970,
      "unclipped_grad_norm": 0.6754798293113708
    },
    {
      "bits_per_byte": 1.7663952828981142,
      "learning_rate": 0.00023023226480788593,
      "loss_nats": 1.2243719100952148,
      "step": 2980,
      "unclipped_grad_norm": 0.688591480255127
    },
    {
      "bits_per_byte": 1.7069632382394535,
      "learning_rate": 0.00022974313457471416,
      "loss_nats": 1.1831767559051514,
      "step": 2990,
      "unclipped_grad_norm": 0.7298949956893921
    },
    {
      "bits_per_byte": 1.79113911480835,
      "learning_rate": 0.0002292528980600049,
      "loss_nats": 1.241523027420044,
      "step": 3000,
      "unclipped_grad_norm": 0.713433027267456
    },
    {
      "bits_per_byte": 1.6766162115641075,
      "learning_rate": 0.00022876156364054758,
      "loss_nats": 1.1621417999267578,
      "step": 3010,
      "unclipped_grad_norm": 0.7228698134422302
    },
    {
      "bits_per_byte": 1.7365501015979858,
      "learning_rate": 0.0002282691397118918,
      "loss_nats": 1.2036848068237305,
      "step": 3020,
      "unclipped_grad_norm": 0.6969108581542969
    },
    {
      "bits_per_byte": 1.6833617150962485,
      "learning_rate": 0.00022777563468820376,
      "loss_nats": 1.1668174266815186,
      "step": 3030,
      "unclipped_grad_norm": 0.669261634349823
    },
    {
      "bits_per_byte": 1.6671776317021056,
      "learning_rate": 0.00022728105700212282,
      "loss_nats": 1.1555994749069214,
      "step": 3040,
      "unclipped_grad_norm": 0.6839757561683655
    },
    {
      "bits_per_byte": 1.8241205717296125,
      "learning_rate": 0.00022678541510461698,
      "loss_nats": 1.2643840312957764,
      "step": 3050,
      "unclipped_grad_norm": 0.7963703870773315
    },
    {
      "bits_per_byte": 1.7604708245411922,
      "learning_rate": 0.00022628871746483876,
      "loss_nats": 1.2202653884887695,
      "step": 3060,
      "unclipped_grad_norm": 0.7280165553092957
    },
    {
      "bits_per_byte": 1.6594571585222233,
      "learning_rate": 0.0002257909725699804,
      "loss_nats": 1.1502480506896973,
      "step": 3070,
      "unclipped_grad_norm": 0.7014188766479492
    },
    {
      "bits_per_byte": 1.7706030104341237,
      "learning_rate": 0.00022529218892512886,
      "loss_nats": 1.2272884845733643,
      "step": 3080,
      "unclipped_grad_norm": 0.7385008335113525
    },
    {
      "bits_per_byte": 1.7493225651469633,
      "learning_rate": 0.00022479237505312034,
      "loss_nats": 1.2125380039215088,
      "step": 3090,
      "unclipped_grad_norm": 0.6976860165596008
    },
    {
      "bits_per_byte": 1.706454857523503,
      "learning_rate": 0.00022429153949439496,
      "loss_nats": 1.1828243732452393,
      "step": 3100,
      "unclipped_grad_norm": 0.7350029349327087
    },
    {
      "bits_per_byte": 1.8454660264587992,
      "learning_rate": 0.00022378969080685053,
      "loss_nats": 1.279179573059082,
      "step": 3110,
      "unclipped_grad_norm": 0.8262335658073425
    },
    {
      "bits_per_byte": 1.7237298268720687,
      "learning_rate": 0.0002232868375656965,
      "loss_nats": 1.194798469543457,
      "step": 3120,
      "unclipped_grad_norm": 0.7770306468009949
    },
    {
      "bits_per_byte": 1.6658043502349662,
      "learning_rate": 0.00022278298836330733,
      "loss_nats": 1.1546475887298584,
      "step": 3130,
      "unclipped_grad_norm": 0.6939164996147156
    },
    {
      "bits_per_byte": 1.7147224075157799,
      "learning_rate": 0.00022227815180907572,
      "loss_nats": 1.1885550022125244,
      "step": 3140,
      "unclipped_grad_norm": 0.744745135307312
    },
    {
      "bits_per_byte": 1.7012960659282221,
      "learning_rate": 0.00022177233652926546,
      "loss_nats": 1.179248571395874,
      "step": 3150,
      "unclipped_grad_norm": 0.6927894353866577
    },
    {
      "bits_per_byte": 1.700822425707746,
      "learning_rate": 0.00022126555116686416,
      "loss_nats": 1.1789202690124512,
      "step": 3160,
      "unclipped_grad_norm": 0.6962236166000366
    },
    {
      "bits_per_byte": 1.7730090477196945,
      "learning_rate": 0.00022075780438143532,
      "loss_nats": 1.2289562225341797,
      "step": 3170,
      "unclipped_grad_norm": 0.7581587433815002
    },
    {
      "bits_per_byte": 1.7445914944043783,
      "learning_rate": 0.00022024910484897065,
      "loss_nats": 1.2092586755752563,
      "step": 3180,
      "unclipped_grad_norm": 0.7753618359565735
    },
    {
      "bits_per_byte": 1.7320080397886992,
      "learning_rate": 0.00021973946126174158,
      "loss_nats": 1.2005364894866943,
      "step": 3190,
      "unclipped_grad_norm": 0.7117310166358948
    },
    {
      "bits_per_byte": 1.7215322325593436,
      "learning_rate": 0.00021922888232815086,
      "loss_nats": 1.1932752132415771,
      "step": 3200,
      "unclipped_grad_norm": 0.712298572063446
    },
    {
      "bits_per_byte": 1.6960516269728116,
      "learning_rate": 0.0002187173767725837,
      "loss_nats": 1.1756134033203125,
      "step": 3210,
      "unclipped_grad_norm": 0.7590813636779785
    },
    {
      "bits_per_byte": 1.6593746068498092,
      "learning_rate": 0.0002182049533352588,
      "loss_nats": 1.150190830230713,
      "step": 3220,
      "unclipped_grad_norm": 0.6987801194190979
    },
    {
      "bits_per_byte": 1.7324070395387008,
      "learning_rate": 0.00021769162077207884,
      "loss_nats": 1.2008130550384521,
      "step": 3230,
      "unclipped_grad_norm": 0.6938228607177734
    },
    {
      "bits_per_byte": 1.7474627447605329,
      "learning_rate": 0.00021717738785448102,
      "loss_nats": 1.2112488746643066,
      "step": 3240,
      "unclipped_grad_norm": 0.78226238489151
    },
    {
      "bits_per_byte": 1.7110863503112386,
      "learning_rate": 0.00021666226336928708,
      "loss_nats": 1.1860346794128418,
      "step": 3250,
      "unclipped_grad_norm": 0.6999595165252686
    },
    {
      "bits_per_byte": 1.7142417160066183,
      "learning_rate": 0.0002161462561185532,
      "loss_nats": 1.18822181224823,
      "step": 3260,
      "unclipped_grad_norm": 0.700433075428009
    },
    {
      "bits_per_byte": 1.748484321706658,
      "learning_rate": 0.00021562937491941957,
      "loss_nats": 1.2119569778442383,
      "step": 3270,
      "unclipped_grad_norm": 0.7274234294891357
    },
    {
      "bits_per_byte": 1.738793787261142,
      "learning_rate": 0.00021511162860395975,
      "loss_nats": 1.20524001121521,
      "step": 3280,
      "unclipped_grad_norm": 0.7183425426483154
    },
    {
      "bits_per_byte": 1.6962500949519075,
      "learning_rate": 0.0002145930260190298,
      "loss_nats": 1.175750970840454,
      "step": 3290,
      "unclipped_grad_norm": 0.7457373738288879
    },
    {
      "bits_per_byte": 1.7210296992535226,
      "learning_rate": 0.00021407357602611698,
      "loss_nats": 1.1929268836975098,
      "step": 3300,
      "unclipped_grad_norm": 0.7834483981132507
    },
    {
      "bits_per_byte": 1.6835460804979734,
      "learning_rate": 0.0002135532875011885,
      "loss_nats": 1.166945219039917,
      "step": 3310,
      "unclipped_grad_norm": 0.6523665189743042
    },
    {
      "bits_per_byte": 1.6871995079502522,
      "learning_rate": 0.0002130321693345397,
      "loss_nats": 1.1694775819778442,
      "step": 3320,
      "unclipped_grad_norm": 0.7348048686981201
    },
    {
      "bits_per_byte": 1.7008401399207849,
      "learning_rate": 0.00021251023043064224,
      "loss_nats": 1.178932547569275,
      "step": 3330,
      "unclipped_grad_norm": 0.7228167653083801
    },
    {
      "bits_per_byte": 1.643402750046831,
      "learning_rate": 0.00021198747970799188,
      "loss_nats": 1.1391199827194214,
      "step": 3340,
      "unclipped_grad_norm": 0.7094708681106567
    },
    {
      "bits_per_byte": 1.686364876145615,
      "learning_rate": 0.00021146392609895618,
      "loss_nats": 1.1688990592956543,
      "step": 3350,
      "unclipped_grad_norm": 0.7186631560325623
    },
    {
      "bits_per_byte": 1.7065993229502279,
      "learning_rate": 0.00021093957854962178,
      "loss_nats": 1.182924509048462,
      "step": 3360,
      "unclipped_grad_norm": 0.6950210332870483
    },
    {
      "bits_per_byte": 1.6548356407282379,
      "learning_rate": 0.00021041444601964157,
      "loss_nats": 1.1470446586608887,
      "step": 3370,
      "unclipped_grad_norm": 0.6926314830780029
    },
    {
      "bits_per_byte": 1.6926787032240904,
      "learning_rate": 0.00020988853748208164,
      "loss_nats": 1.1732754707336426,
      "step": 3380,
      "unclipped_grad_norm": 0.7107762694358826
    },
    {
      "bits_per_byte": 1.7003921251152874,
      "learning_rate": 0.00020936186192326782,
      "loss_nats": 1.1786220073699951,
      "step": 3390,
      "unclipped_grad_norm": 0.700124979019165
    },
    {
      "bits_per_byte": 1.6889117672222422,
      "learning_rate": 0.00020883442834263225,
      "loss_nats": 1.1706644296646118,
      "step": 3400,
      "unclipped_grad_norm": 0.7049819827079773
    },
    {
      "bits_per_byte": 1.6802605239358903,
      "learning_rate": 0.0002083062457525596,
      "loss_nats": 1.1646678447723389,
      "step": 3410,
      "unclipped_grad_norm": 0.6922183036804199
    },
    {
      "bits_per_byte": 1.664921907253389,
      "learning_rate": 0.00020777732317823306,
      "loss_nats": 1.1540359258651733,
      "step": 3420,
      "unclipped_grad_norm": 0.7144856452941895
    },
    {
      "bits_per_byte": 1.698013089105902,
      "learning_rate": 0.00020724766965748005,
      "loss_nats": 1.1769729852676392,
      "step": 3430,
      "unclipped_grad_norm": 0.6953245401382446
    },
    {
      "bits_per_byte": 1.7072490734051875,
      "learning_rate": 0.00020671729424061788,
      "loss_nats": 1.1833748817443848,
      "step": 3440,
      "unclipped_grad_norm": 0.727461040019989
    },
    {
      "bits_per_byte": 1.7747906159999822,
      "learning_rate": 0.00020618620599029906,
      "loss_nats": 1.2301911115646362,
      "step": 3450,
      "unclipped_grad_norm": 0.7409155368804932
    },
    {
      "bits_per_byte": 1.7588486841782542,
      "learning_rate": 0.00020565441398135649,
      "loss_nats": 1.2191410064697266,
      "step": 3460,
      "unclipped_grad_norm": 0.748551070690155
    },
    {
      "bits_per_byte": 1.7339454243506687,
      "learning_rate": 0.00020512192730064827,
      "loss_nats": 1.2018793821334839,
      "step": 3470,
      "unclipped_grad_norm": 0.7151429653167725
    },
    {
      "bits_per_byte": 1.7067943512763062,
      "learning_rate": 0.00020458875504690273,
      "loss_nats": 1.1830596923828125,
      "step": 3480,
      "unclipped_grad_norm": 0.7189637422561646
    },
    {
      "bits_per_byte": 1.6200879219826143,
      "learning_rate": 0.00020405490633056245,
      "loss_nats": 1.1229593753814697,
      "step": 3490,
      "unclipped_grad_norm": 0.6848708391189575
    },
    {
      "bits_per_byte": 1.6373195517031627,
      "learning_rate": 0.00020352039027362917,
      "loss_nats": 1.1349034309387207,
      "step": 3500,
      "unclipped_grad_norm": 0.6704034209251404
    },
    {
      "bits_per_byte": 1.6849599498707166,
      "learning_rate": 0.0002029852160095075,
      "loss_nats": 1.167925238609314,
      "step": 3510,
      "unclipped_grad_norm": 0.7942876219749451
    },
    {
      "bits_per_byte": 1.6514716600773613,
      "learning_rate": 0.000202449392682849,
      "loss_nats": 1.1447129249572754,
      "step": 3520,
      "unclipped_grad_norm": 0.7149106860160828
    },
    {
      "bits_per_byte": 1.6705435041621417,
      "learning_rate": 0.0002019129294493959,
      "loss_nats": 1.1579325199127197,
      "step": 3530,
      "unclipped_grad_norm": 0.7720932364463806
    },
    {
      "bits_per_byte": 1.6828739722984016,
      "learning_rate": 0.00020137583547582478,
      "loss_nats": 1.1664793491363525,
      "step": 3540,
      "unclipped_grad_norm": 0.6750161647796631
    },
    {
      "bits_per_byte": 1.7101762181228726,
      "learning_rate": 0.00020083811993958973,
      "loss_nats": 1.185403823852539,
      "step": 3550,
      "unclipped_grad_norm": 0.6993052363395691
    },
    {
      "bits_per_byte": 1.6591575647444203,
      "learning_rate": 0.00020029979202876563,
      "loss_nats": 1.1500403881072998,
      "step": 3560,
      "unclipped_grad_norm": 0.6886851787567139
    },
    {
      "bits_per_byte": 1.7182958630354075,
      "learning_rate": 0.00019976086094189118,
      "loss_nats": 1.1910319328308105,
      "step": 3570,
      "unclipped_grad_norm": 0.749891459941864
    },
    {
      "bits_per_byte": 1.7607525321233055,
      "learning_rate": 0.00019922133588781175,
      "loss_nats": 1.2204606533050537,
      "step": 3580,
      "unclipped_grad_norm": 0.8161401152610779
    },
    {
      "bits_per_byte": 1.690903842267186,
      "learning_rate": 0.0001986812260855219,
      "loss_nats": 1.1720452308654785,
      "step": 3590,
      "unclipped_grad_norm": 0.6925192475318909
    },
    {
      "bits_per_byte": 1.6772946831217612,
      "learning_rate": 0.00019814054076400787,
      "loss_nats": 1.1626120805740356,
      "step": 3600,
      "unclipped_grad_norm": 0.7061766982078552
    },
    {
      "bits_per_byte": 1.6673109182565242,
      "learning_rate": 0.00019759928916209004,
      "loss_nats": 1.1556918621063232,
      "step": 3610,
      "unclipped_grad_norm": 0.7162109017372131
    },
    {
      "bits_per_byte": 1.6598764522250269,
      "learning_rate": 0.00019705748052826493,
      "loss_nats": 1.150538682937622,
      "step": 3620,
      "unclipped_grad_norm": 0.6953511238098145
    },
    {
      "bits_per_byte": 1.6688818077895051,
      "learning_rate": 0.00019651512412054723,
      "loss_nats": 1.15678071975708,
      "step": 3630,
      "unclipped_grad_norm": 0.7265485525131226
    },
    {
      "bits_per_byte": 1.6378186453559664,
      "learning_rate": 0.00019597222920631144,
      "loss_nats": 1.135249376296997,
      "step": 3640,
      "unclipped_grad_norm": 0.68751460313797
    },
    {
      "bits_per_byte": 1.6996694540163617,
      "learning_rate": 0.0001954288050621338,
      "loss_nats": 1.1781210899353027,
      "step": 3650,
      "unclipped_grad_norm": 0.7145417928695679
    },
    {
      "bits_per_byte": 1.6073453834148872,
      "learning_rate": 0.00019488486097363363,
      "loss_nats": 1.1141269207000732,
      "step": 3660,
      "unclipped_grad_norm": 0.71860671043396
    },
    {
      "bits_per_byte": 1.62879970316207,
      "learning_rate": 0.00019434040623531452,
      "loss_nats": 1.1289979219436646,
      "step": 3670,
      "unclipped_grad_norm": 0.6769548654556274
    },
    {
      "bits_per_byte": 1.702227523965295,
      "learning_rate": 0.0001937954501504058,
      "loss_nats": 1.179894208908081,
      "step": 3680,
      "unclipped_grad_norm": 0.7466121315956116
    },
    {
      "bits_per_byte": 1.653425382991163,
      "learning_rate": 0.00019325000203070333,
      "loss_nats": 1.1460671424865723,
      "step": 3690,
      "unclipped_grad_norm": 0.6637522578239441
    },
    {
      "bits_per_byte": 1.5989233930021354,
      "learning_rate": 0.0001927040711964106,
      "loss_nats": 1.1082892417907715,
      "step": 3700,
      "unclipped_grad_norm": 0.7335078120231628
    },
    {
      "bits_per_byte": 1.6701862961962997,
      "learning_rate": 0.00019215766697597923,
      "loss_nats": 1.1576849222183228,
      "step": 3710,
      "unclipped_grad_norm": 0.7247957587242126
    },
    {
      "bits_per_byte": 1.6538059805975223,
      "learning_rate": 0.00019161079870594978,
      "loss_nats": 1.1463309526443481,
      "step": 3720,
      "unclipped_grad_norm": 0.7534144520759583
    },
    {
      "bits_per_byte": 1.7119505631318241,
      "learning_rate": 0.0001910634757307922,
      "loss_nats": 1.1866337060928345,
      "step": 3730,
      "unclipped_grad_norm": 0.6951875686645508
    },
    {
      "bits_per_byte": 1.6437530787066386,
      "learning_rate": 0.0001905157074027459,
      "loss_nats": 1.1393628120422363,
      "step": 3740,
      "unclipped_grad_norm": 0.7089563608169556
    },
    {
      "bits_per_byte": 1.6267918057132467,
      "learning_rate": 0.00018996750308166037,
      "loss_nats": 1.1276061534881592,
      "step": 3750,
      "unclipped_grad_norm": 0.6922760009765625
    },
    {
      "bits_per_byte": 1.6201740852906965,
      "learning_rate": 0.00018941887213483482,
      "loss_nats": 1.1230190992355347,
      "step": 3760,
      "unclipped_grad_norm": 0.6971311569213867
    },
    {
      "bits_per_byte": 1.6730486034546093,
      "learning_rate": 0.00018886982393685845,
      "loss_nats": 1.1596689224243164,
      "step": 3770,
      "unclipped_grad_norm": 0.6925700902938843
    },
    {
      "bits_per_byte": 1.6258194158052683,
      "learning_rate": 0.00018832036786945008,
      "loss_nats": 1.126932144165039,
      "step": 3780,
      "unclipped_grad_norm": 0.71840900182724
    },
    {
      "bits_per_byte": 1.6611288298886098,
      "learning_rate": 0.0001877705133212979,
      "loss_nats": 1.1514067649841309,
      "step": 3790,
      "unclipped_grad_norm": 0.6731990575790405
    },
    {
      "bits_per_byte": 1.622545554063444,
      "learning_rate": 0.00018722026968789906,
      "loss_nats": 1.1246628761291504,
      "step": 3800,
      "unclipped_grad_norm": 0.7068047523498535
    },
    {
      "bits_per_byte": 1.613386446009094,
      "learning_rate": 0.00018666964637139907,
      "loss_nats": 1.118314266204834,
      "step": 3810,
      "unclipped_grad_norm": 0.679490327835083
    },
    {
      "bits_per_byte": 1.723293162921528,
      "learning_rate": 0.00018611865278043115,
      "loss_nats": 1.1944957971572876,
      "step": 3820,
      "unclipped_grad_norm": 0.7167435884475708
    },
    {
      "bits_per_byte": 1.6410942269242994,
      "learning_rate": 0.00018556729832995555,
      "loss_nats": 1.1375198364257812,
      "step": 3830,
      "unclipped_grad_norm": 0.749890148639679
    },
    {
      "bits_per_byte": 1.6271811744348,
      "learning_rate": 0.00018501559244109864,
      "loss_nats": 1.1278760433197021,
      "step": 3840,
      "unclipped_grad_norm": 0.6869876980781555
    },
    {
      "bits_per_byte": 1.6520392028252087,
      "learning_rate": 0.00018446354454099178,
      "loss_nats": 1.145106315612793,
      "step": 3850,
      "unclipped_grad_norm": 0.6890377998352051
    },
    {
      "bits_per_byte": 1.6079827511189846,
      "learning_rate": 0.00018391116406261048,
      "loss_nats": 1.1145687103271484,
      "step": 3860,
      "unclipped_grad_norm": 0.717820942401886
    },
    {
      "bits_per_byte": 1.6751354409401786,
      "learning_rate": 0.00018335846044461305,
      "loss_nats": 1.1611154079437256,
      "step": 3870,
      "unclipped_grad_norm": 0.7368612289428711
    },
    {
      "bits_per_byte": 1.6338718155013159,
      "learning_rate": 0.0001828054431311793,
      "loss_nats": 1.1325136423110962,
      "step": 3880,
      "unclipped_grad_norm": 0.7661339640617371
    },
    {
      "bits_per_byte": 1.656585564200767,
      "learning_rate": 0.00018225212157184934,
      "loss_nats": 1.1482576131820679,
      "step": 3890,
      "unclipped_grad_norm": 0.7343658804893494
    },
    {
      "bits_per_byte": 1.6647341021986468,
      "learning_rate": 0.00018169850522136192,
      "loss_nats": 1.1539057493209839,
      "step": 3900,
      "unclipped_grad_norm": 0.7533658146858215
    },
    {
      "bits_per_byte": 1.6762937440937398,
      "learning_rate": 0.000181144603539493,
      "loss_nats": 1.16191828250885,
      "step": 3910,
      "unclipped_grad_norm": 0.7147406935691833
    },
    {
      "bits_per_byte": 1.6322013480134854,
      "learning_rate": 0.00018059042599089412,
      "loss_nats": 1.1313557624816895,
      "step": 3920,
      "unclipped_grad_norm": 0.6887187361717224
    },
    {
      "bits_per_byte": 1.6221845624792828,
      "learning_rate": 0.00018003598204493048,
      "loss_nats": 1.1244126558303833,
      "step": 3930,
      "unclipped_grad_norm": 0.7910217642784119
    },
    {
      "bits_per_byte": 1.63472175376188,
      "learning_rate": 0.00017948128117551936,
      "loss_nats": 1.1331027746200562,
      "step": 3940,
      "unclipped_grad_norm": 0.730076789855957
    },
    {
      "bits_per_byte": 1.6972579132859635,
      "learning_rate": 0.00017892633286096816,
      "loss_nats": 1.1764495372772217,
      "step": 3950,
      "unclipped_grad_norm": 0.7732967138290405
    },
    {
      "bits_per_byte": 1.7135035664691152,
      "learning_rate": 0.00017837114658381247,
      "loss_nats": 1.187710165977478,
      "step": 3960,
      "unclipped_grad_norm": 0.7405394911766052
    },
    {
      "bits_per_byte": 1.685746770498414,
      "learning_rate": 0.00017781573183065386,
      "loss_nats": 1.1684706211090088,
      "step": 3970,
      "unclipped_grad_norm": 0.7160924673080444
    },
    {
      "bits_per_byte": 1.6665311489175123,
      "learning_rate": 0.0001772600980919981,
      "loss_nats": 1.1551513671875,
      "step": 3980,
      "unclipped_grad_norm": 0.7374541759490967
    },
    {
      "bits_per_byte": 1.7040839046987084,
      "learning_rate": 0.0001767042548620927,
      "loss_nats": 1.1811809539794922,
      "step": 3990,
      "unclipped_grad_norm": 0.7841057181358337
    },
    {
      "bits_per_byte": 1.693502844087025,
      "learning_rate": 0.00017614821163876486,
      "loss_nats": 1.17384672164917,
      "step": 4000,
      "unclipped_grad_norm": 0.7609017491340637
    },
    {
      "bits_per_byte": 1.699465998540391,
      "learning_rate": 0.00017559197792325906,
      "loss_nats": 1.1779800653457642,
      "step": 4010,
      "unclipped_grad_norm": 0.7629905343055725
    },
    {
      "bits_per_byte": 1.5838037102341824,
      "learning_rate": 0.0001750355632200748,
      "loss_nats": 1.097809076309204,
      "step": 4020,
      "unclipped_grad_norm": 0.6868616342544556
    },
    {
      "bits_per_byte": 1.617556165379263,
      "learning_rate": 0.00017447897703680418,
      "loss_nats": 1.1212044954299927,
      "step": 4030,
      "unclipped_grad_norm": 0.749467670917511
    },
    {
      "bits_per_byte": 1.6189757101794846,
      "learning_rate": 0.0001739222288839694,
      "loss_nats": 1.1221884489059448,
      "step": 4040,
      "unclipped_grad_norm": 0.7323976755142212
    },
    {
      "bits_per_byte": 1.6111234982890412,
      "learning_rate": 0.00017336532827486026,
      "loss_nats": 1.1167457103729248,
      "step": 4050,
      "unclipped_grad_norm": 0.7304336428642273
    },
    {
      "bits_per_byte": 1.6428754512392856,
      "learning_rate": 0.00017280828472537163,
      "loss_nats": 1.1387544870376587,
      "step": 4060,
      "unclipped_grad_norm": 0.745829164981842
    },
    {
      "bits_per_byte": 1.6097484969953932,
      "learning_rate": 0.0001722511077538408,
      "loss_nats": 1.1157926321029663,
      "step": 4070,
      "unclipped_grad_norm": 0.7523763179779053
    },
    {
      "bits_per_byte": 1.6445053308215125,
      "learning_rate": 0.00017169380688088497,
      "loss_nats": 1.1398842334747314,
      "step": 4080,
      "unclipped_grad_norm": 0.7511109113693237
    },
    {
      "bits_per_byte": 1.6034616711931031,
      "learning_rate": 0.0001711363916292384,
      "loss_nats": 1.1114349365234375,
      "step": 4090,
      "unclipped_grad_norm": 0.7438745498657227
    },
    {
      "bits_per_byte": 1.561935084281071,
      "learning_rate": 0.00017057887152358974,
      "loss_nats": 1.082650899887085,
      "step": 4100,
      "unclipped_grad_norm": 0.6872318387031555
    },
    {
      "bits_per_byte": 1.6907129415247284,
      "learning_rate": 0.0001700212560904194,
      "loss_nats": 1.1719129085540771,
      "step": 4110,
      "unclipped_grad_norm": 0.7602059841156006
    },
    {
      "bits_per_byte": 1.6125045189754696,
      "learning_rate": 0.00016946355485783656,
      "loss_nats": 1.1177029609680176,
      "step": 4120,
      "unclipped_grad_norm": 0.7136654257774353
    },
    {
      "bits_per_byte": 1.5891117827704122,
      "learning_rate": 0.0001689057773554166,
      "loss_nats": 1.1014883518218994,
      "step": 4130,
      "unclipped_grad_norm": 0.7062129378318787
    },
    {
      "bits_per_byte": 1.6146775197691212,
      "learning_rate": 0.00016834793311403806,
      "loss_nats": 1.1192091703414917,
      "step": 4140,
      "unclipped_grad_norm": 0.7150430679321289
    },
    {
      "bits_per_byte": 1.6618133208390438,
      "learning_rate": 0.00016779003166571981,
      "loss_nats": 1.151881217956543,
      "step": 4150,
      "unclipped_grad_norm": 0.7448095679283142
    },
    {
      "bits_per_byte": 1.6228898633304711,
      "learning_rate": 0.00016723208254345838,
      "loss_nats": 1.124901533126831,
      "step": 4160,
      "unclipped_grad_norm": 0.7362097501754761
    },
    {
      "bits_per_byte": 1.5905947891688021,
      "learning_rate": 0.00016667409528106481,
      "loss_nats": 1.1025162935256958,
      "step": 4170,
      "unclipped_grad_norm": 0.7651577591896057
    },
    {
      "bits_per_byte": 1.5608787668394717,
      "learning_rate": 0.00016611607941300184,
      "loss_nats": 1.081918716430664,
      "step": 4180,
      "unclipped_grad_norm": 0.7135135531425476
    },
    {
      "bits_per_byte": 1.713072749928704,
      "learning_rate": 0.0001655580444742211,
      "loss_nats": 1.1874115467071533,
      "step": 4190,
      "unclipped_grad_norm": 0.7463747262954712
    },
    {
      "bits_per_byte": 1.5972510337051453,
      "learning_rate": 0.00016499999999999997,
      "loss_nats": 1.1071300506591797,
      "step": 4200,
      "unclipped_grad_norm": 0.7313898801803589
    },
    {
      "bits_per_byte": 1.6169509584308768,
      "learning_rate": 0.0001644419555257789,
      "loss_nats": 1.1207849979400635,
      "step": 4210,
      "unclipped_grad_norm": 0.7539675831794739
    },
    {
      "bits_per_byte": 1.6536405332873922,
      "learning_rate": 0.0001638839205869981,
      "loss_nats": 1.1462162733078003,
      "step": 4220,
      "unclipped_grad_norm": 0.7351350784301758
    },
    {
      "bits_per_byte": 1.6357479742395782,
      "learning_rate": 0.0001633259047189352,
      "loss_nats": 1.1338140964508057,
      "step": 4230,
      "unclipped_grad_norm": 0.7141795754432678
    },
    {
      "bits_per_byte": 1.6650933739562992,
      "learning_rate": 0.0001627679174565416,
      "loss_nats": 1.1541547775268555,
      "step": 4240,
      "unclipped_grad_norm": 0.7625938653945923
    },
    {
      "bits_per_byte": 1.6824343846427963,
      "learning_rate": 0.00016220996833428016,
      "loss_nats": 1.1661746501922607,
      "step": 4250,
      "unclipped_grad_norm": 0.702212393283844
    },
    {
      "bits_per_byte": 1.6672134040934852,
      "learning_rate": 0.00016165206688596191,
      "loss_nats": 1.155624270439148,
      "step": 4260,
      "unclipped_grad_norm": 0.775193989276886
    },
    {
      "bits_per_byte": 1.643573356836487,
      "learning_rate": 0.00016109422264458335,
      "loss_nats": 1.1392382383346558,
      "step": 4270,
      "unclipped_grad_norm": 0.7224892973899841
    },
    {
      "bits_per_byte": 1.6848497089915135,
      "learning_rate": 0.00016053644514216336,
      "loss_nats": 1.167848825454712,
      "step": 4280,
      "unclipped_grad_norm": 0.792568564414978
    },
    {
      "bits_per_byte": 1.6287462165576516,
      "learning_rate": 0.00015997874390958058,
      "loss_nats": 1.1289608478546143,
      "step": 4290,
      "unclipped_grad_norm": 0.7352984547615051
    },
    {
      "bits_per_byte": 1.5679441581022175,
      "learning_rate": 0.00015942112847641018,
      "loss_nats": 1.0868160724639893,
      "step": 4300,
      "unclipped_grad_norm": 0.7459802031517029
    },
    {
      "bits_per_byte": 1.7134996108681453,
      "learning_rate": 0.00015886360837076156,
      "loss_nats": 1.1877074241638184,
      "step": 4310,
      "unclipped_grad_norm": 0.8436940312385559
    },
    {
      "bits_per_byte": 1.585466610485375,
      "learning_rate": 0.00015830619311911495,
      "loss_nats": 1.0989617109298706,
      "step": 4320,
      "unclipped_grad_norm": 0.7326236367225647
    },
    {
      "bits_per_byte": 1.5870087789156617,
      "learning_rate": 0.00015774889224615915,
      "loss_nats": 1.1000306606292725,
      "step": 4330,
      "unclipped_grad_norm": 0.7154185175895691
    },
    {
      "bits_per_byte": 1.5845594020020737,
      "learning_rate": 0.00015719171527462834,
      "loss_nats": 1.0983328819274902,
      "step": 4340,
      "unclipped_grad_norm": 0.7299889326095581
    },
    {
      "bits_per_byte": 1.5087386145947432,
      "learning_rate": 0.0001566346717251397,
      "loss_nats": 1.0457779169082642,
      "step": 4350,
      "unclipped_grad_norm": 0.7116599082946777
    },
    {
      "bits_per_byte": 1.6642632137005844,
      "learning_rate": 0.00015607777111603056,
      "loss_nats": 1.1535793542861938,
      "step": 4360,
      "unclipped_grad_norm": 0.7841640114784241
    },
    {
      "bits_per_byte": 1.6118272512963718,
      "learning_rate": 0.00015552102296319576,
      "loss_nats": 1.1172335147857666,
      "step": 4370,
      "unclipped_grad_norm": 0.7682546973228455
    },
    {
      "bits_per_byte": 1.5975416843851036,
      "learning_rate": 0.00015496443677992516,
      "loss_nats": 1.1073315143585205,
      "step": 4380,
      "unclipped_grad_norm": 0.7542603611946106
    },
    {
      "bits_per_byte": 1.6190350441940322,
      "learning_rate": 0.0001544080220767409,
      "loss_nats": 1.1222295761108398,
      "step": 4390,
      "unclipped_grad_norm": 0.7354364991188049
    },
    {
      "bits_per_byte": 1.5791293937663822,
      "learning_rate": 0.00015385178836123511,
      "loss_nats": 1.0945690870285034,
      "step": 4400,
      "unclipped_grad_norm": 0.7447035908699036
    },
    {
      "bits_per_byte": 1.5777399459300616,
      "learning_rate": 0.00015329574513790726,
      "loss_nats": 1.0936059951782227,
      "step": 4410,
      "unclipped_grad_norm": 0.7433028221130371
    },
    {
      "bits_per_byte": 1.5774630538621723,
      "learning_rate": 0.00015273990190800186,
      "loss_nats": 1.093414068222046,
      "step": 4420,
      "unclipped_grad_norm": 0.7514091730117798
    },
    {
      "bits_per_byte": 1.5840658117940973,
      "learning_rate": 0.0001521842681693461,
      "loss_nats": 1.0979907512664795,
      "step": 4430,
      "unclipped_grad_norm": 0.748140811920166
    },
    {
      "bits_per_byte": 1.5992336497042918,
      "learning_rate": 0.00015162885341618752,
      "loss_nats": 1.108504295349121,
      "step": 4440,
      "unclipped_grad_norm": 0.7693572044372559
    },
    {
      "bits_per_byte": 1.5825853851354703,
      "learning_rate": 0.00015107366713903178,
      "loss_nats": 1.0969645977020264,
      "step": 4450,
      "unclipped_grad_norm": 0.7571823596954346
    },
    {
      "bits_per_byte": 1.5385392523882981,
      "learning_rate": 0.0001505187188244806,
      "loss_nats": 1.0664341449737549,
      "step": 4460,
      "unclipped_grad_norm": 0.7259396314620972
    },
    {
      "bits_per_byte": 1.6428487939284018,
      "learning_rate": 0.00014996401795506952,
      "loss_nats": 1.1387360095977783,
      "step": 4470,
      "unclipped_grad_norm": 0.7504101991653442
    },
    {
      "bits_per_byte": 1.5687149843433847,
      "learning_rate": 0.00014940957400910585,
      "loss_nats": 1.0873503684997559,
      "step": 4480,
      "unclipped_grad_norm": 0.7544133067131042
    },
    {
      "bits_per_byte": 1.5307804270772734,
      "learning_rate": 0.00014885539646050699,
      "loss_nats": 1.061056137084961,
      "step": 4490,
      "unclipped_grad_norm": 0.7395938634872437
    },
    {
      "bits_per_byte": 1.6123227333135077,
      "learning_rate": 0.00014830149477863805,
      "loss_nats": 1.1175769567489624,
      "step": 4500,
      "unclipped_grad_norm": 0.7624281644821167
    },
    {
      "bits_per_byte": 1.5895855949735391,
      "learning_rate": 0.00014774787842815066,
      "loss_nats": 1.1018167734146118,
      "step": 4510,
      "unclipped_grad_norm": 0.7442159652709961
    },
    {
      "bits_per_byte": 1.5720557473363947,
      "learning_rate": 0.00014719455686882066,
      "loss_nats": 1.0896660089492798,
      "step": 4520,
      "unclipped_grad_norm": 0.7299280166625977
    },
    {
      "bits_per_byte": 1.5930625682260326,
      "learning_rate": 0.00014664153955538695,
      "loss_nats": 1.10422682762146,
      "step": 4530,
      "unclipped_grad_norm": 0.7508688569068909
    },
    {
      "bits_per_byte": 1.551802554422838,
      "learning_rate": 0.00014608883593738946,
      "loss_nats": 1.0756275653839111,
      "step": 4540,
      "unclipped_grad_norm": 0.7326505780220032
    },
    {
      "bits_per_byte": 1.6245931795046165,
      "learning_rate": 0.00014553645545900817,
      "loss_nats": 1.126082181930542,
      "step": 4550,
      "unclipped_grad_norm": 0.7591264247894287
    },
    {
      "bits_per_byte": 1.6222865481912445,
      "learning_rate": 0.0001449844075589013,
      "loss_nats": 1.124483346939087,
      "step": 4560,
      "unclipped_grad_norm": 0.7490679621696472
    },
    {
      "bits_per_byte": 1.5636793323261218,
      "learning_rate": 0.0001444327016700444,
      "loss_nats": 1.083859920501709,
      "step": 4570,
      "unclipped_grad_norm": 0.7205959558486938
    },
    {
      "bits_per_byte": 1.5242402708302625,
      "learning_rate": 0.0001438813472195688,
      "loss_nats": 1.0565228462219238,
      "step": 4580,
      "unclipped_grad_norm": 0.7278034687042236
    },
    {
      "bits_per_byte": 1.649223158899981,
      "learning_rate": 0.00014333035362860093,
      "loss_nats": 1.1431543827056885,
      "step": 4590,
      "unclipped_grad_norm": 0.80039381980896
    },
    {
      "bits_per_byte": 1.621592598195013,
      "learning_rate": 0.00014277973031210088,
      "loss_nats": 1.1240023374557495,
      "step": 4600,
      "unclipped_grad_norm": 0.7594562768936157
    },
    {
      "bits_per_byte": 1.591440943811047,
      "learning_rate": 0.00014222948667870207,
      "loss_nats": 1.1031028032302856,
      "step": 4610,
      "unclipped_grad_norm": 0.7573962211608887
    },
    {
      "bits_per_byte": 1.5204752266375738,
      "learning_rate": 0.00014167963213054991,
      "loss_nats": 1.0539131164550781,
      "step": 4620,
      "unclipped_grad_norm": 0.7093440890312195
    },
    {
      "bits_per_byte": 1.6007937043302685,
      "learning_rate": 0.00014113017606314152,
      "loss_nats": 1.1095856428146362,
      "step": 4630,
      "unclipped_grad_norm": 0.7716385126113892
    },
    {
      "bits_per_byte": 1.566827990698118,
      "learning_rate": 0.00014058112786516518,
      "loss_nats": 1.0860424041748047,
      "step": 4640,
      "unclipped_grad_norm": 0.7477832436561584
    },
    {
      "bits_per_byte": 1.5540792607549598,
      "learning_rate": 0.0001400324969183396,
      "loss_nats": 1.0772056579589844,
      "step": 4650,
      "unclipped_grad_norm": 0.7563350200653076
    },
    {
      "bits_per_byte": 1.5711925664117143,
      "learning_rate": 0.00013948429259725406,
      "loss_nats": 1.0890676975250244,
      "step": 4660,
      "unclipped_grad_norm": 0.7477205991744995
    },
    {
      "bits_per_byte": 1.6065126434194095,
      "learning_rate": 0.00013893652426920776,
      "loss_nats": 1.1135497093200684,
      "step": 4670,
      "unclipped_grad_norm": 0.7434800267219543
    },
    {
      "bits_per_byte": 1.5613855997115644,
      "learning_rate": 0.00013838920129405016,
      "loss_nats": 1.0822700262069702,
      "step": 4680,
      "unclipped_grad_norm": 0.7484313249588013
    },
    {
      "bits_per_byte": 1.533432227570927,
      "learning_rate": 0.0001378423330240207,
      "loss_nats": 1.0628942251205444,
      "step": 4690,
      "unclipped_grad_norm": 0.7767173647880554
    },
    {
      "bits_per_byte": 1.6430371149310967,
      "learning_rate": 0.00013729592880358936,
      "loss_nats": 1.1388665437698364,
      "step": 4700,
      "unclipped_grad_norm": 0.7674224972724915
    },
    {
      "bits_per_byte": 1.552380416129737,
      "learning_rate": 0.00013674999796929659,
      "loss_nats": 1.0760281085968018,
      "step": 4710,
      "unclipped_grad_norm": 0.7836162447929382
    },
    {
      "bits_per_byte": 1.5203768525612804,
      "learning_rate": 0.00013620454984959417,
      "loss_nats": 1.053844928741455,
      "step": 4720,
      "unclipped_grad_norm": 0.7613595128059387
    },
    {
      "bits_per_byte": 1.6524877335786587,
      "learning_rate": 0.00013565959376468542,
      "loss_nats": 1.1454172134399414,
      "step": 4730,
      "unclipped_grad_norm": 0.7924392223358154
    },
    {
      "bits_per_byte": 1.587415861850254,
      "learning_rate": 0.00013511513902636634,
      "loss_nats": 1.1003128290176392,
      "step": 4740,
      "unclipped_grad_norm": 0.7647171020507812
    },
    {
      "bits_per_byte": 1.6396726183322676,
      "learning_rate": 0.0001345711949378662,
      "loss_nats": 1.1365344524383545,
      "step": 4750,
      "unclipped_grad_norm": 0.8181672096252441
    },
    {
      "bits_per_byte": 1.5091890371573529,
      "learning_rate": 0.00013402777079368853,
      "loss_nats": 1.0460901260375977,
      "step": 4760,
      "unclipped_grad_norm": 0.7407304048538208
    },
    {
      "bits_per_byte": 1.6098291568586478,
      "learning_rate": 0.0001334848758794528,
      "loss_nats": 1.1158485412597656,
      "step": 4770,
      "unclipped_grad_norm": 0.7601819038391113
    },
    {
      "bits_per_byte": 1.5582154435082103,
      "learning_rate": 0.00013294251947173504,
      "loss_nats": 1.0800726413726807,
      "step": 4780,
      "unclipped_grad_norm": 0.7307054996490479
    },
    {
      "bits_per_byte": 1.5944504682184955,
      "learning_rate": 0.00013240071083790996,
      "loss_nats": 1.1051888465881348,
      "step": 4790,
      "unclipped_grad_norm": 0.7617009282112122
    },
    {
      "bits_per_byte": 1.5823225956449518,
      "learning_rate": 0.0001318594592359921,
      "loss_nats": 1.0967824459075928,
      "step": 4800,
      "unclipped_grad_norm": 0.7428550124168396
    },
    {
      "bits_per_byte": 1.542078311377753,
      "learning_rate": 0.0001313187739144781,
      "loss_nats": 1.0688872337341309,
      "step": 4810,
      "unclipped_grad_norm": 0.773262083530426
    },
    {
      "bits_per_byte": 1.6108488419956133,
      "learning_rate": 0.0001307786641121882,
      "loss_nats": 1.1165553331375122,
      "step": 4820,
      "unclipped_grad_norm": 0.8227523565292358
    },
    {
      "bits_per_byte": 1.5982464692883394,
      "learning_rate": 0.00013023913905810876,
      "loss_nats": 1.1078200340270996,
      "step": 4830,
      "unclipped_grad_norm": 0.783271312713623
    },
    {
      "bits_per_byte": 1.6134836162068316,
      "learning_rate": 0.00012970020797123432,
      "loss_nats": 1.1183816194534302,
      "step": 4840,
      "unclipped_grad_norm": 0.7595719695091248
    },
    {
      "bits_per_byte": 1.5430993723759254,
      "learning_rate": 0.00012916188006041027,
      "loss_nats": 1.0695949792861938,
      "step": 4850,
      "unclipped_grad_norm": 0.8344528079032898
    },
    {
      "bits_per_byte": 1.551674427347945,
      "learning_rate": 0.00012862416452417516,
      "loss_nats": 1.0755387544631958,
      "step": 4860,
      "unclipped_grad_norm": 0.7924710512161255
    },
    {
      "bits_per_byte": 1.5551434893984988,
      "learning_rate": 0.00012808707055060405,
      "loss_nats": 1.0779433250427246,
      "step": 4870,
      "unclipped_grad_norm": 0.7731344103813171
    },
    {
      "bits_per_byte": 1.7162552888829201,
      "learning_rate": 0.000127550607317151,
      "loss_nats": 1.1896175146102905,
      "step": 4880,
      "unclipped_grad_norm": 0.8454892039299011
    },
    {
      "bits_per_byte": 1.5015098398136777,
      "learning_rate": 0.00012701478399049248,
      "loss_nats": 1.0407673120498657,
      "step": 4890,
      "unclipped_grad_norm": 0.7279435396194458
    },
    {
      "bits_per_byte": 1.6017640304464364,
      "learning_rate": 0.0001264796097263708,
      "loss_nats": 1.1102582216262817,
      "step": 4900,
      "unclipped_grad_norm": 0.7495299577713013
    },
    {
      "bits_per_byte": 1.5795956387328711,
      "learning_rate": 0.0001259450936694375,
      "loss_nats": 1.0948922634124756,
      "step": 4910,
      "unclipped_grad_norm": 0.8354763388633728
    },
    {
      "bits_per_byte": 1.5130457481029513,
      "learning_rate": 0.00012541124495309727,
      "loss_nats": 1.048763394355774,
      "step": 4920,
      "unclipped_grad_norm": 0.7128972411155701
    },
    {
      "bits_per_byte": 1.5263470583033318,
      "learning_rate": 0.00012487807269935167,
      "loss_nats": 1.057983160018921,
      "step": 4930,
      "unclipped_grad_norm": 0.7301722168922424
    },
    {
      "bits_per_byte": 1.56106777577277,
      "learning_rate": 0.00012434558601864349,
      "loss_nats": 1.0820497274398804,
      "step": 4940,
      "unclipped_grad_norm": 0.7844029664993286
    },
    {
      "bits_per_byte": 1.5809642766684375,
      "learning_rate": 0.0001238137940097009,
      "loss_nats": 1.0958409309387207,
      "step": 4950,
      "unclipped_grad_norm": 0.7815173864364624
    },
    {
      "bits_per_byte": 1.6004612618661507,
      "learning_rate": 0.0001232827057593821,
      "loss_nats": 1.1093552112579346,
      "step": 4960,
      "unclipped_grad_norm": 0.8055645227432251
    },
    {
      "bits_per_byte": 1.5618184800437862,
      "learning_rate": 0.00012275233034251992,
      "loss_nats": 1.0825700759887695,
      "step": 4970,
      "unclipped_grad_norm": 0.7773133516311646
    },
    {
      "bits_per_byte": 1.606830467358204,
      "learning_rate": 0.0001222226768217669,
      "loss_nats": 1.1137700080871582,
      "step": 4980,
      "unclipped_grad_norm": 0.7920810580253601
    },
    {
      "bits_per_byte": 1.567882244347907,
      "learning_rate": 0.00012169375424744034,
      "loss_nats": 1.086773157119751,
      "step": 4990,
      "unclipped_grad_norm": 0.7929123640060425
    },
    {
      "bits_per_byte": 1.5708692390280923,
      "learning_rate": 0.00012116557165736772,
      "loss_nats": 1.088843584060669,
      "step": 5000,
      "unclipped_grad_norm": 0.762309730052948
    },
    {
      "bits_per_byte": 1.6157285057485438,
      "learning_rate": 0.00012063813807673214,
      "loss_nats": 1.1199376583099365,
      "step": 5010,
      "unclipped_grad_norm": 0.7734562158584595
    },
    {
      "bits_per_byte": 1.5685849654593325,
      "learning_rate": 0.00012011146251791832,
      "loss_nats": 1.0872602462768555,
      "step": 5020,
      "unclipped_grad_norm": 0.8221466541290283
    },
    {
      "bits_per_byte": 1.5125002191344146,
      "learning_rate": 0.0001195855539803584,
      "loss_nats": 1.0483852624893188,
      "step": 5030,
      "unclipped_grad_norm": 0.7515868544578552
    },
    {
      "bits_per_byte": 1.5645020973278494,
      "learning_rate": 0.00011906042145037817,
      "loss_nats": 1.08443021774292,
      "step": 5040,
      "unclipped_grad_norm": 0.7403401732444763
    },
    {
      "bits_per_byte": 1.5465288784167808,
      "learning_rate": 0.0001185360739010438,
      "loss_nats": 1.071972131729126,
      "step": 5050,
      "unclipped_grad_norm": 0.8082988262176514
    },
    {
      "bits_per_byte": 1.6139199361920706,
      "learning_rate": 0.00011801252029200808,
      "loss_nats": 1.1186840534210205,
      "step": 5060,
      "unclipped_grad_norm": 0.8066977262496948
    },
    {
      "bits_per_byte": 1.5533101543403012,
      "learning_rate": 0.00011748976956935777,
      "loss_nats": 1.0766725540161133,
      "step": 5070,
      "unclipped_grad_norm": 0.7855141758918762
    },
    {
      "bits_per_byte": 1.6163216739113697,
      "learning_rate": 0.00011696783066546024,
      "loss_nats": 1.1203488111495972,
      "step": 5080,
      "unclipped_grad_norm": 0.776370644569397
    },
    {
      "bits_per_byte": 1.5614495772576855,
      "learning_rate": 0.00011644671249881147,
      "loss_nats": 1.082314372062683,
      "step": 5090,
      "unclipped_grad_norm": 0.776827871799469
    },
    {
      "bits_per_byte": 1.5419546558517827,
      "learning_rate": 0.00011592642397388293,
      "loss_nats": 1.0688015222549438,
      "step": 5100,
      "unclipped_grad_norm": 0.7785685062408447
    },
    {
      "bits_per_byte": 1.5800214677764075,
      "learning_rate": 0.00011540697398097015,
      "loss_nats": 1.0951874256134033,
      "step": 5110,
      "unclipped_grad_norm": 0.8104537725448608
    },
    {
      "bits_per_byte": 1.487382496941203,
      "learning_rate": 0.00011488837139604016,
      "loss_nats": 1.0309749841690063,
      "step": 5120,
      "unclipped_grad_norm": 0.7389267086982727
    },
    {
      "bits_per_byte": 1.6243671943013827,
      "learning_rate": 0.00011437062508058039,
      "loss_nats": 1.1259255409240723,
      "step": 5130,
      "unclipped_grad_norm": 0.8053697943687439
    },
    {
      "bits_per_byte": 1.5876562935961602,
      "learning_rate": 0.00011385374388144673,
      "loss_nats": 1.1004794836044312,
      "step": 5140,
      "unclipped_grad_norm": 0.8099423050880432
    },
    {
      "bits_per_byte": 1.5188835272038386,
      "learning_rate": 0.00011333773663071286,
      "loss_nats": 1.0528098344802856,
      "step": 5150,
      "unclipped_grad_norm": 0.7463088035583496
    },
    {
      "bits_per_byte": 1.5574305146896725,
      "learning_rate": 0.00011282261214551897,
      "loss_nats": 1.079528570175171,
      "step": 5160,
      "unclipped_grad_norm": 0.7795574069023132
    },
    {
      "bits_per_byte": 1.604379370618107,
      "learning_rate": 0.00011230837922792112,
      "loss_nats": 1.1120710372924805,
      "step": 5170,
      "unclipped_grad_norm": 0.8112488985061646
    },
    {
      "bits_per_byte": 1.570943535533265,
      "learning_rate": 0.0001117950466647412,
      "loss_nats": 1.0888950824737549,
      "step": 5180,
      "unclipped_grad_norm": 0.7960402965545654
    },
    {
      "bits_per_byte": 1.4686855750353036,
      "learning_rate": 0.00011128262322741627,
      "loss_nats": 1.0180152654647827,
      "step": 5190,
      "unclipped_grad_norm": 0.7735135555267334
    },
    {
      "bits_per_byte": 1.5401951013508053,
      "learning_rate": 0.00011077111767184913,
      "loss_nats": 1.0675818920135498,
      "step": 5200,
      "unclipped_grad_norm": 0.7935121655464172
    },
    {
      "bits_per_byte": 1.522642552003747,
      "learning_rate": 0.00011026053873825837,
      "loss_nats": 1.055415391921997,
      "step": 5210,
      "unclipped_grad_norm": 0.7616920471191406
    },
    {
      "bits_per_byte": 1.5296819738862126,
      "learning_rate": 0.0001097508951510293,
      "loss_nats": 1.0602947473526,
      "step": 5220,
      "unclipped_grad_norm": 0.799717128276825
    },
    {
      "bits_per_byte": 1.60920967535024,
      "learning_rate": 0.0001092421956185646,
      "loss_nats": 1.1154191493988037,
      "step": 5230,
      "unclipped_grad_norm": 0.8263183832168579
    },
    {
      "bits_per_byte": 1.5759482306733728,
      "learning_rate": 0.00010873444883313581,
      "loss_nats": 1.0923640727996826,
      "step": 5240,
      "unclipped_grad_norm": 0.9089767336845398
    },
    {
      "bits_per_byte": 1.5751560785834986,
      "learning_rate": 0.00010822766347073447,
      "loss_nats": 1.0918149948120117,
      "step": 5250,
      "unclipped_grad_norm": 0.8223497271537781
    },
    {
      "bits_per_byte": 1.518346597367845,
      "learning_rate": 0.00010772184819092424,
      "loss_nats": 1.052437663078308,
      "step": 5260,
      "unclipped_grad_norm": 0.7989961504936218
    },
    {
      "bits_per_byte": 1.5652323356634128,
      "learning_rate": 0.0001072170116366926,
      "loss_nats": 1.0849363803863525,
      "step": 5270,
      "unclipped_grad_norm": 0.7858845591545105
    },
    {
      "bits_per_byte": 1.4824085866955998,
      "learning_rate": 0.00010671316243430345,
      "loss_nats": 1.0275273323059082,
      "step": 5280,
      "unclipped_grad_norm": 0.7963522672653198
    },
    {
      "bits_per_byte": 1.5109090356486319,
      "learning_rate": 0.00010621030919314944,
      "loss_nats": 1.047282338142395,
      "step": 5290,
      "unclipped_grad_norm": 0.7786324620246887
    },
    {
      "bits_per_byte": 1.4973765807654917,
      "learning_rate": 0.000105708460505605,
      "loss_nats": 1.0379023551940918,
      "step": 5300,
      "unclipped_grad_norm": 0.8200122714042664
    },
    {
      "bits_per_byte": 1.4979238295605373,
      "learning_rate": 0.00010520762494687964,
      "loss_nats": 1.0382816791534424,
      "step": 5310,
      "unclipped_grad_norm": 0.798004686832428
    },
    {
      "bits_per_byte": 1.4963071926424267,
      "learning_rate": 0.00010470781107487111,
      "loss_nats": 1.037161111831665,
      "step": 5320,
      "unclipped_grad_norm": 0.7881578207015991
    },
    {
      "bits_per_byte": 1.5549873291515153,
      "learning_rate": 0.00010420902743001955,
      "loss_nats": 1.0778350830078125,
      "step": 5330,
      "unclipped_grad_norm": 0.8092938661575317
    },
    {
      "bits_per_byte": 1.6144941862633015,
      "learning_rate": 0.00010371128253516117,
      "loss_nats": 1.1190820932388306,
      "step": 5340,
      "unclipped_grad_norm": 0.8349001407623291
    },
    {
      "bits_per_byte": 1.4860422361430292,
      "learning_rate": 0.000103214584895383,
      "loss_nats": 1.030045986175537,
      "step": 5350,
      "unclipped_grad_norm": 0.7843849062919617
    },
    {
      "bits_per_byte": 1.5082066722556244,
      "learning_rate": 0.00010271894299787715,
      "loss_nats": 1.0454092025756836,
      "step": 5360,
      "unclipped_grad_norm": 0.7918040752410889
    },
    {
      "bits_per_byte": 1.577472340925319,
      "learning_rate": 0.00010222436531179622,
      "loss_nats": 1.0934205055236816,
      "step": 5370,
      "unclipped_grad_norm": 0.8359649181365967
    },
    {
      "bits_per_byte": 1.5013394050066726,
      "learning_rate": 0.0001017308602881082,
      "loss_nats": 1.040649175643921,
      "step": 5380,
      "unclipped_grad_norm": 0.7734270095825195
    },
    {
      "bits_per_byte": 1.5016145772480531,
      "learning_rate": 0.00010123843635945242,
      "loss_nats": 1.0408399105072021,
      "step": 5390,
      "unclipped_grad_norm": 0.7885358333587646
    },
    {
      "bits_per_byte": 1.5491196250693782,
      "learning_rate": 0.00010074710193999505,
      "loss_nats": 1.073767900466919,
      "step": 5400,
      "unclipped_grad_norm": 0.8202832341194153
    },
    {
      "bits_per_byte": 1.4666546319112648,
      "learning_rate": 0.0001002568654252858,
      "loss_nats": 1.0166075229644775,
      "step": 5410,
      "unclipped_grad_norm": 0.8083077073097229
    },
    {
      "bits_per_byte": 1.6009971598062391,
      "learning_rate": 9.976773519211404e-05,
      "loss_nats": 1.1097266674041748,
      "step": 5420,
      "unclipped_grad_norm": 0.847724974155426
    },
    {
      "bits_per_byte": 1.5289176829857782,
      "learning_rate": 9.92797195983659e-05,
      "loss_nats": 1.0597649812698364,
      "step": 5430,
      "unclipped_grad_norm": 0.8225540518760681
    },
    {
      "bits_per_byte": 1.530357349756151,
      "learning_rate": 9.879282698288142e-05,
      "loss_nats": 1.060762882232666,
      "step": 5440,
      "unclipped_grad_norm": 0.8045809864997864
    },
    {
      "bits_per_byte": 1.5424515137301253,
      "learning_rate": 9.830706566531197e-05,
      "loss_nats": 1.069145917892456,
      "step": 5450,
      "unclipped_grad_norm": 0.8051854968070984
    },
    {
      "bits_per_byte": 1.5648894022575925,
      "learning_rate": 9.782244394597826e-05,
      "loss_nats": 1.0846986770629883,
      "step": 5460,
      "unclipped_grad_norm": 0.7835660576820374
    },
    {
      "bits_per_byte": 1.5450825043230245,
      "learning_rate": 9.733897010572829e-05,
      "loss_nats": 1.070969581604004,
      "step": 5470,
      "unclipped_grad_norm": 0.8599129915237427
    },
    {
      "bits_per_byte": 1.4780804713039863,
      "learning_rate": 9.685665240579612e-05,
      "loss_nats": 1.0245273113250732,
      "step": 5480,
      "unclipped_grad_norm": 0.8000594973564148
    },
    {
      "bits_per_byte": 1.5424891779306642,
      "learning_rate": 9.637549908766041e-05,
      "loss_nats": 1.0691720247268677,
      "step": 5490,
      "unclipped_grad_norm": 0.8055532574653625
    },
    {
      "bits_per_byte": 1.5569413960306187,
      "learning_rate": 9.589551837290389e-05,
      "loss_nats": 1.0791895389556885,
      "step": 5500,
      "unclipped_grad_norm": 0.8300883769989014
    },
    {
      "bits_per_byte": 1.5474785666148452,
      "learning_rate": 9.541671846307257e-05,
      "loss_nats": 1.0726304054260254,
      "step": 5510,
      "unclipped_grad_norm": 0.826352059841156
    },
    {
      "bits_per_byte": 1.5350409532870977,
      "learning_rate": 9.493910753953597e-05,
      "loss_nats": 1.0640093088150024,
      "step": 5520,
      "unclipped_grad_norm": 0.853265643119812
    },
    {
      "bits_per_byte": 1.5635150888945477,
      "learning_rate": 9.446269376334689e-05,
      "loss_nats": 1.083746075630188,
      "step": 5530,
      "unclipped_grad_norm": 0.8060055375099182
    },
    {
      "bits_per_byte": 1.5843347926600468,
      "learning_rate": 9.39874852751024e-05,
      "loss_nats": 1.098177194595337,
      "step": 5540,
      "unclipped_grad_norm": 0.8706573843955994
    },
    {
      "bits_per_byte": 1.5165710484803372,
      "learning_rate": 9.351349019480433e-05,
      "loss_nats": 1.0512069463729858,
      "step": 5550,
      "unclipped_grad_norm": 0.7865509986877441
    },
    {
      "bits_per_byte": 1.5106121935932426,
      "learning_rate": 9.304071662172086e-05,
      "loss_nats": 1.0470765829086304,
      "step": 5560,
      "unclipped_grad_norm": 0.8064424991607666
    },
    {
      "bits_per_byte": 1.5654490338035,
      "learning_rate": 9.256917263424787e-05,
      "loss_nats": 1.0850865840911865,
      "step": 5570,
      "unclipped_grad_norm": 0.8376733064651489
    },
    {
      "bits_per_byte": 1.4567845475782482,
      "learning_rate": 9.209886628977114e-05,
      "loss_nats": 1.0097661018371582,
      "step": 5580,
      "unclipped_grad_norm": 0.7972759008407593
    },
    {
      "bits_per_byte": 1.5257076268074239,
      "learning_rate": 9.162980562452843e-05,
      "loss_nats": 1.057539939880371,
      "step": 5590,
      "unclipped_grad_norm": 0.8159511089324951
    },
    {
      "bits_per_byte": 1.5137768463517691,
      "learning_rate": 9.116199865347238e-05,
      "loss_nats": 1.0492701530456543,
      "step": 5600,
      "unclipped_grad_norm": 0.7912799715995789
    },
    {
      "bits_per_byte": 1.5312812405565859,
      "learning_rate": 9.069545337013333e-05,
      "loss_nats": 1.0614032745361328,
      "step": 5610,
      "unclipped_grad_norm": 0.7786760330200195
    },
    {
      "bits_per_byte": 1.4626550033827992,
      "learning_rate": 9.0230177746483e-05,
      "loss_nats": 1.0138351917266846,
      "step": 5620,
      "unclipped_grad_norm": 0.8055174350738525
    },
    {
      "bits_per_byte": 1.4690646247978054,
      "learning_rate": 8.976617973279797e-05,
      "loss_nats": 1.0182780027389526,
      "step": 5630,
      "unclipped_grad_norm": 0.7851108312606812
    },
    {
      "bits_per_byte": 1.4663743001903584,
      "learning_rate": 8.930346725752419e-05,
      "loss_nats": 1.0164132118225098,
      "step": 5640,
      "unclipped_grad_norm": 0.8267287015914917
    },
    {
      "bits_per_byte": 1.5738650048234715,
      "learning_rate": 8.88420482271411e-05,
      "loss_nats": 1.090920090675354,
      "step": 5650,
      "unclipped_grad_norm": 0.8359448313713074
    },
    {
      "bits_per_byte": 1.5300369460775933,
      "learning_rate": 8.838193052602691e-05,
      "loss_nats": 1.060540795326233,
      "step": 5660,
      "unclipped_grad_norm": 0.8635727167129517
    },
    {
      "bits_per_byte": 1.5230305448640935,
      "learning_rate": 8.792312201632352e-05,
      "loss_nats": 1.0556843280792236,
      "step": 5670,
      "unclipped_grad_norm": 0.8095638751983643
    },
    {
      "bits_per_byte": 1.524155483383387,
      "learning_rate": 8.746563053780261e-05,
      "loss_nats": 1.0564640760421753,
      "step": 5680,
      "unclipped_grad_norm": 0.8353701233863831
    },
    {
      "bits_per_byte": 1.566590998605229,
      "learning_rate": 8.700946390773114e-05,
      "loss_nats": 1.0858781337738037,
      "step": 5690,
      "unclipped_grad_norm": 0.8297926187515259
    },
    {
      "bits_per_byte": 1.499949785187701,
      "learning_rate": 8.655462992073833e-05,
      "loss_nats": 1.0396859645843506,
      "step": 5700,
      "unclipped_grad_norm": 0.8335801959037781
    },
    {
      "bits_per_byte": 1.5306388853556132,
      "learning_rate": 8.610113634868199e-05,
      "loss_nats": 1.0609580278396606,
      "step": 5710,
      "unclipped_grad_norm": 0.8218200206756592
    },
    {
      "bits_per_byte": 1.5663932185567369,
      "learning_rate": 8.564899094051613e-05,
      "loss_nats": 1.0857410430908203,
      "step": 5720,
      "unclipped_grad_norm": 0.8909743428230286
    },
    {
      "bits_per_byte": 1.4614645394735268,
      "learning_rate": 8.519820142215816e-05,
      "loss_nats": 1.013010025024414,
      "step": 5730,
      "unclipped_grad_norm": 0.7809724807739258
    },
    {
      "bits_per_byte": 1.5314872757723195,
      "learning_rate": 8.474877549635728e-05,
      "loss_nats": 1.0615460872650146,
      "step": 5740,
      "unclipped_grad_norm": 0.8163876533508301
    },
    {
      "bits_per_byte": 1.499649159513993,
      "learning_rate": 8.430072084256247e-05,
      "loss_nats": 1.0394775867462158,
      "step": 5750,
      "unclipped_grad_norm": 0.8667106628417969
    },
    {
      "bits_per_byte": 1.4999243317553734,
      "learning_rate": 8.385404511679161e-05,
      "loss_nats": 1.039668321609497,
      "step": 5760,
      "unclipped_grad_norm": 0.8516601324081421
    },
    {
      "bits_per_byte": 1.450708744488567,
      "learning_rate": 8.340875595150034e-05,
      "loss_nats": 1.0055546760559082,
      "step": 5770,
      "unclipped_grad_norm": 0.7943668365478516
    },
    {
      "bits_per_byte": 1.5466515020468459,
      "learning_rate": 8.296486095545203e-05,
      "loss_nats": 1.0720571279525757,
      "step": 5780,
      "unclipped_grad_norm": 0.8325088024139404
    },
    {
      "bits_per_byte": 1.494320105094358,
      "learning_rate": 8.252236771358729e-05,
      "loss_nats": 1.0357837677001953,
      "step": 5790,
      "unclipped_grad_norm": 0.8366720080375671
    },
    {
      "bits_per_byte": 1.4906410522271007,
      "learning_rate": 8.208128378689484e-05,
      "loss_nats": 1.033233642578125,
      "step": 5800,
      "unclipped_grad_norm": 0.8333538770675659
    },
    {
      "bits_per_byte": 1.5321304908865463,
      "learning_rate": 8.16416167122819e-05,
      "loss_nats": 1.0619919300079346,
      "step": 5810,
      "unclipped_grad_norm": 0.8446563482284546
    },
    {
      "bits_per_byte": 1.51830377368778,
      "learning_rate": 8.120337400244578e-05,
      "loss_nats": 1.05240797996521,
      "step": 5820,
      "unclipped_grad_norm": 0.8496891856193542
    },
    {
      "bits_per_byte": 1.4932847695361637,
      "learning_rate": 8.076656314574521e-05,
      "loss_nats": 1.0350661277770996,
      "step": 5830,
      "unclipped_grad_norm": 0.8205961585044861
    },
    {
      "bits_per_byte": 1.456290785387621,
      "learning_rate": 8.03311916060725e-05,
      "loss_nats": 1.009423851966858,
      "step": 5840,
      "unclipped_grad_norm": 0.8371089100837708
    },
    {
      "bits_per_byte": 1.454681199758196,
      "learning_rate": 7.989726682272606e-05,
      "loss_nats": 1.0083081722259521,
      "step": 5850,
      "unclipped_grad_norm": 0.8174164295196533
    },
    {
      "bits_per_byte": 1.4966561454410274,
      "learning_rate": 7.946479621028318e-05,
      "loss_nats": 1.0374029874801636,
      "step": 5860,
      "unclipped_grad_norm": 0.9058852791786194
    },
    {
      "bits_per_byte": 1.5418609253070623,
      "learning_rate": 7.903378715847343e-05,
      "loss_nats": 1.0687365531921387,
      "step": 5870,
      "unclipped_grad_norm": 0.8718268275260925
    },
    {
      "bits_per_byte": 1.5420466665699941,
      "learning_rate": 7.860424703205224e-05,
      "loss_nats": 1.0688652992248535,
      "step": 5880,
      "unclipped_grad_norm": 0.8315351009368896
    },
    {
      "bits_per_byte": 1.525186175410008,
      "learning_rate": 7.817618317067534e-05,
      "loss_nats": 1.0571784973144531,
      "step": 5890,
      "unclipped_grad_norm": 0.8407886028289795
    },
    {
      "bits_per_byte": 1.4989159974733648,
      "learning_rate": 7.774960288877295e-05,
      "loss_nats": 1.0389693975448608,
      "step": 5900,
      "unclipped_grad_norm": 0.8770707249641418
    },
    {
      "bits_per_byte": 1.4937374278732347,
      "learning_rate": 7.732451347542522e-05,
      "loss_nats": 1.0353798866271973,
      "step": 5910,
      "unclipped_grad_norm": 0.8901946544647217
    },
    {
      "bits_per_byte": 1.4996551789067731,
      "learning_rate": 7.690092219423722e-05,
      "loss_nats": 1.03948175907135,
      "step": 5920,
      "unclipped_grad_norm": 0.85027015209198
    },
    {
      "bits_per_byte": 1.5573538104273876,
      "learning_rate": 7.647883628321536e-05,
      "loss_nats": 1.0794754028320312,
      "step": 5930,
      "unclipped_grad_norm": 0.8851437568664551
    },
    {
      "bits_per_byte": 1.5404470559343193,
      "learning_rate": 7.605826295464316e-05,
      "loss_nats": 1.0677565336227417,
      "step": 5940,
      "unclipped_grad_norm": 0.8759228587150574
    },
    {
      "bits_per_byte": 1.5066732749405318,
      "learning_rate": 7.563920939495851e-05,
      "loss_nats": 1.0443463325500488,
      "step": 5950,
      "unclipped_grad_norm": 0.8818607926368713
    },
    {
      "bits_per_byte": 1.5044743047665998,
      "learning_rate": 7.522168276463047e-05,
      "loss_nats": 1.0428221225738525,
      "step": 5960,
      "unclipped_grad_norm": 0.816109299659729
    },
    {
      "bits_per_byte": 1.5173669841885302,
      "learning_rate": 7.480569019803719e-05,
      "loss_nats": 1.0517586469650269,
      "step": 5970,
      "unclipped_grad_norm": 0.8590478301048279
    },
    {
      "bits_per_byte": 1.4670959393933787,
      "learning_rate": 7.439123880334388e-05,
      "loss_nats": 1.0169134140014648,
      "step": 5980,
      "unclipped_grad_norm": 0.8544116616249084
    },
    {
      "bits_per_byte": 1.4408131207318973,
      "learning_rate": 7.39783356623813e-05,
      "loss_nats": 0.9986955523490906,
      "step": 5990,
      "unclipped_grad_norm": 0.8525132536888123
    },
    {
      "bits_per_byte": 1.5172739415744134,
      "learning_rate": 7.356698783052497e-05,
      "loss_nats": 1.0516941547393799,
      "step": 6000,
      "unclipped_grad_norm": 0.860197126865387
    },
    {
      "bits_per_byte": 1.499894750739425,
      "learning_rate": 7.315720233657431e-05,
      "loss_nats": 1.0396478176116943,
      "step": 6010,
      "unclipped_grad_norm": 0.8407278060913086
    },
    {
      "bits_per_byte": 1.4776142263374972,
      "learning_rate": 7.274898618263286e-05,
      "loss_nats": 1.024204134941101,
      "step": 6020,
      "unclipped_grad_norm": 0.8550556302070618
    },
    {
      "bits_per_byte": 1.5367169242197558,
      "learning_rate": 7.234234634398831e-05,
      "loss_nats": 1.0651710033416748,
      "step": 6030,
      "unclipped_grad_norm": 0.8657232522964478
    },
    {
      "bits_per_byte": 1.4931389282482321,
      "learning_rate": 7.193728976899362e-05,
      "loss_nats": 1.0349650382995605,
      "step": 6040,
      "unclipped_grad_norm": 0.8972119092941284
    },
    {
      "bits_per_byte": 1.498401597364634,
      "learning_rate": 7.153382337894797e-05,
      "loss_nats": 1.0386128425598145,
      "step": 6050,
      "unclipped_grad_norm": 0.8974336981773376
    },
    {
      "bits_per_byte": 1.5272425719663745,
      "learning_rate": 7.113195406797887e-05,
      "loss_nats": 1.0586038827896118,
      "step": 6060,
      "unclipped_grad_norm": 0.8948551416397095
    },
    {
      "bits_per_byte": 1.5059438965182226,
      "learning_rate": 7.073168870292397e-05,
      "loss_nats": 1.043840765953064,
      "step": 6070,
      "unclipped_grad_norm": 0.843971312046051
    },
    {
      "bits_per_byte": 1.4791374766761891,
      "learning_rate": 7.033303412321404e-05,
      "loss_nats": 1.0252599716186523,
      "step": 6080,
      "unclipped_grad_norm": 0.8345775008201599
    },
    {
      "bits_per_byte": 1.4884991802932552,
      "learning_rate": 6.993599714075586e-05,
      "loss_nats": 1.0317490100860596,
      "step": 6090,
      "unclipped_grad_norm": 0.8372970223426819
    },
    {
      "bits_per_byte": 1.486896301987214,
      "learning_rate": 6.954058453981609e-05,
      "loss_nats": 1.0306379795074463,
      "step": 6100,
      "unclipped_grad_norm": 0.827723503112793
    },
    {
      "bits_per_byte": 1.4668594632484424,
      "learning_rate": 6.914680307690504e-05,
      "loss_nats": 1.0167495012283325,
      "step": 6110,
      "unclipped_grad_norm": 0.871588945388794
    },
    {
      "bits_per_byte": 1.4677823221529722,
      "learning_rate": 6.875465948066139e-05,
      "loss_nats": 1.017389178276062,
      "step": 6120,
      "unclipped_grad_norm": 0.8457695245742798
    },
    {
      "bits_per_byte": 1.5061057321926845,
      "learning_rate": 6.836416045173733e-05,
      "loss_nats": 1.0439529418945312,
      "step": 6130,
      "unclipped_grad_norm": 0.9012824296951294
    },
    {
      "bits_per_byte": 1.5178127632195666,
      "learning_rate": 6.797531266268377e-05,
      "loss_nats": 1.0520676374435425,
      "step": 6140,
      "unclipped_grad_norm": 0.8450613617897034
    },
    {
      "bits_per_byte": 1.4675592606548031,
      "learning_rate": 6.758812275783663e-05,
      "loss_nats": 1.0172345638275146,
      "step": 6150,
      "unclipped_grad_norm": 0.885280191898346
    },
    {
      "bits_per_byte": 1.5554258849112155,
      "learning_rate": 6.720259735320298e-05,
      "loss_nats": 1.078139066696167,
      "step": 6160,
      "unclipped_grad_norm": 0.9099593758583069
    },
    {
      "bits_per_byte": 1.421598961003528,
      "learning_rate": 6.681874303634842e-05,
      "loss_nats": 0.985377311706543,
      "step": 6170,
      "unclipped_grad_norm": 0.7732160091400146
    },
    {
      "bits_per_byte": 1.5028951600663776,
      "learning_rate": 6.643656636628402e-05,
      "loss_nats": 1.0417275428771973,
      "step": 6180,
      "unclipped_grad_norm": 0.869156002998352
    },
    {
      "bits_per_byte": 1.4956368042693635,
      "learning_rate": 6.605607387335466e-05,
      "loss_nats": 1.036696434020996,
      "step": 6190,
      "unclipped_grad_norm": 0.8908746838569641
    },
    {
      "bits_per_byte": 1.4533810109176732,
      "learning_rate": 6.567727205912724e-05,
      "loss_nats": 1.0074069499969482,
      "step": 6200,
      "unclipped_grad_norm": 0.8600793480873108
    },
    {
      "bits_per_byte": 1.475978499345141,
      "learning_rate": 6.530016739627959e-05,
      "loss_nats": 1.0230703353881836,
      "step": 6210,
      "unclipped_grad_norm": 0.8739057779312134
    },
    {
      "bits_per_byte": 1.4582183669384914,
      "learning_rate": 6.492476632848993e-05,
      "loss_nats": 1.010759949684143,
      "step": 6220,
      "unclipped_grad_norm": 0.8542900681495667
    },
    {
      "bits_per_byte": 1.4401650041121208,
      "learning_rate": 6.455107527032678e-05,
      "loss_nats": 0.9982463121414185,
      "step": 6230,
      "unclipped_grad_norm": 0.8145909309387207
    },
    {
      "bits_per_byte": 1.458188613939892,
      "learning_rate": 6.417910060713926e-05,
      "loss_nats": 1.0107393264770508,
      "step": 6240,
      "unclipped_grad_norm": 0.8962635397911072
    },
    {
      "bits_per_byte": 1.4595658510080016,
      "learning_rate": 6.380884869494799e-05,
      "loss_nats": 1.0116939544677734,
      "step": 6250,
      "unclipped_grad_norm": 0.855821430683136
    },
    {
      "bits_per_byte": 1.5147881043388425,
      "learning_rate": 6.34403258603367e-05,
      "loss_nats": 1.049971103668213,
      "step": 6260,
      "unclipped_grad_norm": 0.8663405179977417
    },
    {
      "bits_per_byte": 1.4764731214490225,
      "learning_rate": 6.30735384003437e-05,
      "loss_nats": 1.0234131813049316,
      "step": 6270,
      "unclipped_grad_norm": 0.8501848578453064
    },
    {
      "bits_per_byte": 1.59573105103682,
      "learning_rate": 6.270849258235484e-05,
      "loss_nats": 1.1060764789581299,
      "step": 6280,
      "unclipped_grad_norm": 0.9987632036209106
    },
    {
      "bits_per_byte": 1.4579987450933396,
      "learning_rate": 6.234519464399581e-05,
      "loss_nats": 1.0106077194213867,
      "step": 6290,
      "unclipped_grad_norm": 0.8792169690132141
    },
    {
      "bits_per_byte": 1.4898052165439075,
      "learning_rate": 6.198365079302614e-05,
      "loss_nats": 1.0326542854309082,
      "step": 6300,
      "unclipped_grad_norm": 0.8601147532463074
    },
    {
      "bits_per_byte": 1.537688110249178,
      "learning_rate": 6.16238672072326e-05,
      "loss_nats": 1.065844178199768,
      "step": 6310,
      "unclipped_grad_norm": 0.907733678817749
    },
    {
      "bits_per_byte": 1.487509248154889,
      "learning_rate": 6.126585003432406e-05,
      "loss_nats": 1.0310628414154053,
      "step": 6320,
      "unclipped_grad_norm": 0.8578711152076721
    },
    {
      "bits_per_byte": 1.4237945775157679,
      "learning_rate": 6.090960539182614e-05,
      "loss_nats": 0.986899197101593,
      "step": 6330,
      "unclipped_grad_norm": 0.845673143863678
    },
    {
      "bits_per_byte": 1.4966876182661353,
      "learning_rate": 6.0555139366976935e-05,
      "loss_nats": 1.0374248027801514,
      "step": 6340,
      "unclipped_grad_norm": 0.8802160620689392
    },
    {
      "bits_per_byte": 1.485631197607467,
      "learning_rate": 6.020245801662274e-05,
      "loss_nats": 1.0297610759735107,
      "step": 6350,
      "unclipped_grad_norm": 0.884225606918335
    },
    {
      "bits_per_byte": 1.4432266392627804,
      "learning_rate": 5.985156736711483e-05,
      "loss_nats": 1.0003684759140015,
      "step": 6360,
      "unclipped_grad_norm": 0.8594306707382202
    },
    {
      "bits_per_byte": 1.497669639202562,
      "learning_rate": 5.950247341420618e-05,
      "loss_nats": 1.0381054878234863,
      "step": 6370,
      "unclipped_grad_norm": 0.9075121283531189
    },
    {
      "bits_per_byte": 1.4643473126672892,
      "learning_rate": 5.915518212294935e-05,
      "loss_nats": 1.0150082111358643,
      "step": 6380,
      "unclipped_grad_norm": 0.8724440932273865
    },
    {
      "bits_per_byte": 1.4983035672536422,
      "learning_rate": 5.880969942759425e-05,
      "loss_nats": 1.0385448932647705,
      "step": 6390,
      "unclipped_grad_norm": 0.9329665303230286
    },
    {
      "bits_per_byte": 1.4855978329731998,
      "learning_rate": 5.846603123148688e-05,
      "loss_nats": 1.029737949371338,
      "step": 6400,
      "unclipped_grad_norm": 0.8683961033821106
    },
    {
      "bits_per_byte": 1.4585226042478678,
      "learning_rate": 5.812418340696858e-05,
      "loss_nats": 1.0109708309173584,
      "step": 6410,
      "unclipped_grad_norm": 0.8944246768951416
    },
    {
      "bits_per_byte": 1.4417726979323862,
      "learning_rate": 5.778416179527541e-05,
      "loss_nats": 0.9993606805801392,
      "step": 6420,
      "unclipped_grad_norm": 0.8450288772583008
    },
    {
      "bits_per_byte": 1.474598510554618,
      "learning_rate": 5.7445972206438635e-05,
      "loss_nats": 1.0221138000488281,
      "step": 6430,
      "unclipped_grad_norm": 0.8708812594413757
    },
    {
      "bits_per_byte": 1.4773634756325393,
      "learning_rate": 5.710962041918516e-05,
      "loss_nats": 1.024030327796936,
      "step": 6440,
      "unclipped_grad_norm": 0.8653560876846313
    },
    {
      "bits_per_byte": 1.4567353605401014,
      "learning_rate": 5.677511218083908e-05,
      "loss_nats": 1.0097320079803467,
      "step": 6450,
      "unclipped_grad_norm": 0.8851431012153625
    },
    {
      "bits_per_byte": 1.4433678370191387,
      "learning_rate": 5.644245320722316e-05,
      "loss_nats": 1.0004663467407227,
      "step": 6460,
      "unclipped_grad_norm": 0.902314305305481
    },
    {
      "bits_per_byte": 1.478021997202693,
      "learning_rate": 5.61116491825615e-05,
      "loss_nats": 1.024486780166626,
      "step": 6470,
      "unclipped_grad_norm": 0.8634862899780273
    },
    {
      "bits_per_byte": 1.49569390250945,
      "learning_rate": 5.5782705759382104e-05,
      "loss_nats": 1.036736011505127,
      "step": 6480,
      "unclipped_grad_norm": 0.8877258896827698
    },
    {
      "bits_per_byte": 1.4739315618345719,
      "learning_rate": 5.5455628558420546e-05,
      "loss_nats": 1.0216515064239502,
      "step": 6490,
      "unclipped_grad_norm": 0.9130761027336121
    },
    {
      "bits_per_byte": 1.4798123365981748,
      "learning_rate": 5.513042316852368e-05,
      "loss_nats": 1.0257277488708496,
      "step": 6500,
      "unclipped_grad_norm": 0.9244931936264038
    },
    {
      "bits_per_byte": 1.429752142532981,
      "learning_rate": 5.480709514655443e-05,
      "loss_nats": 0.9910286664962769,
      "step": 6510,
      "unclipped_grad_norm": 0.8583832383155823
    },
    {
      "bits_per_byte": 1.3992985726054255,
      "learning_rate": 5.448565001729653e-05,
      "loss_nats": 0.9699198603630066,
      "step": 6520,
      "unclipped_grad_norm": 0.8968297243118286
    },
    {
      "bits_per_byte": 1.5067831718544331,
      "learning_rate": 5.41660932733604e-05,
      "loss_nats": 1.0444225072860718,
      "step": 6530,
      "unclipped_grad_norm": 0.9009160399436951
    },
    {
      "bits_per_byte": 1.468959715380779,
      "learning_rate": 5.384843037508902e-05,
      "loss_nats": 1.0182052850723267,
      "step": 6540,
      "unclipped_grad_norm": 0.8687822222709656
    },
    {
      "bits_per_byte": 1.5110841139872102,
      "learning_rate": 5.353266675046498e-05,
      "loss_nats": 1.0474036931991577,
      "step": 6550,
      "unclipped_grad_norm": 0.9212287068367004
    },
    {
      "bits_per_byte": 1.4657298811975754,
      "learning_rate": 5.321880779501729e-05,
      "loss_nats": 1.015966534614563,
      "step": 6560,
      "unclipped_grad_norm": 0.8736171126365662
    },
    {
      "bits_per_byte": 1.3923212364599213,
      "learning_rate": 5.290685887172962e-05,
      "loss_nats": 0.9650835394859314,
      "step": 6570,
      "unclipped_grad_norm": 0.9459646344184875
    },
    {
      "bits_per_byte": 1.5011586512406159,
      "learning_rate": 5.2596825310948295e-05,
      "loss_nats": 1.040523886680603,
      "step": 6580,
      "unclipped_grad_norm": 0.8902279138565063
    },
    {
      "bits_per_byte": 1.4631396504929306,
      "learning_rate": 5.228871241029154e-05,
      "loss_nats": 1.0141711235046387,
      "step": 6590,
      "unclipped_grad_norm": 0.8934556245803833
    },
    {
      "bits_per_byte": 1.3892133379761795,
      "learning_rate": 5.198252543455865e-05,
      "loss_nats": 0.9629293084144592,
      "step": 6600,
      "unclipped_grad_norm": 0.8376922607421875
    },
    {
      "bits_per_byte": 1.5024242715683152,
      "learning_rate": 5.167826961564027e-05,
      "loss_nats": 1.0414011478424072,
      "step": 6610,
      "unclipped_grad_norm": 0.8684409856796265
    },
    {
      "bits_per_byte": 1.5173028346597586,
      "learning_rate": 5.137595015242884e-05,
      "loss_nats": 1.0517141819000244,
      "step": 6620,
      "unclipped_grad_norm": 0.9338745474815369
    },
    {
      "bits_per_byte": 1.4964176055042808,
      "learning_rate": 5.1075572210729925e-05,
      "loss_nats": 1.0372376441955566,
      "step": 6630,
      "unclipped_grad_norm": 0.9484766721725464
    },
    {
      "bits_per_byte": 1.4696447222791655,
      "learning_rate": 5.07771409231737e-05,
      "loss_nats": 1.0186800956726074,
      "step": 6640,
      "unclipped_grad_norm": 0.9511991143226624
    },
    {
      "bits_per_byte": 1.4476113369465278,
      "learning_rate": 5.048066138912756e-05,
      "loss_nats": 1.0034077167510986,
      "step": 6650,
      "unclipped_grad_norm": 0.8744685649871826
    },
    {
      "bits_per_byte": 1.5102728718230902,
      "learning_rate": 5.018613867460863e-05,
      "loss_nats": 1.0468413829803467,
      "step": 6660,
      "unclipped_grad_norm": 0.9144671559333801
    },
    {
      "bits_per_byte": 1.4729612357184039,
      "learning_rate": 4.989357781219755e-05,
      "loss_nats": 1.0209789276123047,
      "step": 6670,
      "unclipped_grad_norm": 0.918013870716095
    },
    {
      "bits_per_byte": 1.4976474534406006,
      "learning_rate": 4.960298380095222e-05,
      "loss_nats": 1.0380901098251343,
      "step": 6680,
      "unclipped_grad_norm": 0.9484038352966309
    },
    {
      "bits_per_byte": 1.4621077545877539,
      "learning_rate": 4.931436160632256e-05,
      "loss_nats": 1.013455867767334,
      "step": 6690,
      "unclipped_grad_norm": 0.9366878867149353
    },
    {
      "bits_per_byte": 1.513278096664267,
      "learning_rate": 4.9027716160065485e-05,
      "loss_nats": 1.048924446105957,
      "step": 6700,
      "unclipped_grad_norm": 0.9588149785995483
    },
    {
      "bits_per_byte": 1.448871969777352,
      "learning_rate": 4.874305236016084e-05,
      "loss_nats": 1.0042815208435059,
      "step": 6710,
      "unclipped_grad_norm": 0.8966986536979675
    },
    {
      "bits_per_byte": 1.4475960304906008,
      "learning_rate": 4.846037507072752e-05,
      "loss_nats": 1.0033971071243286,
      "step": 6720,
      "unclipped_grad_norm": 0.9147130250930786
    },
    {
      "bits_per_byte": 1.4208115384365525,
      "learning_rate": 4.817968912194049e-05,
      "loss_nats": 0.9848315119743347,
      "step": 6730,
      "unclipped_grad_norm": 0.9434075951576233
    },
    {
      "bits_per_byte": 1.4530764296429952,
      "learning_rate": 4.790099930994813e-05,
      "loss_nats": 1.0071958303451538,
      "step": 6740,
      "unclipped_grad_norm": 0.9062299728393555
    },
    {
      "bits_per_byte": 1.4277258429405155,
      "learning_rate": 4.762431039679047e-05,
      "loss_nats": 0.9896241426467896,
      "step": 6750,
      "unclipped_grad_norm": 0.9092971086502075
    },
    {
      "bits_per_byte": 1.4128754850038137,
      "learning_rate": 4.734962711031755e-05,
      "loss_nats": 0.9793306589126587,
      "step": 6760,
      "unclipped_grad_norm": 0.8910818099975586
    },
    {
      "bits_per_byte": 1.434429210723195,
      "learning_rate": 4.707695414410892e-05,
      "loss_nats": 0.9942705631256104,
      "step": 6770,
      "unclipped_grad_norm": 0.8744387030601501
    },
    {
      "bits_per_byte": 1.4315308730995298,
      "learning_rate": 4.680629615739322e-05,
      "loss_nats": 0.9922615885734558,
      "step": 6780,
      "unclipped_grad_norm": 0.9047504663467407
    },
    {
      "bits_per_byte": 1.53544803622169,
      "learning_rate": 4.653765777496863e-05,
      "loss_nats": 1.0642914772033691,
      "step": 6790,
      "unclipped_grad_norm": 0.9123643636703491
    },
    {
      "bits_per_byte": 1.504298710480069,
      "learning_rate": 4.6271043587123986e-05,
      "loss_nats": 1.0427004098892212,
      "step": 6800,
      "unclipped_grad_norm": 0.978726327419281
    },
    {
      "bits_per_byte": 1.4819994399691971,
      "learning_rate": 4.600645814956006e-05,
      "loss_nats": 1.027243733406067,
      "step": 6810,
      "unclipped_grad_norm": 0.9510568380355835
    },
    {
      "bits_per_byte": 1.4201767504722178,
      "learning_rate": 4.574390598331204e-05,
      "loss_nats": 0.9843915104866028,
      "step": 6820,
      "unclipped_grad_norm": 0.9054594039916992
    },
    {
      "bits_per_byte": 1.441129998766112,
      "learning_rate": 4.548339157467199e-05,
      "loss_nats": 0.9989151954650879,
      "step": 6830,
      "unclipped_grad_norm": 0.9485244154930115
    },
    {
      "bits_per_byte": 1.4736460706341397,
      "learning_rate": 4.52249193751124e-05,
      "loss_nats": 1.021453619003296,
      "step": 6840,
      "unclipped_grad_norm": 0.9141860604286194
    },
    {
      "bits_per_byte": 1.439767810179953,
      "learning_rate": 4.4968493801209985e-05,
      "loss_nats": 0.9979709982872009,
      "step": 6850,
      "unclipped_grad_norm": 0.9174992442131042
    },
    {
      "bits_per_byte": 1.4496288654237992,
      "learning_rate": 4.471411923457034e-05,
      "loss_nats": 1.0048061609268188,
      "step": 6860,
      "unclipped_grad_norm": 0.9317811727523804
    },
    {
      "bits_per_byte": 1.4588201342338605,
      "learning_rate": 4.446180002175291e-05,
      "loss_nats": 1.0111770629882812,
      "step": 6870,
      "unclipped_grad_norm": 0.9191097617149353
    },
    {
      "bits_per_byte": 1.4995917173086046,
      "learning_rate": 4.421154047419693e-05,
      "loss_nats": 1.0394377708435059,
      "step": 6880,
      "unclipped_grad_norm": 0.9317643642425537
    },
    {
      "bits_per_byte": 1.3990970949299397,
      "learning_rate": 4.396334486814753e-05,
      "loss_nats": 0.9697802066802979,
      "step": 6890,
      "unclipped_grad_norm": 0.9130836725234985
    },
    {
      "bits_per_byte": 1.4230204836042344,
      "learning_rate": 4.371721744458285e-05,
      "loss_nats": 0.986362636089325,
      "step": 6900,
      "unclipped_grad_norm": 0.8910167217254639
    },
    {
      "bits_per_byte": 1.5914774041330302,
      "learning_rate": 4.347316240914145e-05,
      "loss_nats": 1.1031280755996704,
      "step": 6910,
      "unclipped_grad_norm": 1.0252712965011597
    },
    {
      "bits_per_byte": 1.394897880535148,
      "learning_rate": 4.3231183932050546e-05,
      "loss_nats": 0.9668695330619812,
      "step": 6920,
      "unclipped_grad_norm": 0.8750832676887512
    },
    {
      "bits_per_byte": 1.42465509270936,
      "learning_rate": 4.299128614805464e-05,
      "loss_nats": 0.9874956607818604,
      "step": 6930,
      "unclipped_grad_norm": 0.8865536451339722
    },
    {
      "bits_per_byte": 1.4536008047454758,
      "learning_rate": 4.2753473156344965e-05,
      "loss_nats": 1.0075592994689941,
      "step": 6940,
      "unclipped_grad_norm": 0.913940966129303
    },
    {
      "bits_per_byte": 1.4401447961506444,
      "learning_rate": 4.251774902048942e-05,
      "loss_nats": 0.9982323050498962,
      "step": 6950,
      "unclipped_grad_norm": 0.941871702671051
    },
    {
      "bits_per_byte": 1.4963969675861772,
      "learning_rate": 4.228411776836306e-05,
      "loss_nats": 1.0372233390808105,
      "step": 6960,
      "unclipped_grad_norm": 0.9974261522293091
    },
    {
      "bits_per_byte": 1.4704031657694705,
      "learning_rate": 4.2052583392079427e-05,
      "loss_nats": 1.0192058086395264,
      "step": 6970,
      "unclipped_grad_norm": 0.9634589552879333
    },
    {
      "bits_per_byte": 1.5157009882496222,
      "learning_rate": 4.182314984792214e-05,
      "loss_nats": 1.0506038665771484,
      "step": 6980,
      "unclipped_grad_norm": 0.9805173873901367
    },
    {
      "bits_per_byte": 1.4572141602401034,
      "learning_rate": 4.159582105627748e-05,
      "loss_nats": 1.010063886642456,
      "step": 6990,
      "unclipped_grad_norm": 0.9425977468490601
    },
    {
      "bits_per_byte": 1.4146713278441232,
      "learning_rate": 4.137060090156724e-05,
      "loss_nats": 0.980575442314148,
      "step": 7000,
      "unclipped_grad_norm": 0.9211949110031128
    },
    {
      "bits_per_byte": 1.4423556331274856,
      "learning_rate": 4.114749323218249e-05,
      "loss_nats": 0.9997647404670715,
      "step": 7010,
      "unclipped_grad_norm": 0.9508169293403625
    },
    {
      "bits_per_byte": 1.4268384124620634,
      "learning_rate": 4.0926501860417706e-05,
      "loss_nats": 0.9890090227127075,
      "step": 7020,
      "unclipped_grad_norm": 0.9370734095573425
    },
    {
      "bits_per_byte": 1.4214180352548202,
      "learning_rate": 4.070763056240571e-05,
      "loss_nats": 0.9852519035339355,
      "step": 7030,
      "unclipped_grad_norm": 0.8771427273750305
    },
    {
      "bits_per_byte": 1.5093840654834314,
      "learning_rate": 4.049088307805306e-05,
      "loss_nats": 1.0462253093719482,
      "step": 7040,
      "unclipped_grad_norm": 0.9992957711219788
    },
    {
      "bits_per_byte": 1.466185635222362,
      "learning_rate": 4.027626311097629e-05,
      "loss_nats": 1.0162824392318726,
      "step": 7050,
      "unclipped_grad_norm": 0.9372990131378174
    },
    {
      "bits_per_byte": 1.4783773133593756,
      "learning_rate": 4.006377432843846e-05,
      "loss_nats": 1.024733066558838,
      "step": 7060,
      "unclipped_grad_norm": 1.0182687044143677
    },
    {
      "bits_per_byte": 1.4142496263842075,
      "learning_rate": 3.985342036128656e-05,
      "loss_nats": 0.9802831411361694,
      "step": 7070,
      "unclipped_grad_norm": 0.8880259394645691
    },
    {
      "bits_per_byte": 1.3447492013960853,
      "learning_rate": 3.964520480388956e-05,
      "loss_nats": 0.9321091175079346,
      "step": 7080,
      "unclipped_grad_norm": 0.910698413848877
    },
    {
      "bits_per_byte": 1.413691198716856,
      "learning_rate": 3.943913121407678e-05,
      "loss_nats": 0.979896068572998,
      "step": 7090,
      "unclipped_grad_norm": 0.9244340062141418
    },
    {
      "bits_per_byte": 1.4025504205679395,
      "learning_rate": 3.923520311307735e-05,
      "loss_nats": 0.9721738696098328,
      "step": 7100,
      "unclipped_grad_norm": 0.9016135334968567
    },
    {
      "bits_per_byte": 1.4533139376838367,
      "learning_rate": 3.9033423985459816e-05,
      "loss_nats": 1.0073604583740234,
      "step": 7110,
      "unclipped_grad_norm": 0.970280110836029
    },
    {
      "bits_per_byte": 1.4606615524766484,
      "learning_rate": 3.88337972790728e-05,
      "loss_nats": 1.0124534368515015,
      "step": 7120,
      "unclipped_grad_norm": 0.9771916270256042
    },
    {
      "bits_per_byte": 1.396041221198084,
      "learning_rate": 3.8636326404985845e-05,
      "loss_nats": 0.9676620364189148,
      "step": 7130,
      "unclipped_grad_norm": 0.8866344094276428
    },
    {
      "bits_per_byte": 1.5095894127685614,
      "learning_rate": 3.844101473743143e-05,
      "loss_nats": 1.0463676452636719,
      "step": 7140,
      "unclipped_grad_norm": 0.9947805404663086
    },
    {
      "bits_per_byte": 1.5291790966150898,
      "learning_rate": 3.824786561374701e-05,
      "loss_nats": 1.0599461793899536,
      "step": 7150,
      "unclipped_grad_norm": 0.9418985247612
    },
    {
      "bits_per_byte": 1.4640222654571586,
      "learning_rate": 3.805688233431824e-05,
      "loss_nats": 1.0147829055786133,
      "step": 7160,
      "unclipped_grad_norm": 1.0358314514160156
    },
    {
      "bits_per_byte": 1.4559156912260893,
      "learning_rate": 3.786806816252236e-05,
      "loss_nats": 1.0091638565063477,
      "step": 7170,
      "unclipped_grad_norm": 0.9539772272109985
    },
    {
      "bits_per_byte": 1.3591113005869877,
      "learning_rate": 3.768142632467269e-05,
      "loss_nats": 0.9420641660690308,
      "step": 7180,
      "unclipped_grad_norm": 0.8915972709655762
    },
    {
      "bits_per_byte": 1.3840849873101015,
      "learning_rate": 3.7496960009963236e-05,
      "loss_nats": 0.9593746066093445,
      "step": 7190,
      "unclipped_grad_norm": 0.9118176102638245
    },
    {
      "bits_per_byte": 1.4715360154907038,
      "learning_rate": 3.7314672370414325e-05,
      "loss_nats": 1.0199910402297974,
      "step": 7200,
      "unclipped_grad_norm": 0.973518431186676
    },
    {
      "bits_per_byte": 1.4678313372084681,
      "learning_rate": 3.713456652081882e-05,
      "loss_nats": 1.017423152923584,
      "step": 7210,
      "unclipped_grad_norm": 0.9181845188140869
    },
    {
      "bits_per_byte": 1.4265923052886786,
      "learning_rate": 3.695664553868873e-05,
      "loss_nats": 0.9888384342193604,
      "step": 7220,
      "unclipped_grad_norm": 0.926552951335907
    },
    {
      "bits_per_byte": 1.4337483314084294,
      "learning_rate": 3.678091246420277e-05,
      "loss_nats": 0.9937986135482788,
      "step": 7230,
      "unclipped_grad_norm": 0.9526586532592773
    },
    {
      "bits_per_byte": 1.383381664259398,
      "learning_rate": 3.660737030015427e-05,
      "loss_nats": 0.9588871002197266,
      "step": 7240,
      "unclipped_grad_norm": 0.8890237212181091
    },
    {
      "bits_per_byte": 1.431437056563484,
      "learning_rate": 3.6436022011900056e-05,
      "loss_nats": 0.9921965599060059,
      "step": 7250,
      "unclipped_grad_norm": 0.9829798340797424
    },
    {
      "bits_per_byte": 1.4272578781475178,
      "learning_rate": 3.626687052730954e-05,
      "loss_nats": 0.9892997741699219,
      "step": 7260,
      "unclipped_grad_norm": 0.9537968039512634
    },
    {
      "bits_per_byte": 1.5069463833901018,
      "learning_rate": 3.6099918736714955e-05,
      "loss_nats": 1.0445356369018555,
      "step": 7270,
      "unclipped_grad_norm": 0.9953339099884033
    },
    {
      "bits_per_byte": 1.4318632295723221,
      "learning_rate": 3.5935169492861716e-05,
      "loss_nats": 0.9924919605255127,
      "step": 7280,
      "unclipped_grad_norm": 0.9815621972084045
    },
    {
      "bits_per_byte": 1.435238647069481,
      "learning_rate": 3.5772625610859886e-05,
      "loss_nats": 0.9948316216468811,
      "step": 7290,
      "unclipped_grad_norm": 0.9373346567153931
    },
    {
      "bits_per_byte": 1.44098209368637,
      "learning_rate": 3.5612289868135887e-05,
      "loss_nats": 0.9988126754760742,
      "step": 7300,
      "unclipped_grad_norm": 0.9782799482345581
    },
    {
      "bits_per_byte": 1.467732619166873,
      "learning_rate": 3.545416500438522e-05,
      "loss_nats": 1.0173547267913818,
      "step": 7310,
      "unclipped_grad_norm": 0.9545702338218689
    },
    {
      "bits_per_byte": 1.4680886232541588,
      "learning_rate": 3.52982537215255e-05,
      "loss_nats": 1.017601490020752,
      "step": 7320,
      "unclipped_grad_norm": 0.9820490479469299
    },
    {
      "bits_per_byte": 1.4597027491980883,
      "learning_rate": 3.5144558683650396e-05,
      "loss_nats": 1.0117888450622559,
      "step": 7330,
      "unclipped_grad_norm": 1.0537965297698975
    },
    {
      "bits_per_byte": 1.4358094574876945,
      "learning_rate": 3.499308251698404e-05,
      "loss_nats": 0.9952272772789001,
      "step": 7340,
      "unclipped_grad_norm": 0.9470505714416504
    },
    {
      "bits_per_byte": 1.4763711357370608,
      "learning_rate": 3.484382780983616e-05,
      "loss_nats": 1.023342490196228,
      "step": 7350,
      "unclipped_grad_norm": 0.93792724609375
    },
    {
      "bits_per_byte": 1.4524729425211176,
      "learning_rate": 3.469679711255795e-05,
      "loss_nats": 1.0067775249481201,
      "step": 7360,
      "unclipped_grad_norm": 0.959013044834137
    },
    {
      "bits_per_byte": 1.4678055398108387,
      "learning_rate": 3.4551992937498284e-05,
      "loss_nats": 1.0174052715301514,
      "step": 7370,
      "unclipped_grad_norm": 0.9769075512886047
    },
    {
      "bits_per_byte": 1.5064952728968888,
      "learning_rate": 3.440941775896104e-05,
      "loss_nats": 1.0442229509353638,
      "step": 7380,
      "unclipped_grad_norm": 1.0261726379394531
    },
    {
      "bits_per_byte": 1.4743861119808024,
      "learning_rate": 3.4269074013162596e-05,
      "loss_nats": 1.021966576576233,
      "step": 7390,
      "unclipped_grad_norm": 0.977978527545929
    },
    {
      "bits_per_byte": 1.4106443540741704,
      "learning_rate": 3.41309640981904e-05,
      "loss_nats": 0.9777841567993164,
      "step": 7400,
      "unclipped_grad_norm": 0.9216173887252808
    },
    {
      "bits_per_byte": 1.4343313525948542,
      "learning_rate": 3.3995090373961795e-05,
      "loss_nats": 0.994202733039856,
      "step": 7410,
      "unclipped_grad_norm": 0.9778451919555664
    },
    {
      "bits_per_byte": 1.5075760118749106,
      "learning_rate": 3.3861455162183876e-05,
      "loss_nats": 1.0449720621109009,
      "step": 7420,
      "unclipped_grad_norm": 1.0143206119537354
    },
    {
      "bits_per_byte": 1.4176087055295346,
      "learning_rate": 3.3730060746313664e-05,
      "loss_nats": 0.9826114773750305,
      "step": 7430,
      "unclipped_grad_norm": 0.9865745902061462
    },
    {
      "bits_per_byte": 1.4384498211350658,
      "learning_rate": 3.3600909371519225e-05,
      "loss_nats": 0.9970574378967285,
      "step": 7440,
      "unclipped_grad_norm": 0.9753794074058533
    },
    {
      "bits_per_byte": 1.385612537215065,
      "learning_rate": 3.3474003244641194e-05,
      "loss_nats": 0.9604334235191345,
      "step": 7450,
      "unclipped_grad_norm": 0.9882722496986389
    },
    {
      "bits_per_byte": 1.40335280562554,
      "learning_rate": 3.334934453415514e-05,
      "loss_nats": 0.9727300405502319,
      "step": 7460,
      "unclipped_grad_norm": 0.8938714861869812
    },
    {
      "bits_per_byte": 1.4650283639647061,
      "learning_rate": 3.3226935370134486e-05,
      "loss_nats": 1.0154802799224854,
      "step": 7470,
      "unclipped_grad_norm": 0.95097815990448
    },
    {
      "bits_per_byte": 1.378530205661209,
      "learning_rate": 3.31067778442141e-05,
      "loss_nats": 0.9555243253707886,
      "step": 7480,
      "unclipped_grad_norm": 0.9593113660812378
    },
    {
      "bits_per_byte": 1.4079058743244819,
      "learning_rate": 3.298887400955457e-05,
      "loss_nats": 0.9758859872817993,
      "step": 7490,
      "unclipped_grad_norm": 0.9498793482780457
    },
    {
      "bits_per_byte": 1.4795925427703722,
      "learning_rate": 3.287322588080714e-05,
      "loss_nats": 1.0255753993988037,
      "step": 7500,
      "unclipped_grad_norm": 0.9999135732650757
    },
    {
      "bits_per_byte": 1.462181535144974,
      "learning_rate": 3.275983543407924e-05,
      "loss_nats": 1.0135070085525513,
      "step": 7510,
      "unclipped_grad_norm": 0.96773761510849
    },
    {
      "bits_per_byte": 1.424363066168195,
      "learning_rate": 3.2648704606900735e-05,
      "loss_nats": 0.9872932434082031,
      "step": 7520,
      "unclipped_grad_norm": 0.9349227547645569
    },
    {
      "bits_per_byte": 1.4575968216382733,
      "learning_rate": 3.253983529819086e-05,
      "loss_nats": 1.0103291273117065,
      "step": 7530,
      "unclipped_grad_norm": 0.9571959972381592
    },
    {
      "bits_per_byte": 1.484410464751643,
      "learning_rate": 3.243322936822571e-05,
      "loss_nats": 1.0289149284362793,
      "step": 7540,
      "unclipped_grad_norm": 0.9889418482780457
    },
    {
      "bits_per_byte": 1.4190770074112757,
      "learning_rate": 3.232888863860648e-05,
      "loss_nats": 0.9836292266845703,
      "step": 7550,
      "unclipped_grad_norm": 0.9660657048225403
    },
    {
      "bits_per_byte": 1.458957376389249,
      "learning_rate": 3.222681489222838e-05,
      "loss_nats": 1.0112721920013428,
      "step": 7560,
      "unclipped_grad_norm": 0.9621381163597107
    },
    {
      "bits_per_byte": 1.4744461339259534,
      "learning_rate": 3.212700987325009e-05,
      "loss_nats": 1.0220081806182861,
      "step": 7570,
      "unclipped_grad_norm": 0.9809478521347046
    },
    {
      "bits_per_byte": 1.424117474942763,
      "learning_rate": 3.202947528706403e-05,
      "loss_nats": 0.9871230125427246,
      "step": 7580,
      "unclipped_grad_norm": 1.0034832954406738
    },
    {
      "bits_per_byte": 1.4167208450944553,
      "learning_rate": 3.1934212800267116e-05,
      "loss_nats": 0.9819960594177246,
      "step": 7590,
      "unclipped_grad_norm": 0.9541960954666138
    },
    {
      "bits_per_byte": 1.3862604818521906,
      "learning_rate": 3.1841224040632484e-05,
      "loss_nats": 0.9608825445175171,
      "step": 7600,
      "unclipped_grad_norm": 0.950609564781189
    },
    {
      "bits_per_byte": 1.3616721222583348,
      "learning_rate": 3.175051059708143e-05,
      "loss_nats": 0.9438391923904419,
      "step": 7610,
      "unclipped_grad_norm": 0.9224535226821899
    },
    {
      "bits_per_byte": 1.4445553772233464,
      "learning_rate": 3.1662074019656414e-05,
      "loss_nats": 1.0012894868850708,
      "step": 7620,
      "unclipped_grad_norm": 0.9717100858688354
    },
    {
      "bits_per_byte": 1.4175787805482842,
      "learning_rate": 3.157591581949455e-05,
      "loss_nats": 0.9825907349586487,
      "step": 7630,
      "unclipped_grad_norm": 0.9714697599411011
    },
    {
      "bits_per_byte": 1.4152020663046858,
      "learning_rate": 3.149203746880175e-05,
      "loss_nats": 0.9809433221817017,
      "step": 7640,
      "unclipped_grad_norm": 0.9795277118682861
    },
    {
      "bits_per_byte": 1.5113582543326856,
      "learning_rate": 3.141044040082763e-05,
      "loss_nats": 1.0475937128067017,
      "step": 7650,
      "unclipped_grad_norm": 1.0563151836395264
    },
    {
      "bits_per_byte": 1.4375178471500403,
      "learning_rate": 3.133112600984091e-05,
      "loss_nats": 0.9964114427566528,
      "step": 7660,
      "unclipped_grad_norm": 0.9650751948356628
    },
    {
      "bits_per_byte": 1.4172367070557181,
      "learning_rate": 3.125409565110571e-05,
      "loss_nats": 0.9823536276817322,
      "step": 7670,
      "unclipped_grad_norm": 0.9894714951515198
    },
    {
      "bits_per_byte": 1.3937812831744212,
      "learning_rate": 3.117935064085835e-05,
      "loss_nats": 0.9660955667495728,
      "step": 7680,
      "unclipped_grad_norm": 0.9648637175559998
    },
    {
      "bits_per_byte": 1.4266868097553278,
      "learning_rate": 3.110689225628479e-05,
      "loss_nats": 0.9889039397239685,
      "step": 7690,
      "unclipped_grad_norm": 0.9686183333396912
    },
    {
      "bits_per_byte": 1.4503658110827464,
      "learning_rate": 3.1036721735498926e-05,
      "loss_nats": 1.005316972732544,
      "step": 7700,
      "unclipped_grad_norm": 0.9993582367897034
    },
    {
      "bits_per_byte": 1.3691781330725912,
      "learning_rate": 3.096884027752132e-05,
      "loss_nats": 0.9490419626235962,
      "step": 7710,
      "unclipped_grad_norm": 0.9620972275733948
    },
    {
      "bits_per_byte": 1.3650053180147068,
      "learning_rate": 3.090324904225885e-05,
      "loss_nats": 0.9461495876312256,
      "step": 7720,
      "unclipped_grad_norm": 0.9128993153572083
    },
    {
      "bits_per_byte": 1.4252105106803215,
      "learning_rate": 3.083994915048472e-05,
      "loss_nats": 0.9878806471824646,
      "step": 7730,
      "unclipped_grad_norm": 0.9594771265983582
    },
    {
      "bits_per_byte": 1.4081796707046554,
      "learning_rate": 3.077894168381942e-05,
      "loss_nats": 0.9760757684707642,
      "step": 7740,
      "unclipped_grad_norm": 0.9255442023277283
    },
    {
      "bits_per_byte": 1.433292233418341,
      "learning_rate": 3.0720227684712254e-05,
      "loss_nats": 0.9934824705123901,
      "step": 7750,
      "unclipped_grad_norm": 0.9785766005516052
    },
    {
      "bits_per_byte": 1.3999133245909345,
      "learning_rate": 3.0663808156423456e-05,
      "loss_nats": 0.9703459739685059,
      "step": 7760,
      "unclipped_grad_norm": 1.0032256841659546
    },
    {
      "bits_per_byte": 1.4048038232508695,
      "learning_rate": 3.060968406300711e-05,
      "loss_nats": 0.9737358093261719,
      "step": 7770,
      "unclipped_grad_norm": 1.0204466581344604
    },
    {
      "bits_per_byte": 1.536527399338505,
      "learning_rate": 3.055785632929463e-05,
      "loss_nats": 1.0650396347045898,
      "step": 7780,
      "unclipped_grad_norm": 1.0556561946868896
    },
    {
      "bits_per_byte": 1.4128328333063997,
      "learning_rate": 3.0508325840878984e-05,
      "loss_nats": 0.9793010950088501,
      "step": 7790,
      "unclipped_grad_norm": 1.0078489780426025
    },
    {
      "bits_per_byte": 1.451735308931567,
      "learning_rate": 3.046109344409957e-05,
      "loss_nats": 1.0062662363052368,
      "step": 7800,
      "unclipped_grad_norm": 1.0289596319198608
    },
    {
      "bits_per_byte": 1.3852072600983067,
      "learning_rate": 3.0416159946027727e-05,
      "loss_nats": 0.9601525068283081,
      "step": 7810,
      "unclipped_grad_norm": 0.9891387224197388
    },
    {
      "bits_per_byte": 1.3953124447150527,
      "learning_rate": 3.0373526114452946e-05,
      "loss_nats": 0.9671568870544434,
      "step": 7820,
      "unclipped_grad_norm": 0.9778857231140137
    },
    {
      "bits_per_byte": 1.45392653988621,
      "learning_rate": 3.0333192677869803e-05,
      "loss_nats": 1.0077850818634033,
      "step": 7830,
      "unclipped_grad_norm": 1.0185887813568115
    },
    {
      "bits_per_byte": 1.4113820736550464,
      "learning_rate": 3.029516032546544e-05,
      "loss_nats": 0.9782955050468445,
      "step": 7840,
      "unclipped_grad_norm": 0.9330953359603882
    },
    {
      "bits_per_byte": 1.380841996454107,
      "learning_rate": 3.02594297071078e-05,
      "loss_nats": 0.9571267366409302,
      "step": 7850,
      "unclipped_grad_norm": 0.9742368459701538
    },
    {
      "bits_per_byte": 1.3846048048723343,
      "learning_rate": 3.0226001433334603e-05,
      "loss_nats": 0.9597349166870117,
      "step": 7860,
      "unclipped_grad_norm": 0.9645414352416992
    },
    {
      "bits_per_byte": 1.4429803601067448,
      "learning_rate": 3.019487607534278e-05,
      "loss_nats": 1.0001977682113647,
      "step": 7870,
      "unclipped_grad_norm": 0.9958038926124573
    },
    {
      "bits_per_byte": 1.4256185395194934,
      "learning_rate": 3.016605416497886e-05,
      "loss_nats": 0.9881634712219238,
      "step": 7880,
      "unclipped_grad_norm": 0.9739922881126404
    },
    {
      "bits_per_byte": 1.4834710955126302,
      "learning_rate": 3.0139536194729734e-05,
      "loss_nats": 1.028263807296753,
      "step": 7890,
      "unclipped_grad_norm": 0.9912284016609192
    },
    {
      "bits_per_byte": 1.4598313922209336,
      "learning_rate": 3.0115322617714362e-05,
      "loss_nats": 1.0118780136108398,
      "step": 7900,
      "unclipped_grad_norm": 0.9979069232940674
    },
    {
      "bits_per_byte": 1.465345499972897,
      "learning_rate": 3.0093413847675965e-05,
      "loss_nats": 1.015700101852417,
      "step": 7910,
      "unclipped_grad_norm": 1.0503500699996948
    },
    {
      "bits_per_byte": 1.3660396216769959,
      "learning_rate": 3.0073810258974985e-05,
      "loss_nats": 0.946866512298584,
      "step": 7920,
      "unclipped_grad_norm": 0.9824531078338623
    },
    {
      "bits_per_byte": 1.3848080023743288,
      "learning_rate": 3.0056512186582658e-05,
      "loss_nats": 0.959875762462616,
      "step": 7930,
      "unclipped_grad_norm": 1.0091050863265991
    },
    {
      "bits_per_byte": 1.4375059803471308,
      "learning_rate": 3.0041519926075316e-05,
      "loss_nats": 0.9964032173156738,
      "step": 7940,
      "unclipped_grad_norm": 1.0658900737762451
    },
    {
      "bits_per_byte": 1.4049340141175726,
      "learning_rate": 3.0028833733629313e-05,
      "loss_nats": 0.9738260507583618,
      "step": 7950,
      "unclipped_grad_norm": 1.0148085355758667
    },
    {
      "bits_per_byte": 1.415199744538899,
      "learning_rate": 3.0018453826016686e-05,
      "loss_nats": 0.9809417128562927,
      "step": 7960,
      "unclipped_grad_norm": 0.9737400412559509
    },
    {
      "bits_per_byte": 1.4008050546356583,
      "learning_rate": 3.001038038060143e-05,
      "loss_nats": 0.9709640741348267,
      "step": 7970,
      "unclipped_grad_norm": 0.9626508951187134
    },
    {
      "bits_per_byte": 1.4372590132604919,
      "learning_rate": 3.0004613535336433e-05,
      "loss_nats": 0.9962320327758789,
      "step": 7980,
      "unclipped_grad_norm": 0.9673776626586914
    },
    {
      "bits_per_byte": 1.4403348369798477,
      "learning_rate": 3.000115338876117e-05,
      "loss_nats": 0.9983640313148499,
      "step": 7990,
      "unclipped_grad_norm": 0.9839751720428467
    },
    {
      "bits_per_byte": 1.3635582559903472,
      "learning_rate": 3e-05,
      "loss_nats": 0.9451465606689453,
      "step": 8000,
      "unclipped_grad_norm": 0.9773924946784973
    }
  ]
}