{
  "architecture": "block",
  "batch_size": 32,
  "canonical_sha256_without_self": "9dfebe28b0512ff1872838a4dcfef72cfaea9ff8cbaf3793992bd389f7da76b5",
  "depth": 32,
  "diagnostics": [
    {
      "activation_grad_rms_by_block": [
        0.0002677410375326872,
        0.0002473295899108052,
        0.00023034414334688336,
        0.00021438254043459892,
        0.00020044577831868082,
        0.00018819731485564262,
        0.00017973134526982903,
        0.000171217747265473,
        0.0001640921109355986,
        0.00016061548376455903,
        0.00015463090676348656,
        0.00015024171443656087,
        0.0001478576596127823,
        0.00014403100067283958,
        0.00013951722939964384,
        0.00013661097909789532,
        0.00013379570737015456,
        0.0001311724045081064,
        0.0001286817860091105,
        0.00012687484559137374,
        0.0001249351626029238,
        0.00012369138130452484,
        0.00012227990373503417,
        0.00012101328320568427,
        0.00011953213106608018,
        0.0001181897969217971,
        0.0001171672047348693,
        0.00011609455395955592,
        0.00011492414341773838,
        0.0001140913664130494,
        0.00011346775136189535,
        0.00011285000800853595
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00021242368711682502,
        "first_to_last_ratio": 1.8345658968465297,
        "imbalance_abs_log_ratio": 0.6068078850437921,
        "last_quartile_mean": 0.0001157896194854402,
        "mean": 0.0001511171253696375,
        "normalized": [
          1.7717451736709768,
          1.636674793183293,
          1.5242755762025908,
          1.4186515254985965,
          1.3264266232459345,
          1.2453738409548603,
          1.1893512719369177,
          1.1330135273991528,
          1.085860457802012,
          1.0628542818802846,
          1.023252039669589,
          0.9942070699734702,
          0.9784308644775869,
          0.9531083940389613,
          0.9232390376562555,
          0.9040072643238835,
          0.8853775311228679,
          0.8680181295617843,
          0.8515367513400655,
          0.8395795332993111,
          0.8267439067367663,
          0.8185133286645813,
          0.8091730400239778,
          0.800791325997512,
          0.7909899739933552,
          0.7821072339266707,
          0.7753403490721149,
          0.7682422073314642,
          0.7604971517068638,
          0.7549863467424895,
          0.7508596466770359,
          0.7467718018887739
        ],
        "population_cv": 0.2714186609524326
      },
      "activation_output_rms_by_block": [
        0.0034885562490671873,
        0.004831995815038681,
        0.005952076520770788,
        0.006925336085259914,
        0.00385080324485898,
        0.005542940925806761,
        0.006729288958013058,
        0.007927131839096546,
        0.004081308841705322,
        0.005811207927763462,
        0.007534808944910765,
        0.008581922389566898,
        0.00499920267611742,
        0.007102519739419222,
        0.008542952127754688,
        0.010447698645293713,
        0.005041283555328846,
        0.007636935915797949,
        0.009387043304741383,
        0.011020516976714134,
        0.005807018838822842,
        0.008367395959794521,
        0.01024632342159748,
        0.012378672137856483,
        0.006453148555010557,
        0.009127611294388771,
        0.011037957854568958,
        0.012611051090061665,
        0.007010402157902718,
        0.009794782847166061,
        0.012445803731679916,
        0.014087481424212456
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.0056560162047389895,
        "first_to_last_ratio": 0.5480088979804595,
        "imbalance_abs_log_ratio": 0.6014637549753238,
        "last_quartile_mean": 0.010321029869373888,
        "mean": 0.007962599374877755,
        "normalized": [
          0.4381177659237371,
          0.606836484864952,
          0.747504205668035,
          0.8697330807712821,
          0.4836113263475722,
          0.6961220406611074,
          0.845112084785299,
          0.9955457339856744,
          0.512559862622497,
          0.7298129234151854,
          0.9462750278110585,
          1.077778999737589,
          0.6278355146046978,
          0.8919850673170736,
          1.072884836415099,
          1.3120964842531853,
          0.6331203314377778,
          0.9591008609440675,
          1.1788918244911069,
          1.384035094304029,
          0.7292868277593074,
          1.0508372411895934,
          1.2868063479276561,
          1.554601902603762,
          0.8104324041933391,
          1.1463105029730196,
          1.3862254440922965,
          1.5837857082010076,
          0.8804162846646224,
          1.2300986632667847,
          1.5630327667805075,
          1.7692063559870779
        ],
        "population_cv": 0.3451071044649734
      },
      "bits_per_byte": 7.959341947457508,
      "branch_output_rms_by_sublayer": [
        0.0022265110164880753,
        0.0027073738165199757,
        0.0021613112185150385,
        0.002722575329244137,
        0.0025601957459002733,
        0.0026850115973502398,
        0.0025323182344436646,
        0.0026861224323511124,
        0.0027253651060163975,
        0.00271838391199708,
        0.002904455177485943,
        0.0026645169127732515,
        0.0029208383057266474,
        0.0026859452482312918,
        0.00299624796025455,
        0.0027359756641089916,
        0.0030937574338167906,
        0.0026767742820084095,
        0.0033804215490818024,
        0.0027451682835817337,
        0.0034370317589491606,
        0.0026999011170119047,
        0.0038969130255281925,
        0.002663193503394723,
        0.004255065228790045,
        0.002626648638397455,
        0.004119740333408117,
        0.0026531771291047335,
        0.003970506135374308,
        0.0026517712976783514,
        0.004385598469525576,
        0.002676903735846281,
        0.0041999719105660915,
        0.002670086920261383,
        0.005232699681073427,
        0.0026355870068073273,
        0.004664790350943804,
        0.0026610493659973145,
        0.004898969549685717,
        0.0026388936676084995,
        0.005131533369421959,
        0.0026469926815479994,
        0.005069608334451914,
        0.00264862016774714,
        0.0051318020559847355,
        0.002641494618728757,
        0.0058372304774820805,
        0.002674198243767023,
        0.0059690922498703,
        0.002557571977376938,
        0.00554921617731452,
        0.00252733426168561,
        0.0058550527319312096,
        0.0025468147359788418,
        0.006255472078919411,
        0.0026856842450797558,
        0.006502739619463682,
        0.002551089273765683,
        0.006122888531535864,
        0.0025778315030038357,
        0.007071305997669697,
        0.0026001501828432083,
        0.006241494789719582,
        0.0025662274565547705
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0056993408498569815,
        0.005586029321860134,
        0.005830344684528519,
        0.00540288712216694,
        0.00538025811909819,
        0.0052226633069546126,
        0.005297605878918106,
        0.0052023753769565,
        0.0048107019260181325,
        0.005248672753719287,
        0.004962353375385338,
        0.005501319898161136,
        0.005588445553926757,
        0.005229606380765812,
        0.004884302670286647,
        0.005031817046498285,
        0.005000470961685071,
        0.0053937604587735305,
        0.004980398173225761,
        0.004658922297714218,
        0.00496598185096497,
        0.004578385934052906,
        0.004596161734371138,
        0.004990442293511754,
        0.005201771295872185,
        0.00459813521112035,
        0.004706502430387731,
        0.004781878207244352,
        0.0048378722082161795,
        0.004473110720597102,
        0.005084432926394394,
        0.004685705425265166
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.005452688082542498,
        "first_to_last_ratio": 1.1368823875795573,
        "imbalance_abs_log_ratio": 0.12828976841498158,
        "last_quartile_mean": 0.004796176053137182,
        "mean": 0.005075395512328069,
        "normalized": [
          1.122935313319594,
          1.1006096585560166,
          1.1487468652180286,
          1.064525337787646,
          1.060066768398564,
          1.0290160233362764,
          1.0437818818356701,
          1.025018713185997,
          0.947847692723257,
          1.0341406380981206,
          0.9777274230809889,
          1.0839194472230789,
          1.1010857262951226,
          1.0303840100861434,
          0.9623491722808084,
          0.9914137793352396,
          0.9852376922229987,
          1.0627271206100406,
          0.9812827711906272,
          0.9179427074003902,
          0.9784423379227619,
          0.9020747098294245,
          0.9055770576316904,
          0.9832617539638114,
          1.024899691706223,
          0.905965889742295,
          0.9273173724009683,
          0.9421685848185097,
          0.9532010257062827,
          0.8813324419214177,
          1.00178063247374,
          0.9232197596982638
        ],
        "population_cv": 0.06921897980134169
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.02802751213312149,
        0.014061377383768559,
        0.01413465291261673,
        0.014165290631353855,
        0.014225807972252369,
        0.014281988143920898,
        0.01433455292135477,
        0.014350161887705326,
        0.01440478302538395,
        0.00966146495193243,
        0.009703479707241058,
        0.009741888381540775,
        0.00977881159633398,
        0.009820535778999329,
        0.009858808480203152,
        0.00991273857653141,
        0.009950309991836548,
        0.0074927168898284435,
        0.00752287870272994,
        0.007546746172010899,
        0.007571155205368996,
        0.00766373984515667,
        0.007693085819482803,
        0.007762390188872814,
        0.007788818795233965,
        0.006325984839349985,
        0.006353048607707024,
        0.006375366821885109,
        0.006389216054230928,
        0.006445344537496567,
        0.0064707002602517605,
        0.006555322092026472,
        0.00656962301582098,
        0.005531130358576775,
        0.005553483963012695,
        0.005611324217170477,
        0.0056249531917274,
        0.0056777093559503555,
        0.005688921548426151,
        0.00574630219489336,
        0.005768724251538515,
        0.004946626722812653,
        0.004966245498508215,
        0.0050392793491482735,
        0.005057469941675663,
        0.005084683652967215,
        0.005098348017781973,
        0.005203457549214363,
        0.005216081161051989,
        0.004667261149734259,
        0.0046758754178881645,
        0.004717131145298481,
        0.004727107938379049,
        0.004768412560224533,
        0.004777040798217058,
        0.004826710093766451,
        0.004837613552808762,
        0.0044052014127373695,
        0.0044186655431985855,
        0.004504534415900707,
        0.004510571248829365,
        0.004574235528707504,
        0.0045849340967834,
        0.004644096828997135
      ],
      "loss_nats": 5.516995429992676,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 2.1972246170043945,
        "mean_weights": [
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519
        ],
        "sources": 9
      },
      "step": 0,
      "stream_state_rms_by_sublayer": [
        0.0022265110164880753,
        0.0034885562490671873,
        0.0040087271481752396,
        0.004831995815038681,
        0.005379301495850086,
        0.005952076520770788,
        0.006410556845366955,
        0.006925336085259914,
        0.0027253651060163975,
        0.00385080324485898,
        0.004838948138058186,
        0.005542940925806761,
        0.006175771821290255,
        0.006729288958013058,
        0.00742313964292407,
        0.007927131839096546,
        0.0030937574338167906,
        0.004081308841705322,
        0.005150026176124811,
        0.005811207927763462,
        0.007019025273621082,
        0.007534808944910765,
        0.008143599145114422,
        0.008581922389566898,
        0.004255065228790045,
        0.00499920267611742,
        0.0066224560141563416,
        0.007102519739419222,
        0.008129785768687725,
        0.008542952127754688,
        0.010099578648805618,
        0.010447698645293713,
        0.0041999719105660915,
        0.005041283555328846,
        0.00719039561226964,
        0.007636935915797949,
        0.009030314162373543,
        0.009387043304741383,
        0.010736005380749702,
        0.011020516976714134,
        0.005131533369421959,
        0.005807018838822842,
        0.00787449348717928,
        0.008367395959794521,
        0.009912566281855106,
        0.01024632342159748,
        0.012113703414797783,
        0.012378672137856483,
        0.0059690922498703,
        0.006453148555010557,
        0.00876442901790142,
        0.009127611294388771,
        0.010806876234710217,
        0.011037957854568958,
        0.012353328987956047,
        0.012611051090061665,
        0.006502739619463682,
        0.007010402157902718,
        0.009470963850617409,
        0.009794782847166061,
        0.012123330496251583,
        0.012445803731679916,
        0.01384514570236206,
        0.014087481424212456
      ]
    },
    {
      "activation_grad_rms_by_block": [
        7.003638165770099e-05,
        6.680699152639136e-05,
        6.333583587547764e-05,
        6.047846545698121e-05,
        5.581479490501806e-05,
        5.2793002396356314e-05,
        4.981113306712359e-05,
        4.656345481635071e-05,
        4.2050978663610294e-05,
        3.9379403460770845e-05,
        3.663913594209589e-05,
        3.422236841288395e-05,
        3.138875399599783e-05,
        2.928113826783374e-05,
        2.7496294933371246e-05,
        2.5903584173647687e-05,
        2.449963358230889e-05,
        2.3339889594353735e-05,
        2.2383910618373193e-05,
        2.1501236915355548e-05,
        2.0739182218676433e-05,
        2.0070132450200617e-05,
        1.950136174855288e-05,
        1.9030989278689958e-05,
        1.87676414498128e-05,
        1.8413003999739885e-05,
        1.8116044884664007e-05,
        1.7846368791651912e-05,
        1.7435917470720597e-05,
        1.7234746337635443e-05,
        1.7042602848960087e-05,
        1.689592500042636e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 5.8205007462674985e-05,
        "first_to_last_ratio": 3.284886533563498,
        "imbalance_abs_log_ratio": 1.189332110567881,
        "last_quartile_mean": 1.7719031347951386e-05,
        "mean": 3.265063452317918e-05,
        "normalized": [
          2.145023601546889,
          2.046116178200582,
          1.9398041355219169,
          1.8522906626530224,
          1.7094551367874427,
          1.616905863157966,
          1.5255793277696308,
          1.4261117891382666,
          1.2879069358899493,
          1.2060838643982497,
          1.1221569343801026,
          1.0481379278736214,
          0.9613520366262555,
          0.8968015077026028,
          0.8421366180143057,
          0.7933562257498653,
          0.7503570432885839,
          0.714837243906681,
          0.6855582118161445,
          0.6585243205638621,
          0.6351846609275962,
          0.6146934889106834,
          0.5972735915655352,
          0.5828673640378894,
          0.5748017373594797,
          0.5639401582431183,
          0.5548451094205582,
          0.5465856652489405,
          0.5340146592968225,
          0.5278533354505021,
          0.5219685037625007,
          0.5174761607904339
        ],
        "population_cv": 0.5072011380661324
      },
      "activation_output_rms_by_block": [
        0.005866481456905603,
        0.010142817161977291,
        0.01482025533914566,
        0.01956234686076641,
        0.0067119961604475975,
        0.011908270418643951,
        0.017094247043132782,
        0.02295343205332756,
        0.007494525518268347,
        0.013766046613454819,
        0.02038806490600109,
        0.02682213857769966,
        0.009193498641252518,
        0.016781754791736603,
        0.024297412484884262,
        0.03140362724661827,
        0.009884056635200977,
        0.018740201368927956,
        0.02799132652580738,
        0.03797563910484314,
        0.012623735703527927,
        0.02351362258195877,
        0.034899674355983734,
        0.047332651913166046,
        0.01744839735329151,
        0.029514454305171967,
        0.044660378247499466,
        0.0601838044822216,
        0.018972838297486305,
        0.0376126803457737,
        0.0565216988325119,
        0.07583927363157272
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.013632480811793357,
        "first_to_last_ratio": 0.32005493218522185,
        "imbalance_abs_log_ratio": 1.1392626348419668,
        "last_quartile_mean": 0.04259419068694115,
        "mean": 0.025403792154975235,
        "normalized": [
          0.23092935972382672,
          0.3992639012357397,
          0.5833875213879504,
          0.7700561688360058,
          0.2642123711098415,
          0.4687595594389148,
          0.6729013896370168,
          0.9035435305603466,
          0.2950160146386087,
          0.5418894364067922,
          0.8025599005701267,
          1.0558320747576517,
          0.36189473544610173,
          0.6606003816028687,
          0.9564482474371727,
          1.2361787191078082,
          0.389077999650742,
          0.7376930678145923,
          1.1018562250488806,
          1.4948807199009364,
          0.4969232792693754,
          0.9255949835565679,
          1.3737978229029388,
          1.8632120600111322,
          0.6868422339014575,
          1.1618129342705898,
          1.7580201402629136,
          2.3690874226600394,
          0.7468506348084945,
          1.4805931380763324,
          2.2249315569778956,
          2.985352468990339
        ],
        "population_cv": 0.6518934572295025
      },
      "bits_per_byte": 5.150682269327325,
      "branch_output_rms_by_sublayer": [
        0.002919268561527133,
        0.005079356953501701,
        0.0026904908008873463,
        0.0052208867855370045,
        0.0028405722696334124,
        0.005689430516213179,
        0.0029144894797354937,
        0.005868426524102688,
        0.0030634324066340923,
        0.00590060418471694,
        0.0032732728868722916,
        0.0059102908708155155,
        0.003848863998427987,
        0.0058776624500751495,
        0.003392934100702405,
        0.006663382984697819,
        0.0037359180860221386,
        0.006277414970099926,
        0.004302032291889191,
        0.006413500756025314,
        0.004344160668551922,
        0.006856496445834637,
        0.004835603758692741,
        0.006499431096017361,
        0.005531937349587679,
        0.006718362681567669,
        0.005483300890773535,
        0.0070298356004059315,
        0.005397035740315914,
        0.006969383917748928,
        0.005739198997616768,
        0.0068367342464625835,
        0.005659298039972782,
        0.007033169735223055,
        0.007230010814964771,
        0.007157569285482168,
        0.006938732694834471,
        0.0075136395171284676,
        0.007343437522649765,
        0.007708695251494646,
        0.007882392033934593,
        0.007887008599936962,
        0.00830139685422182,
        0.008205465972423553,
        0.008992243558168411,
        0.00814912747591734,
        0.009393857792019844,
        0.008287571370601654,
        0.012831038795411587,
        0.008026733994483948,
        0.00905293133109808,
        0.008120592683553696,
        0.012243949808180332,
        0.008479002863168716,
        0.012258118949830532,
        0.00907627958804369,
        0.013431218452751637,
        0.00841403380036354,
        0.01483316533267498,
        0.008773921057581902,
        0.015388047322630882,
        0.008821850642561913,
        0.014634119346737862,
        0.009164483286440372
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00039664652254938686,
        0.00033413090452743513,
        0.0003161152806455274,
        0.0003097753114302562,
        0.00028318594507881776,
        0.00028174654548060826,
        0.00029366070958977545,
        0.0002617199768643518,
        0.00023850476284541366,
        0.0002578321956415913,
        0.00023580491915923377,
        0.00024129375767696618,
        0.00024677521358199467,
        0.00023282043614108414,
        0.00020802360655182162,
        0.00020816264493602737,
        0.00019546204845585216,
        0.00020244473639840523,
        0.00019508742079131332,
        0.00018509966437476374,
        0.00018490270602469487,
        0.0001750081453781441,
        0.00017332318168420213,
        0.00016951527760705314,
        0.0001849276373753642,
        0.00014904354377519617,
        0.00017214209100244258,
        0.0001582098123018746,
        0.0001606899226932654,
        0.0001683991686835421,
        0.00016911992547040899,
        0.00016138660409180947
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00030962264952076986,
        "first_to_last_ratio": 1.8709465967014842,
        "imbalance_abs_log_ratio": 0.6264445042344079,
        "last_quartile_mean": 0.00016548983817423794,
        "mean": 0.0002234675193377695,
        "normalized": [
          1.774962749507496,
          1.4952101563466982,
          1.4145916220053505,
          1.3862207463001954,
          1.2672353723620313,
          1.2607941696204648,
          1.314109139708619,
          1.1711768119140573,
          1.0672905107292818,
          1.1537792892929548,
          1.0552089174213117,
          1.0797710485712801,
          1.104300143095945,
          1.0418535849461763,
          0.9308896754583627,
          0.9315118615577912,
          0.8746776669606858,
          0.9059246596477921,
          0.8730012369110346,
          0.828306793413619,
          0.8274254199117672,
          0.7831480203331949,
          0.775607937107967,
          0.7585678585836542,
          0.827536985792765,
          0.6669584206996899,
          0.7703226469447271,
          0.7079767689314242,
          0.719075072607683,
          0.7535733568018358,
          0.7567986881117407,
          0.7221926684024036
        ],
        "population_cv": 0.268339881652204
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.6930912733078003,
          "mean_weights": [
            0.5023526549339294,
            0.49764734506607056
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.692903995513916,
          "mean_weights": [
            0.5103387832641602,
            0.48966121673583984
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6928629875183105,
          "mean_weights": [
            0.5053091049194336,
            0.4946908950805664
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6927823424339294,
          "mean_weights": [
            0.5126834511756897,
            0.4873165488243103
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6928155422210693,
          "mean_weights": [
            0.502953290939331,
            0.49704670906066895
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.692751407623291,
          "mean_weights": [
            0.5129156708717346,
            0.48708435893058777
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6925709247589111,
          "mean_weights": [
            0.5112693905830383,
            0.4887305796146393
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6929247379302979,
          "mean_weights": [
            0.5095131397247314,
            0.49048691987991333
          ],
          "sources": 2
        },
        {
          "entropy_mean": 1.0983518362045288,
          "mean_weights": [
            0.3398353159427643,
            0.32833361625671387,
            0.33183106780052185
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.098447322845459,
          "mean_weights": [
            0.3408135175704956,
            0.3303682208061218,
            0.3288182020187378
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0980432033538818,
          "mean_weights": [
            0.3472674489021301,
            0.3313220739364624,
            0.32141047716140747
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0985240936279297,
          "mean_weights": [
            0.3373557925224304,
            0.33420175313949585,
            0.32844245433807373
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.098268985748291,
          "mean_weights": [
            0.3445510268211365,
            0.33061009645462036,
            0.32483887672424316
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0985302925109863,
          "mean_weights": [
            0.3371829390525818,
            0.3332410156726837,
            0.3295760750770569
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0983161926269531,
          "mean_weights": [
            0.3423102796077728,
            0.3327076733112335,
            0.32498204708099365
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.098489761352539,
          "mean_weights": [
            0.33793729543685913,
            0.3344462811946869,
            0.327616423368454
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.3861088752746582,
          "mean_weights": [
            0.2561652958393097,
            0.2513129711151123,
            0.2458309531211853,
            0.2466908097267151
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3861844539642334,
          "mean_weights": [
            0.2536832094192505,
            0.25214672088623047,
            0.24721932411193848,
            0.24695077538490295
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3860987424850464,
          "mean_weights": [
            0.25705933570861816,
            0.2498370260000229,
            0.2462591528892517,
            0.24684447050094604
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862221240997314,
          "mean_weights": [
            0.2514326870441437,
            0.2528519928455353,
            0.24950753152370453,
            0.24620777368545532
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.386016845703125,
          "mean_weights": [
            0.25482749938964844,
            0.2536851465702057,
            0.24858617782592773,
            0.24290117621421814
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862277269363403,
          "mean_weights": [
            0.25171467661857605,
            0.2530350983142853,
            0.24868467450141907,
            0.24656552076339722
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3859888315200806,
          "mean_weights": [
            0.25458407402038574,
            0.2546530067920685,
            0.248786062002182,
            0.24197682738304138
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862541913986206,
          "mean_weights": [
            0.2501963675022125,
            0.2521768808364868,
            0.24995486438274384,
            0.24767187237739563
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.6090729236602783,
          "mean_weights": [
            0.20360878109931946,
            0.20631682872772217,
            0.19986985623836517,
            0.19557830691337585,
            0.19462622702121735
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6093083620071411,
          "mean_weights": [
            0.2017374038696289,
            0.20362424850463867,
            0.20011676847934723,
            0.1968517303466797,
            0.1976698487997055
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.608644723892212,
          "mean_weights": [
            0.20431089401245117,
            0.20831988751888275,
            0.20422273874282837,
            0.19545811414718628,
            0.18768835067749023
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.609345555305481,
          "mean_weights": [
            0.2002599537372589,
            0.20386311411857605,
            0.2001732438802719,
            0.19764204323291779,
            0.19806164503097534
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6086196899414062,
          "mean_weights": [
            0.20515945553779602,
            0.21025606989860535,
            0.20179016888141632,
            0.1938556730747223,
            0.18893863260746002
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.609370470046997,
          "mean_weights": [
            0.2002836912870407,
            0.20340535044670105,
            0.20010054111480713,
            0.1980183869600296,
            0.1981920301914215
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6083989143371582,
          "mean_weights": [
            0.20363344252109528,
            0.21095217764377594,
            0.20392383635044098,
            0.19413790106773376,
            0.18735264241695404
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6093392372131348,
          "mean_weights": [
            0.20084109902381897,
            0.20331618189811707,
            0.2011999487876892,
            0.19900763034820557,
            0.19563516974449158
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.7908520698547363,
          "mean_weights": [
            0.17160072922706604,
            0.17730243504047394,
            0.16903036832809448,
            0.16099891066551208,
            0.15854957699775696,
            0.1625179797410965
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7916775941848755,
          "mean_weights": [
            0.16706335544586182,
            0.16981256008148193,
            0.16858771443367004,
            0.1652718484401703,
            0.16462719440460205,
            0.16463729739189148
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7898297309875488,
          "mean_weights": [
            0.17284834384918213,
            0.18101516366004944,
            0.1735592484474182,
            0.16231873631477356,
            0.15689018368721008,
            0.153368279337883
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7917213439941406,
          "mean_weights": [
            0.16615921258926392,
            0.16885727643966675,
            0.16764026880264282,
            0.16661542654037476,
            0.16609220206737518,
            0.16463561356067657
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7890949249267578,
          "mean_weights": [
            0.17241932451725006,
            0.18413974344730377,
            0.17484498023986816,
            0.1621202528476715,
            0.15553176403045654,
            0.15094394981861115
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7917208671569824,
          "mean_weights": [
            0.16663745045661926,
            0.168935626745224,
            0.16597260534763336,
            0.16514870524406433,
            0.16661295294761658,
            0.16669265925884247
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.789101004600525,
          "mean_weights": [
            0.17229264974594116,
            0.18365857005119324,
            0.1747075319290161,
            0.1640234887599945,
            0.155386283993721,
            0.14993149042129517
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7917447090148926,
          "mean_weights": [
            0.1658121943473816,
            0.167683407664299,
            0.16629494726657867,
            0.1669958382844925,
            0.16702724993228912,
            0.1661863625049591
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.9436898231506348,
          "mean_weights": [
            0.14778709411621094,
            0.15856821835041046,
            0.1500256359577179,
            0.1408001184463501,
            0.13533659279346466,
            0.130964457988739,
            0.13651786744594574
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9458942413330078,
          "mean_weights": [
            0.142832413315773,
            0.1429906189441681,
            0.14157448709011078,
            0.14285019040107727,
            0.14392998814582825,
            0.14335544407367706,
            0.14246684312820435
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9425597190856934,
          "mean_weights": [
            0.1483290046453476,
            0.1615946888923645,
            0.15280696749687195,
            0.14239417016506195,
            0.13462956249713898,
            0.1307675987482071,
            0.12947802245616913
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9458987712860107,
          "mean_weights": [
            0.1426449418067932,
            0.14291533827781677,
            0.1416938602924347,
            0.14274153113365173,
            0.14325542747974396,
            0.1432831734418869,
            0.14346569776535034
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.941068410873413,
          "mean_weights": [
            0.14912807941436768,
            0.16456490755081177,
            0.15460123121738434,
            0.14255177974700928,
            0.13539987802505493,
            0.1283380389213562,
            0.1254161298274994
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9458861351013184,
          "mean_weights": [
            0.14264336228370667,
            0.1430070698261261,
            0.14314037561416626,
            0.14221979677677155,
            0.14132913947105408,
            0.1431397795677185,
            0.14452049136161804
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9415628910064697,
          "mean_weights": [
            0.14895881712436676,
            0.16342055797576904,
            0.15406624972820282,
            0.14293311536312103,
            0.1363803744316101,
            0.12938986718654633,
            0.1248510330915451
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9458723068237305,
          "mean_weights": [
            0.14191991090774536,
            0.1418565809726715,
            0.14164230227470398,
            0.14289972186088562,
            0.14397987723350525,
            0.14332248270511627,
            0.14437910914421082
          ],
          "sources": 7
        },
        {
          "entropy_mean": 2.073223114013672,
          "mean_weights": [
            0.13280923664569855,
            0.14827430248260498,
            0.13796664774417877,
            0.12756125628948212,
            0.12019773572683334,
            0.11310221254825592,
            0.10922321677207947,
            0.11086538434028625
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079413890838623,
          "mean_weights": [
            0.12475188821554184,
            0.12460128962993622,
            0.12424701452255249,
            0.12420951575040817,
            0.12484367191791534,
            0.12469655275344849,
            0.12630701065063477,
            0.12634305655956268
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.072925567626953,
          "mean_weights": [
            0.13228744268417358,
            0.14843900501728058,
            0.13848665356636047,
            0.12843599915504456,
            0.12169650197029114,
            0.11442162096500397,
            0.11023421585559845,
            0.10599856078624725
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0793263912200928,
          "mean_weights": [
            0.12420415878295898,
            0.12273682653903961,
            0.12360996007919312,
            0.1238352432847023,
            0.1241140365600586,
            0.12607696652412415,
            0.1275939792394638,
            0.12782883644104004
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.073427200317383,
          "mean_weights": [
            0.13135744631290436,
            0.1465209424495697,
            0.13772746920585632,
            0.12966200709342957,
            0.12215200066566467,
            0.11654195189476013,
            0.11090746521949768,
            0.10513069480657578
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079422950744629,
          "mean_weights": [
            0.12477636337280273,
            0.12493626773357391,
            0.1242394968867302,
            0.12470392882823944,
            0.12560302019119263,
            0.12461308389902115,
            0.12591606378555298,
            0.12521176040172577
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0725789070129395,
          "mean_weights": [
            0.13186156749725342,
            0.14827941358089447,
            0.13866659998893738,
            0.12988534569740295,
            0.12183371186256409,
            0.11524046957492828,
            0.1099783331155777,
            0.10425455868244171
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0793380737304688,
          "mean_weights": [
            0.12417487055063248,
            0.1225123256444931,
            0.12277349829673767,
            0.1243998259305954,
            0.12567630410194397,
            0.1261572688817978,
            0.12747105956077576,
            0.12683483958244324
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.1906280517578125,
          "mean_weights": [
            0.11854428052902222,
            0.13291707634925842,
            0.12378416955471039,
            0.11745279282331467,
            0.11140251904726028,
            0.10409101843833923,
            0.09838292002677917,
            0.09486069530248642,
            0.09856453537940979
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.197185516357422,
          "mean_weights": [
            0.11084984987974167,
            0.11021925508975983,
            0.11018812656402588,
            0.11046157777309418,
            0.11131840199232101,
            0.11167057603597641,
            0.11249847710132599,
            0.1118280440568924,
            0.11096569895744324
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1899101734161377,
          "mean_weights": [
            0.1190595030784607,
            0.13352498412132263,
            0.1239522397518158,
            0.11779220402240753,
            0.11234381794929504,
            0.10513705015182495,
            0.09944089502096176,
            0.0946362093091011,
            0.09411308169364929
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.197190284729004,
          "mean_weights": [
            0.11067462712526321,
            0.11021950840950012,
            0.11038416624069214,
            0.11103623360395432,
            0.11016053706407547,
            0.11144962906837463,
            0.11242866516113281,
            0.11139006912708282,
            0.11225657165050507
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1896841526031494,
          "mean_weights": [
            0.11968795955181122,
            0.13428261876106262,
            0.12373492866754532,
            0.11790731549263,
            0.11133000254631042,
            0.10504785925149918,
            0.09994535893201828,
            0.09480959177017212,
            0.09325435757637024
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1971938610076904,
          "mean_weights": [
            0.11116788536310196,
            0.1117495447397232,
            0.11143069714307785,
            0.11117202788591385,
            0.11217275261878967,
            0.11135637760162354,
            0.11160033196210861,
            0.10969291627407074,
            0.10965746641159058
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.188610792160034,
          "mean_weights": [
            0.11987313628196716,
            0.1360916942358017,
            0.12462760508060455,
            0.11755925416946411,
            0.11237476021051407,
            0.10438287258148193,
            0.09920360147953033,
            0.09431086480617523,
            0.09157621115446091
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.028571506962180138,
        0.014360599219799042,
        0.01502692885696888,
        0.014914286322891712,
        0.015751885250210762,
        0.015590262599289417,
        0.016715344041585922,
        0.016720104962587357,
        0.01783851347863674,
        0.01203431747853756,
        0.012985615991055965,
        0.01319553330540657,
        0.014028912410140038,
        0.014193638227880001,
        0.015184907242655754,
        0.015358141623437405,
        0.016659293323755264,
        0.012656479142606258,
        0.013609340414404869,
        0.013699201866984367,
        0.014725575223565102,
        0.01495310477912426,
        0.016002535820007324,
        0.01618308201432228,
        0.017315881326794624,
        0.01409733947366476,
        0.014967667870223522,
        0.01521329116076231,
        0.016154561191797256,
        0.016374915838241577,
        0.01737469807267189,
        0.017517324537038803,
        0.018479224294424057,
        0.015613679774105549,
        0.01647813245654106,
        0.016780734062194824,
        0.017671043053269386,
        0.017834074795246124,
        0.01892261393368244,
        0.01904122717678547,
        0.020303834229707718,
        0.017550688236951828,
        0.01861351728439331,
        0.018688000738620758,
        0.01984807848930359,
        0.019847052171826363,
        0.021230308338999748,
        0.021104788407683372,
        0.022779423743486404,
        0.019968591630458832,
        0.02138628251850605,
        0.021149026229977608,
        0.02279111184179783,
        0.02249406836926937,
        0.024287279695272446,
        0.02368815243244171,
        0.02611272968351841,
        0.02278727851808071,
        0.024725768715143204,
        0.024227207526564598,
        0.026407435536384583,
        0.02562388963997364,
        0.028016721829771996,
        0.0269329734146595
      ],
      "loss_nats": 3.570180892944336,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 2.1930694580078125,
        "mean_weights": [
          0.10071488469839096,
          0.09411877393722534,
          0.10181896388530731,
          0.10989432036876678,
          0.1145806610584259,
          0.11861918121576309,
          0.11984598636627197,
          0.12094368040561676,
          0.11946355551481247
        ],
        "sources": 9
      },
      "step": 100,
      "stream_state_rms_by_sublayer": [
        0.002919268561527133,
        0.005866481456905603,
        0.0065401275642216206,
        0.010142817161977291,
        0.010618876665830612,
        0.01482025533914566,
        0.015120964497327805,
        0.01956234686076641,
        0.0030634324066340923,
        0.0067119961604475975,
        0.007598300464451313,
        0.011908270418643951,
        0.012669830583035946,
        0.017094247043132782,
        0.017725100740790367,
        0.02295343205332756,
        0.0037359180860221386,
        0.007494525518268347,
        0.009108707308769226,
        0.013766046613454819,
        0.015153576619923115,
        0.02038806490600109,
        0.021781234070658684,
        0.02682213857769966,
        0.005531937349587679,
        0.009193498641252518,
        0.0119093032553792,
        0.016781754791736603,
        0.01904802769422531,
        0.024297412484884262,
        0.02626260370016098,
        0.03140362724661827,
        0.005659298039972782,
        0.009884056635200977,
        0.013823030516505241,
        0.018740201368927956,
        0.02223494090139866,
        0.02799132652580738,
        0.0319429449737072,
        0.03797563910484314,
        0.007882392033934593,
        0.012623735703527927,
        0.01784195750951767,
        0.02351362258195877,
        0.028768405318260193,
        0.034899674355983734,
        0.040818072855472565,
        0.047332651913166046,
        0.012831038795411587,
        0.01744839735329151,
        0.023826664313673973,
        0.029514454305171967,
        0.03824474662542343,
        0.044660378247499466,
        0.053081437945365906,
        0.0601838044822216,
        0.013431218452751637,
        0.018972838297486305,
        0.03125866502523422,
        0.0376126803457737,
        0.04969283565878868,
        0.0565216988325119,
        0.06860727071762085,
        0.07583927363157272
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.00014248774095904082,
        0.00014670786913484335,
        0.00014566494792234153,
        0.00013774850231129676,
        0.00013259105617180467,
        0.00011885961430380121,
        0.00010788240615511313,
        9.963813499780372e-05,
        9.571473492542282e-05,
        8.723558858036995e-05,
        8.013081242097542e-05,
        7.453432044712827e-05,
        7.300077413674444e-05,
        6.691268208669499e-05,
        6.17568803136237e-05,
        5.684744246536866e-05,
        5.7382596423849463e-05,
        5.201507519814186e-05,
        4.7512799937976524e-05,
        4.3485648347996175e-05,
        4.6350149204954505e-05,
        4.233795698382892e-05,
        3.877062044921331e-05,
        3.5837489122059196e-05,
        3.0636481824330986e-05,
        2.926486195065081e-05,
        2.8229198505869135e-05,
        2.7580599635257386e-05,
        2.3305021386477165e-05,
        2.3107419110601768e-05,
        2.298491745023057e-05,
        2.3026072085485794e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00012894753399450565,
        "first_to_last_ratio": 4.956313899688395,
        "imbalance_abs_log_ratio": 1.600662299067145,
        "last_quartile_mean": 2.601682149361295e-05,
        "mean": 6.873563796716553e-05,
        "normalized": [
          2.0729820101052394,
          2.1343785185339303,
          2.1192055857825096,
          2.004033226215171,
          1.9290001532413559,
          1.7292283569198776,
          1.5695265126752398,
          1.449584785193964,
          1.392505223726082,
          1.2691464160417294,
          1.1657826244262586,
          1.0843620958713254,
          1.0620513069452608,
          0.9734787377496757,
          0.8984695878304712,
          0.827044661933948,
          0.8348303459591176,
          0.7567409969045322,
          0.6912396733798123,
          0.6326506836056272,
          0.674324856446311,
          0.615953502956551,
          0.5640541296457265,
          0.5213814868377082,
          0.4457146646251511,
          0.42575966145291916,
          0.4106923182898804,
          0.40125618166856064,
          0.3390529581992065,
          0.3361781427218294,
          0.33439592808042723,
          0.33499466603460004
        ],
        "population_cv": 0.5868596491376813
      },
      "activation_output_rms_by_block": [
        0.021530626341700554,
        0.03857670724391937,
        0.04954109713435173,
        0.057694368064403534,
        0.02140580676496029,
        0.025963041931390762,
        0.03339661285281181,
        0.040483180433511734,
        0.02151828072965145,
        0.035047177225351334,
        0.03725159168243408,
        0.043239567428827286,
        0.02124657854437828,
        0.03028978779911995,
        0.03979094699025154,
        0.05222286656498909,
        0.03916332125663757,
        0.04088718071579933,
        0.04756740853190422,
        0.05542457476258278,
        0.020541559904813766,
        0.03680417314171791,
        0.05126549303531647,
        0.06891486048698425,
        0.02905990369617939,
        0.05358703061938286,
        0.07880406081676483,
        0.10752817988395691,
        0.029148343950510025,
        0.054091889411211014,
        0.08628532290458679,
        0.11884573101997375
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.036073930095881224,
        "first_to_last_ratio": 0.5177916953272103,
        "imbalance_abs_log_ratio": 0.6581822501725932,
        "last_quartile_mean": 0.0696688077878207,
        "mean": 0.04647241474594921,
        "normalized": [
          0.46329906589537145,
          0.8300990481085756,
          1.06603234209322,
          1.2414755802942754,
          0.4606131805713009,
          0.5586764120892566,
          0.7186330436105184,
          0.8711228081179162,
          0.4630334112674251,
          0.7541501214633191,
          0.8015850238486076,
          0.9304351337283648,
          0.4571868851775181,
          0.6517799422454194,
          0.8562272308804426,
          1.1237390363826774,
          0.8427218914862018,
          0.8798161433899512,
          1.0235622313138022,
          1.1926338466716728,
          0.4420161942758587,
          0.7919574083446129,
          1.1031381372276345,
          1.482919724824311,
          0.6253151219931478,
          1.1530933116414785,
          1.6957169376190813,
          2.3138065984257823,
          0.6272181919070765,
          1.163956934601208,
          1.8566997944110037,
          2.5573392660929675
        ],
        "population_cv": 0.5056023373965206
      },
      "bits_per_byte": 3.0609520355704922,
      "branch_output_rms_by_sublayer": [
        0.010549778118729591,
        0.017611408606171608,
        0.013795096427202225,
        0.014558257535099983,
        0.012347182258963585,
        0.01080919150263071,
        0.012820021249353886,
        0.010327119380235672,
        0.01913420483469963,
        0.008616750128567219,
        0.018582463264465332,
        0.007770510856062174,
        0.013614931143820286,
        0.007606723811477423,
        0.013612493872642517,
        0.008487755432724953,
        0.019129201769828796,
        0.00865182839334011,
        0.02525762841105461,
        0.009020541794598103,
        0.027986859902739525,
        0.008682658895850182,
        0.014458909630775452,
        0.008706990629434586,
        0.018231205642223358,
        0.009003226645290852,
        0.016914349049329758,
        0.009712434373795986,
        0.022478405386209488,
        0.00981286820024252,
        0.03932488337159157,
        0.011337134055793285,
        0.03611915931105614,
        0.010840619914233685,
        0.0266753900796175,
        0.010956712998449802,
        0.018954744562506676,
        0.012231985107064247,
        0.014813725836575031,
        0.014082341454923153,
        0.013802599161863327,
        0.014436324127018452,
        0.016533223912119865,
        0.016183899715542793,
        0.016679473221302032,
        0.017385432496666908,
        0.01826235093176365,
        0.02254617027938366,
        0.015529165975749493,
        0.022119678556919098,
        0.011229476891458035,
        0.022372635081410408,
        0.008122270926833153,
        0.024847008287906647,
        0.008894859813153744,
        0.026897024363279343,
        0.009738311171531677,
        0.026162855327129364,
        0.007465044502168894,
        0.024674534797668457,
        0.008605006150901318,
        0.030845578759908676,
        0.009693782776594162,
        0.029862264171242714
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0006151943096300597,
        0.0003715258289130363,
        0.0003398511463891951,
        0.0003030496529751275,
        0.00028984271802693696,
        0.0002579116693919001,
        0.00022400787473702318,
        0.00028477639847268295,
        0.000278695861793727,
        0.00022792247112618847,
        0.00023417680672174036,
        0.00019296669269337194,
        0.00020274366236211453,
        0.00017465616055496573,
        0.00015881266908702134,
        0.00017808900330691153,
        0.00016226510475368574,
        0.00014508607560593053,
        0.00013101353173913878,
        0.00011930204103094746,
        0.00012786456583361604,
        0.00012508100764064805,
        0.00012354146515800217,
        0.00012514897623388166,
        0.00011397604830293932,
        9.556324385288696e-05,
        9.289013346881659e-05,
        9.744025747256133e-05,
        8.302691520063822e-05,
        7.404706874242776e-05,
        8.914255818565412e-05,
        8.643783624226864e-05
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.0003357699498169952,
        "first_to_last_ratio": 3.666991625028822,
        "imbalance_abs_log_ratio": 1.2993716052111575,
        "last_quartile_mean": 9.156550768352412e-05,
        "mean": 0.00019143905486393895,
        "normalized": [
          3.2135256312631473,
          1.940700288021637,
          1.7752445896201103,
          1.5830085098911153,
          1.514020836724963,
          1.3472259857070703,
          1.1701263093689624,
          1.487556437609251,
          1.4557941794677367,
          1.1905745736583337,
          1.223244686870066,
          1.0079797606111183,
          1.0590506859021536,
          0.9123329650739171,
          0.8295729896906037,
          0.9302647436986373,
          0.8476071137574935,
          0.7578708310540252,
          0.6843615678747147,
          0.6231854890619823,
          0.6679126467924371,
          0.6533724674390322,
          0.645330521746498,
          0.6537275078109245,
          0.5953646625760102,
          0.49918363795102527,
          0.4852203931681348,
          0.5089883960292994,
          0.4336989401647838,
          0.386791863316788,
          0.46564457941463516,
          0.45151620866339115
        ],
        "population_cv": 0.578773657781326
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.658259391784668,
          "mean_weights": [
            0.3887256383895874,
            0.6112743616104126
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6694332361221313,
          "mean_weights": [
            0.42782124876976013,
            0.5721787214279175
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6890689730644226,
          "mean_weights": [
            0.504529595375061,
            0.49547040462493896
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6740086078643799,
          "mean_weights": [
            0.45225459337234497,
            0.547745406627655
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.679557204246521,
          "mean_weights": [
            0.4461178183555603,
            0.5538821816444397
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6740646362304688,
          "mean_weights": [
            0.44189155101776123,
            0.5581084489822388
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6792672276496887,
          "mean_weights": [
            0.4455937147140503,
            0.5544062852859497
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.674970269203186,
          "mean_weights": [
            0.44674086570739746,
            0.5532591342926025
          ],
          "sources": 2
        },
        {
          "entropy_mean": 1.093062162399292,
          "mean_weights": [
            0.3082374036312103,
            0.33985745906829834,
            0.35190510749816895
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0853323936462402,
          "mean_weights": [
            0.30180174112319946,
            0.3268374502658844,
            0.3713608384132385
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0921990871429443,
          "mean_weights": [
            0.3034760355949402,
            0.3359980285167694,
            0.3605259656906128
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.076724648475647,
          "mean_weights": [
            0.29931002855300903,
            0.30481812357902527,
            0.3958718776702881
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0856893062591553,
          "mean_weights": [
            0.2801036834716797,
            0.3310527801513672,
            0.3888435363769531
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0752873420715332,
          "mean_weights": [
            0.29463040828704834,
            0.31052330136299133,
            0.3948462903499603
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.088913917541504,
          "mean_weights": [
            0.29400017857551575,
            0.32166045904159546,
            0.3843393623828888
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.054490327835083,
          "mean_weights": [
            0.2850497364997864,
            0.2890807092189789,
            0.42586958408355713
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.3700100183486938,
          "mean_weights": [
            0.21610887348651886,
            0.2282472848892212,
            0.2728016972541809,
            0.28284209966659546
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3620216846466064,
          "mean_weights": [
            0.22791308164596558,
            0.2214205265045166,
            0.2562018632888794,
            0.2944645285606384
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3784711360931396,
          "mean_weights": [
            0.23347607254981995,
            0.23307976126670837,
            0.2570483684539795,
            0.2763957977294922
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3619427680969238,
          "mean_weights": [
            0.24715131521224976,
            0.22003202140331268,
            0.23014183342456818,
            0.3026748299598694
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3804662227630615,
          "mean_weights": [
            0.24302782118320465,
            0.23108476400375366,
            0.24228686094284058,
            0.2836005389690399
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3502743244171143,
          "mean_weights": [
            0.2455797642469406,
            0.21441739797592163,
            0.2208172082901001,
            0.31918564438819885
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3820215463638306,
          "mean_weights": [
            0.24522465467453003,
            0.22846508026123047,
            0.25027886033058167,
            0.27603137493133545
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3491160869598389,
          "mean_weights": [
            0.24498313665390015,
            0.21178750693798065,
            0.2205154001712799,
            0.3227139711380005
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.6032840013504028,
          "mean_weights": [
            0.20074918866157532,
            0.17959122359752655,
            0.18619459867477417,
            0.2124267816543579,
            0.22103822231292725
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5699318647384644,
          "mean_weights": [
            0.1938496232032776,
            0.17019830644130707,
            0.16257275640964508,
            0.20397180318832397,
            0.2694075107574463
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6068798303604126,
          "mean_weights": [
            0.19717518985271454,
            0.1942940056324005,
            0.19104912877082825,
            0.20343774557113647,
            0.21404394507408142
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5761134624481201,
          "mean_weights": [
            0.1940101534128189,
            0.18250694870948792,
            0.16849923133850098,
            0.18937936425209045,
            0.26560431718826294
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6027262210845947,
          "mean_weights": [
            0.20452985167503357,
            0.18702910840511322,
            0.17977532744407654,
            0.20095866918563843,
            0.22770702838897705
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.577937126159668,
          "mean_weights": [
            0.1962222307920456,
            0.1818992793560028,
            0.17167484760284424,
            0.18149182200431824,
            0.2687118649482727
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5977532863616943,
          "mean_weights": [
            0.2153339385986328,
            0.18402469158172607,
            0.17019470036029816,
            0.1919252574443817,
            0.23852142691612244
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.578121542930603,
          "mean_weights": [
            0.20422281324863434,
            0.1835813820362091,
            0.16642534732818604,
            0.17722904682159424,
            0.26854145526885986
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.782798409461975,
          "mean_weights": [
            0.15961389243602753,
            0.16475999355316162,
            0.14714640378952026,
            0.15768738090991974,
            0.1760568618774414,
            0.19473549723625183
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7707297801971436,
          "mean_weights": [
            0.16306114196777344,
            0.1578725278377533,
            0.139655202627182,
            0.14299748837947845,
            0.18966570496559143,
            0.20674791932106018
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.781312108039856,
          "mean_weights": [
            0.15919369459152222,
            0.16844786703586578,
            0.14742718636989594,
            0.1539241373538971,
            0.1669306755065918,
            0.20407646894454956
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7736209630966187,
          "mean_weights": [
            0.15695160627365112,
            0.16567477583885193,
            0.1478763222694397,
            0.14320015907287598,
            0.17375096678733826,
            0.2125461995601654
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.781569480895996,
          "mean_weights": [
            0.15368077158927917,
            0.1725173443555832,
            0.14978237450122833,
            0.15674246847629547,
            0.16688688099384308,
            0.20039016008377075
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7793617248535156,
          "mean_weights": [
            0.1678246557712555,
            0.1683025360107422,
            0.14878368377685547,
            0.14351718127727509,
            0.16739681363105774,
            0.20417512953281403
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7772380113601685,
          "mean_weights": [
            0.1536770462989807,
            0.18459689617156982,
            0.14832255244255066,
            0.15016385912895203,
            0.1652815043926239,
            0.19795814156532288
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.780207633972168,
          "mean_weights": [
            0.16951784491539001,
            0.17200809717178345,
            0.15038040280342102,
            0.14669056236743927,
            0.16062122583389282,
            0.20078186690807343
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.937079668045044,
          "mean_weights": [
            0.13008031249046326,
            0.15163566172122955,
            0.1323973685503006,
            0.13641127943992615,
            0.13806134462356567,
            0.1444631665945053,
            0.16695083677768707
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.939245343208313,
          "mean_weights": [
            0.14692582190036774,
            0.14671875536441803,
            0.13336658477783203,
            0.1307726800441742,
            0.131200909614563,
            0.147438645362854,
            0.16357660293579102
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9312366247177124,
          "mean_weights": [
            0.12334205210208893,
            0.16950437426567078,
            0.13870283961296082,
            0.1300538182258606,
            0.12885476648807526,
            0.14288470149040222,
            0.1666574478149414
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9345850944519043,
          "mean_weights": [
            0.1465124636888504,
            0.153631791472435,
            0.13159149885177612,
            0.13120220601558685,
            0.1253916621208191,
            0.14393125474452972,
            0.1677391082048416
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9284589290618896,
          "mean_weights": [
            0.12412485480308533,
            0.1698783040046692,
            0.13807369768619537,
            0.13034150004386902,
            0.13028821349143982,
            0.1419651210308075,
            0.16532832384109497
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9329564571380615,
          "mean_weights": [
            0.1454196274280548,
            0.1566689908504486,
            0.1286500096321106,
            0.12666445970535278,
            0.12782412767410278,
            0.14222374558448792,
            0.17254900932312012
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9259672164916992,
          "mean_weights": [
            0.12082186341285706,
            0.17791953682899475,
            0.1424643099308014,
            0.13298648595809937,
            0.12929362058639526,
            0.1344166398048401,
            0.16209754347801208
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9201786518096924,
          "mean_weights": [
            0.13756656646728516,
            0.17355048656463623,
            0.14306491613388062,
            0.13006147742271423,
            0.11658851057291031,
            0.139857679605484,
            0.15931037068367004
          ],
          "sources": 7
        },
        {
          "entropy_mean": 2.043663501739502,
          "mean_weights": [
            0.10298450291156769,
            0.1751018464565277,
            0.1414714753627777,
            0.12628036737442017,
            0.11767096072435379,
            0.12069195508956909,
            0.12702946364879608,
            0.08876943588256836
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0607244968414307,
          "mean_weights": [
            0.1220984011888504,
            0.1498527079820633,
            0.12913095951080322,
            0.1179434061050415,
            0.10775719583034515,
            0.12020602822303772,
            0.13844498991966248,
            0.11456632614135742
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0388383865356445,
          "mean_weights": [
            0.1025313287973404,
            0.18260905146598816,
            0.14208021759986877,
            0.12606360018253326,
            0.11357533931732178,
            0.11596501618623734,
            0.12797299027442932,
            0.08920245617628098
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0634398460388184,
          "mean_weights": [
            0.12216806411743164,
            0.15090107917785645,
            0.12766355276107788,
            0.1134132519364357,
            0.10988184064626694,
            0.11335282772779465,
            0.13999530673027039,
            0.12262405455112457
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0386834144592285,
          "mean_weights": [
            0.10029368102550507,
            0.17931804060935974,
            0.14185777306556702,
            0.13114282488822937,
            0.11872631311416626,
            0.11815309524536133,
            0.12660562992095947,
            0.08390261977910995
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0626261234283447,
          "mean_weights": [
            0.11644256860017776,
            0.15398627519607544,
            0.1359611600637436,
            0.12036904692649841,
            0.10964122414588928,
            0.11886263638734818,
            0.13317911326885223,
            0.1115579903125763
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.020051956176758,
          "mean_weights": [
            0.09590239822864532,
            0.1913527548313141,
            0.14906750619411469,
            0.1341754049062729,
            0.11847006529569626,
            0.11603336781263351,
            0.12013453245162964,
            0.07486395537853241
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0452136993408203,
          "mean_weights": [
            0.11420649290084839,
            0.15673109889030457,
            0.13583534955978394,
            0.12758757174015045,
            0.1112617775797844,
            0.13417255878448486,
            0.12519711256027222,
            0.09500804543495178
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.1403071880340576,
          "mean_weights": [
            0.08488726615905762,
            0.1673685610294342,
            0.13726134598255157,
            0.12220048159360886,
            0.10616928339004517,
            0.1037440225481987,
            0.10401669889688492,
            0.06658264249563217,
            0.10776971280574799
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.18704891204834,
          "mean_weights": [
            0.10885682702064514,
            0.12684625387191772,
            0.11236055195331573,
            0.10673345625400543,
            0.10181289166212082,
            0.11441421508789062,
            0.11840946972370148,
            0.10466625541448593,
            0.1059001013636589
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1283018589019775,
          "mean_weights": [
            0.08667369186878204,
            0.1812124401330948,
            0.14178472757339478,
            0.12440058588981628,
            0.11160320788621902,
            0.10330478847026825,
            0.11197598278522491,
            0.06831350177526474,
            0.07073107361793518
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.173035144805908,
          "mean_weights": [
            0.10752685368061066,
            0.13639645278453827,
            0.11839058995246887,
            0.11235916614532471,
            0.10138258337974548,
            0.11953261494636536,
            0.11256720125675201,
            0.09452059864997864,
            0.097323939204216
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1028335094451904,
          "mean_weights": [
            0.08200174570083618,
            0.19083696603775024,
            0.15512439608573914,
            0.1322997808456421,
            0.11209174990653992,
            0.10435720533132553,
            0.1024213582277298,
            0.06072331964969635,
            0.06014348566532135
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.147671937942505,
          "mean_weights": [
            0.09926989674568176,
            0.14984291791915894,
            0.12505242228507996,
            0.13004425168037415,
            0.1122550517320633,
            0.12226949632167816,
            0.10689133405685425,
            0.07532045990228653,
            0.07905416935682297
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.109609603881836,
          "mean_weights": [
            0.08386656641960144,
            0.18956288695335388,
            0.1487923562526703,
            0.12990039587020874,
            0.11159682273864746,
            0.10738210380077362,
            0.10535432398319244,
            0.0630122721195221,
            0.06053228676319122
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.04393457621335983,
        0.018067941069602966,
        0.024121688678860664,
        0.027584951370954514,
        0.030756350606679916,
        0.03231721371412277,
        0.03544071316719055,
        0.036101218312978745,
        0.03840360417962074,
        0.025345133617520332,
        0.026969486847519875,
        0.027166159823536873,
        0.027978016063570976,
        0.028526438400149345,
        0.029923241585493088,
        0.02979987859725952,
        0.03152087330818176,
        0.02248864620923996,
        0.024161184206604958,
        0.023894935846328735,
        0.025710709393024445,
        0.02451697364449501,
        0.02641460672020912,
        0.025183402001857758,
        0.02742084674537182,
        0.020499059930443764,
        0.021717891097068787,
        0.02105838619172573,
        0.0224718376994133,
        0.0216353852301836,
        0.023445574566721916,
        0.022715488448739052,
        0.024849263951182365,
        0.01881815865635872,
        0.01992032676935196,
        0.0193756353110075,
        0.02058842033147812,
        0.02001863345503807,
        0.020979218184947968,
        0.021088071167469025,
        0.02214338816702366,
        0.018417155370116234,
        0.01903437450528145,
        0.019219009205698967,
        0.02008410543203354,
        0.020342160016298294,
        0.021635910496115685,
        0.02150510996580124,
        0.023218298330903053,
        0.02099662832915783,
        0.022257626056671143,
        0.02211105264723301,
        0.024358727037906647,
        0.02328667603433132,
        0.025900650769472122,
        0.024160878732800484,
        0.027197737246751785,
        0.02260657213628292,
        0.02823900803923607,
        0.024989914149045944,
        0.02913021855056286,
        0.02536177635192871,
        0.029048049822449684,
        0.027097374200820923
      ],
      "loss_nats": 2.121690273284912,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 2.1622862815856934,
        "mean_weights": [
          0.12356589734554291,
          0.09235234558582306,
          0.08736804127693176,
          0.08735492080450058,
          0.08889827132225037,
          0.10504580289125443,
          0.14004790782928467,
          0.14956718683242798,
          0.12579965591430664
        ],
        "sources": 9
      },
      "step": 500,
      "stream_state_rms_by_sublayer": [
        0.010549778118729591,
        0.021530626341700554,
        0.02910016104578972,
        0.03857670724391937,
        0.04255657270550728,
        0.04954109713435173,
        0.05177471786737442,
        0.057694368064403534,
        0.01913420483469963,
        0.02140580676496029,
        0.02246284671127796,
        0.025963041931390762,
        0.029843268916010857,
        0.03339661285281181,
        0.036163609474897385,
        0.040483180433511734,
        0.019129201769828796,
        0.02151828072965145,
        0.0314205139875412,
        0.035047177225351334,
        0.03302976116538048,
        0.03725159168243408,
        0.03871368616819382,
        0.043239567428827286,
        0.018231205642223358,
        0.02124657854437828,
        0.02580643631517887,
        0.03028978779911995,
        0.0349617674946785,
        0.03979094699025154,
        0.04659247398376465,
        0.05222286656498909,
        0.03611915931105614,
        0.03916332125663757,
        0.03664228320121765,
        0.04088718071579933,
        0.04177439957857132,
        0.04756740853190422,
        0.04776441678404808,
        0.05542457476258278,
        0.013802599161863327,
        0.020541559904813766,
        0.02693362347781658,
        0.03680417314171791,
        0.03932841867208481,
        0.05126549303531647,
        0.0514267161488533,
        0.06891486048698425,
        0.015529165975749493,
        0.02905990369617939,
        0.03558400645852089,
        0.05358703061938286,
        0.05707564577460289,
        0.07880406081676483,
        0.08332383632659912,
        0.10752817988395691,
        0.009738311171531677,
        0.029148343950510025,
        0.03187238425016403,
        0.054091889411211014,
        0.05786185711622238,
        0.08628532290458679,
        0.09122034162282944,
        0.11884573101997375
      ]
    },
    {
      "activation_grad_rms_by_block": [
        6.054548066458665e-05,
        7.096362969605252e-05,
        6.466427294071764e-05,
        5.866752690053545e-05,
        5.6290406064363196e-05,
        5.253175913821906e-05,
        4.870586417382583e-05,
        4.417003583512269e-05,
        3.822600410785526e-05,
        3.59821024176199e-05,
        3.430073047638871e-05,
        3.306051803519949e-05,
        3.607416874729097e-05,
        3.430179276620038e-05,
        3.275281051173806e-05,
        3.160809865221381e-05,
        3.903685137629509e-05,
        3.708450458361767e-05,
        3.9304726669797674e-05,
        4.4666798203252256e-05,
        0.00014839931100141257,
        0.00012039772263960913,
        9.753467020345852e-05,
        7.931287109386176e-05,
        7.865376392146572e-05,
        6.111936090746894e-05,
        4.65054290543776e-05,
        4.1926850826712325e-05,
        2.5328776246169582e-05,
        2.2645597709924914e-05,
        2.1881067368667573e-05,
        2.167661841667723e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 5.706737192667788e-05,
        "first_to_last_ratio": 1.4278557447018398,
        "imbalance_abs_log_ratio": 0.3561738396982594,
        "last_quartile_mean": 3.996718305643299e-05,
        "mean": 5.182250379220932e-05,
        "normalized": [
          1.1683241108409874,
          1.3693593420455452,
          1.2478029467661282,
          1.1320859203517522,
          1.0862154844943164,
          1.0136862423485675,
          0.9398593392770029,
          0.8523331101914637,
          0.7376332926932397,
          0.6943335382230071,
          0.6618887156422048,
          0.6379567874173152,
          0.6961101086882286,
          0.6619092142621851,
          0.6320190672968202,
          0.6099299790483219,
          0.7532799174046017,
          0.7156061917099562,
          0.758449009476584,
          0.8619189528616946,
          2.863607509132394,
          2.3232710469251585,
          1.8820910428129736,
          1.530471615417879,
          1.517753064129064,
          1.1793980690808934,
          0.8973983434078874,
          0.8090471852696426,
          0.4887601793176453,
          0.43698385938136236,
          0.4222309955613731,
          0.41828581852380436
        ],
        "population_cv": 0.5384894543542103
      },
      "activation_output_rms_by_block": [
        0.09451264888048172,
        0.2031717449426651,
        0.22856928408145905,
        0.2430499792098999,
        0.08243431895971298,
        0.09709372371435165,
        0.11282631754875183,
        0.13645486533641815,
        0.05442482605576515,
        0.0885990634560585,
        0.12215005606412888,
        0.15440259873867035,
        0.060873307287693024,
        0.09471380710601807,
        0.1260664463043213,
        0.163865864276886,
        0.08853588253259659,
        0.12131853401660919,
        0.15660560131072998,
        0.2097783088684082,
        0.04455159604549408,
        0.07411240041255951,
        0.10008715838193893,
        0.13061410188674927,
        0.04960336163640022,
        0.07874798029661179,
        0.11358420550823212,
        0.1580832600593567,
        0.06955241411924362,
        0.13302887976169586,
        0.22457914054393768,
        0.32859006524086
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.14976411033421755,
        "first_to_last_ratio": 1.036636701844262,
        "imbalance_abs_log_ratio": 0.035981532131895505,
        "last_quartile_mean": 0.14447116339579225,
        "mean": 0.12951817945577204,
        "normalized": [
          0.7297249643070838,
          1.5686735699681784,
          1.7647660354854744,
          1.8765703797812938,
          0.636469098825333,
          0.749653246535227,
          0.8711234054002422,
          1.0535576234146717,
          0.42020993720306404,
          0.6840666215997374,
          0.9431112804194469,
          1.192130706187048,
          0.4699981697046636,
          0.7312780916470571,
          0.9733494312076132,
          1.2651958587237788,
          0.6835788065012903,
          0.9366911620158862,
          1.2091399212742004,
          1.6196823469097898,
          0.34397948019882096,
          0.5722161994862464,
          0.7727653276358534,
          1.008461533628578,
          0.3829837776042769,
          0.608007158744095,
          0.8769750006177236,
          1.2205488119398642,
          0.5370088925855727,
          1.0271058497013745,
          1.7339584411053826,
          2.5370188696411313
        ],
        "population_cv": 0.4929365839545065
      },
      "bits_per_byte": 2.073383110809672,
      "branch_output_rms_by_sublayer": [
        0.04804428666830063,
        0.07808538526296616,
        0.12356939166784286,
        0.03881331905722618,
        0.05923242121934891,
        0.01480295043438673,
        0.04508936405181885,
        0.02500341460108757,
        0.0799286738038063,
        0.015180142596364021,
        0.03290615975856781,
        0.013721407391130924,
        0.03170505166053772,
        0.016902483999729156,
        0.03961387276649475,
        0.03325798362493515,
        0.04508088901638985,
        0.026172807440161705,
        0.0312613770365715,
        0.027887089177966118,
        0.029994890093803406,
        0.03240044787526131,
        0.03058313950896263,
        0.03682815283536911,
        0.0497574545443058,
        0.029301028698682785,
        0.03759258985519409,
        0.0338764488697052,
        0.030690157786011696,
        0.0393521711230278,
        0.03000541403889656,
        0.046694882214069366,
        0.079553984105587,
        0.029503758996725082,
        0.0414140485227108,
        0.03615153953433037,
        0.04122982919216156,
        0.030687781050801277,
        0.05830265209078789,
        0.05279085040092468,
        0.03440643846988678,
        0.024021649733185768,
        0.033897366374731064,
        0.03336174041032791,
        0.02957344427704811,
        0.04255227372050285,
        0.03406442329287529,
        0.0520954355597496,
        0.025225287303328514,
        0.03984410688281059,
        0.01915411651134491,
        0.04249191656708717,
        0.014526215381920338,
        0.05309459939599037,
        0.011566666886210442,
        0.06231515854597092,
        0.0242468249052763,
        0.06388300657272339,
        0.013597902841866016,
        0.0749494656920433,
        0.020146220922470093,
        0.10156872868537903,
        0.02710210345685482,
        0.11044161766767502
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0002879944983597963,
        0.00022853159560210997,
        0.00014071406888316784,
        0.00014318671869519129,
        0.00013886536811003923,
        0.00010286844600842235,
        9.677924556041764e-05,
        0.0001337288949114584,
        0.00010758657043361882,
        9.13787469069405e-05,
        9.373485859236601e-05,
        9.381017142422257e-05,
        0.00010148426099734976,
        9.552468088000852e-05,
        0.00010200246393672037,
        0.00010753255944705701,
        0.00013288008273706816,
        0.00012966970691820486,
        0.0001395588859415414,
        0.00019785772756123382,
        0.0004949688435464942,
        0.0005919211037074493,
        0.000516172452953512,
        0.0004666614668327092,
        0.0003441119880144902,
        0.00027788329882105144,
        0.00022798541204965527,
        0.00021216050695355764,
        0.00012431994828037904,
        0.00011340006302052245,
        0.00011536177905162644,
        0.00011797314476595472
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00015908360451632537,
        "first_to_last_ratio": 0.8300756844691923,
        "imbalance_abs_log_ratio": 0.18623839624123317,
        "last_quartile_mean": 0.00019164951761965464,
        "mean": 0.00019589404874701053,
        "normalized": [
          1.4701544033720488,
          1.1666081591751138,
          0.7183172219024098,
          0.730939605419618,
          0.7088799736299207,
          0.5251228746681984,
          0.4940387223575344,
          0.682659303673712,
          0.5492079576779928,
          0.4664702551783621,
          0.4784977348312449,
          0.4788821917983569,
          0.5180568866000248,
          0.4876344201930039,
          0.5207022091235275,
          0.5489322423772612,
          0.6783262870260932,
          0.661937959563377,
          0.7124202436684344,
          1.01002418821186,
          2.526717104028653,
          3.0216390313719645,
          2.6349572958192744,
          2.3822135987162336,
          1.7566229817369148,
          1.4185387488719823,
          1.1638200012093756,
          1.0830370208313709,
          0.6346285100316312,
          0.5788846764149236,
          0.5888988450109153,
          0.6022293455086652
        ],
        "population_cv": 0.7015245062301059
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.4738388657569885,
          "mean_weights": [
            0.8067666292190552,
            0.19323334097862244
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6344536542892456,
          "mean_weights": [
            0.4895974397659302,
            0.5104025602340698
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.41773349046707153,
          "mean_weights": [
            0.8461704254150391,
            0.15382957458496094
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6868985891342163,
          "mean_weights": [
            0.5008196830749512,
            0.49918028712272644
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.5841774344444275,
          "mean_weights": [
            0.684928297996521,
            0.315071702003479
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6896515488624573,
          "mean_weights": [
            0.5092520117759705,
            0.49074798822402954
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.5611414909362793,
          "mean_weights": [
            0.7280503511428833,
            0.2719496488571167
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6817618608474731,
          "mean_weights": [
            0.5313050746917725,
            0.46869492530822754
          ],
          "sources": 2
        },
        {
          "entropy_mean": 1.0218309164047241,
          "mean_weights": [
            0.4974328279495239,
            0.24236315488815308,
            0.2602040767669678
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0932960510253906,
          "mean_weights": [
            0.3533782958984375,
            0.29999595880508423,
            0.34662577509880066
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0456881523132324,
          "mean_weights": [
            0.4338536858558655,
            0.23105540871620178,
            0.3350909352302551
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0891339778900146,
          "mean_weights": [
            0.33998236060142517,
            0.2821658253669739,
            0.37785181403160095
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.038641333580017,
          "mean_weights": [
            0.34165239334106445,
            0.23904743790626526,
            0.4193001389503479
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0754427909851074,
          "mean_weights": [
            0.3781457543373108,
            0.24021697044372559,
            0.3816372752189636
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0433809757232666,
          "mean_weights": [
            0.37802231311798096,
            0.21331989765167236,
            0.4086577892303467
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0922105312347412,
          "mean_weights": [
            0.3460429906845093,
            0.3074926435947418,
            0.3464643955230713
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.3404325246810913,
          "mean_weights": [
            0.2880150079727173,
            0.16260021924972534,
            0.2385387420654297,
            0.3108460605144501
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3765960931777954,
          "mean_weights": [
            0.28703436255455017,
            0.2552259564399719,
            0.2101576328277588,
            0.24758204817771912
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3274900913238525,
          "mean_weights": [
            0.33539247512817383,
            0.1987970769405365,
            0.20543015003204346,
            0.2603802978992462
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3747133016586304,
          "mean_weights": [
            0.27988845109939575,
            0.23608103394508362,
            0.20066317915916443,
            0.2833673357963562
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3239198923110962,
          "mean_weights": [
            0.3361120820045471,
            0.20296622812747955,
            0.1885247528553009,
            0.2723969519138336
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3692121505737305,
          "mean_weights": [
            0.28167375922203064,
            0.21946553885936737,
            0.20140981674194336,
            0.2974509000778198
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3342361450195312,
          "mean_weights": [
            0.36468517780303955,
            0.20593920350074768,
            0.18943950533866882,
            0.23993611335754395
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3645737171173096,
          "mean_weights": [
            0.2597482204437256,
            0.22607669234275818,
            0.209133118391037,
            0.30504196882247925
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.5910131931304932,
          "mean_weights": [
            0.2617476284503937,
            0.18299485743045807,
            0.17935843765735626,
            0.17898105084896088,
            0.19691801071166992
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5884685516357422,
          "mean_weights": [
            0.2046908289194107,
            0.17294223606586456,
            0.15044087171554565,
            0.20923088490962982,
            0.26269516348838806
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.599259853363037,
          "mean_weights": [
            0.21657392382621765,
            0.20580387115478516,
            0.19366897642612457,
            0.1731838881969452,
            0.21076934039592743
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5786192417144775,
          "mean_weights": [
            0.20599734783172607,
            0.20593789219856262,
            0.13473063707351685,
            0.17552703619003296,
            0.2778070569038391
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.580681324005127,
          "mean_weights": [
            0.21277281641960144,
            0.18522506952285767,
            0.18489867448806763,
            0.17205308377742767,
            0.2450503408908844
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5705549716949463,
          "mean_weights": [
            0.2125053107738495,
            0.20224159955978394,
            0.12915469706058502,
            0.1714920997619629,
            0.28460633754730225
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5599372386932373,
          "mean_weights": [
            0.2665032148361206,
            0.181569442152977,
            0.15844659507274628,
            0.14596089720726013,
            0.2475198209285736
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.562828540802002,
          "mean_weights": [
            0.21806953847408295,
            0.22324788570404053,
            0.11591021716594696,
            0.17848318815231323,
            0.26428914070129395
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.761409044265747,
          "mean_weights": [
            0.1612042486667633,
            0.1651904582977295,
            0.1752963364124298,
            0.14996090531349182,
            0.2072838991880417,
            0.14106415212154388
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7366840839385986,
          "mean_weights": [
            0.1851401925086975,
            0.1663871705532074,
            0.09063514322042465,
            0.11971023678779602,
            0.18503433465957642,
            0.2530929446220398
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.777672290802002,
          "mean_weights": [
            0.15483248233795166,
            0.16576308012008667,
            0.16957050561904907,
            0.14225167036056519,
            0.16980668902397156,
            0.19777557253837585
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7414222955703735,
          "mean_weights": [
            0.18020270764827728,
            0.1900298297405243,
            0.09944288432598114,
            0.11960913985967636,
            0.15577411651611328,
            0.25494131445884705
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.733978509902954,
          "mean_weights": [
            0.15204599499702454,
            0.14166373014450073,
            0.1616208702325821,
            0.1256796270608902,
            0.1571422517299652,
            0.26184749603271484
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7344707250595093,
          "mean_weights": [
            0.1829337775707245,
            0.18416841328144073,
            0.09014187753200531,
            0.12335793673992157,
            0.15779981017112732,
            0.2615981996059418
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.735896348953247,
          "mean_weights": [
            0.17276549339294434,
            0.15101772546768188,
            0.13220663368701935,
            0.11932596564292908,
            0.15458261966705322,
            0.27010154724121094
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7644665241241455,
          "mean_weights": [
            0.19160008430480957,
            0.1915181279182434,
            0.11520625650882721,
            0.1341419219970703,
            0.1580837219953537,
            0.2094498574733734
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.6835086345672607,
          "mean_weights": [
            0.10065847635269165,
            0.09339895844459534,
            0.09682720899581909,
            0.08155890554189682,
            0.09126688539981842,
            0.10349369794130325,
            0.4327958822250366
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.7937740087509155,
          "mean_weights": [
            0.13723835349082947,
            0.12937892973423004,
            0.07450413703918457,
            0.08537043631076813,
            0.09880411624908447,
            0.12944120168685913,
            0.3452628552913666
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.5790722370147705,
          "mean_weights": [
            0.09500400722026825,
            0.08892714232206345,
            0.09081116318702698,
            0.07538282871246338,
            0.07685283571481705,
            0.08106325566768646,
            0.49195876717567444
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.8034920692443848,
          "mean_weights": [
            0.142653226852417,
            0.13419847190380096,
            0.07437048852443695,
            0.09070360660552979,
            0.1013825386762619,
            0.12073953449726105,
            0.33595213294029236
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.5278494358062744,
          "mean_weights": [
            0.10045769065618515,
            0.08844951540231705,
            0.08561251312494278,
            0.06598107516765594,
            0.06767018139362335,
            0.07361933588981628,
            0.51820969581604
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.8435695171356201,
          "mean_weights": [
            0.14839640259742737,
            0.1482572704553604,
            0.08113027364015579,
            0.09534022957086563,
            0.10612697899341583,
            0.12550383806228638,
            0.2952450215816498
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.6421003341674805,
          "mean_weights": [
            0.11422763019800186,
            0.10543526709079742,
            0.09285669028759003,
            0.07746599614620209,
            0.07571054995059967,
            0.08126918226480484,
            0.4530346691608429
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.8733552694320679,
          "mean_weights": [
            0.14798465371131897,
            0.16052685678005219,
            0.09668852388858795,
            0.10477185249328613,
            0.10857058316469193,
            0.11694246530532837,
            0.26451510190963745
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.784506916999817,
          "mean_weights": [
            0.08589775860309601,
            0.08786045759916306,
            0.07714233547449112,
            0.06314891576766968,
            0.05684572085738182,
            0.056072384119033813,
            0.29875272512435913,
            0.2742796838283539
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.9675533771514893,
          "mean_weights": [
            0.11549945175647736,
            0.12592051923274994,
            0.07530814409255981,
            0.08277833461761475,
            0.08296641707420349,
            0.09016790241003036,
            0.17876628041267395,
            0.24859298765659332
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.7119876146316528,
          "mean_weights": [
            0.08804735541343689,
            0.07527872174978256,
            0.05666808784008026,
            0.05041901767253876,
            0.05142981559038162,
            0.05313319340348244,
            0.2591783106327057,
            0.3658455014228821
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.9094492197036743,
          "mean_weights": [
            0.10337945818901062,
            0.10583682358264923,
            0.08642245084047318,
            0.08043928444385529,
            0.0717030018568039,
            0.0645938515663147,
            0.21096080541610718,
            0.2766643166542053
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.7860023975372314,
          "mean_weights": [
            0.09247160702943802,
            0.0784597396850586,
            0.07236532866954803,
            0.05986309051513672,
            0.05389959365129471,
            0.06059400737285614,
            0.2579019069671631,
            0.3244447112083435
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.9232951402664185,
          "mean_weights": [
            0.1160542219877243,
            0.11346358060836792,
            0.08816123008728027,
            0.08138670027256012,
            0.07161372900009155,
            0.06013647839426994,
            0.21822679042816162,
            0.2509572505950928
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.8294003009796143,
          "mean_weights": [
            0.08831655979156494,
            0.10072161257266998,
            0.0818098857998848,
            0.07514279335737228,
            0.06651484966278076,
            0.06491033732891083,
            0.31431835889816284,
            0.20826560258865356
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.9393219947814941,
          "mean_weights": [
            0.11041727662086487,
            0.12080886960029602,
            0.09995828568935394,
            0.09934850037097931,
            0.08031868189573288,
            0.075287364423275,
            0.2604207992553711,
            0.15344023704528809
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.9925593137741089,
          "mean_weights": [
            0.08698172867298126,
            0.08857265114784241,
            0.07299254834651947,
            0.060478877276182175,
            0.054115522652864456,
            0.056821923702955246,
            0.2503892779350281,
            0.14502599835395813,
            0.18462148308753967
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.068819999694824,
          "mean_weights": [
            0.09429103136062622,
            0.09028280526399612,
            0.09188265353441238,
            0.0735517293214798,
            0.06746295094490051,
            0.060166649520397186,
            0.21415996551513672,
            0.15325874090194702,
            0.15494348108768463
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.0139355659484863,
          "mean_weights": [
            0.0956040769815445,
            0.09831400215625763,
            0.0714857280254364,
            0.0677608773112297,
            0.05654233694076538,
            0.05388312786817551,
            0.24272149801254272,
            0.17641445994377136,
            0.137273907661438
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.0349321365356445,
          "mean_weights": [
            0.10256819427013397,
            0.0938093364238739,
            0.0912778377532959,
            0.08394120633602142,
            0.06161238253116608,
            0.053358763456344604,
            0.2261163890361786,
            0.15197426080703735,
            0.13534162938594818
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.046657085418701,
          "mean_weights": [
            0.08380690217018127,
            0.1217331811785698,
            0.10242430865764618,
            0.09225839376449585,
            0.06788390129804611,
            0.06336672604084015,
            0.2572976052761078,
            0.11757771670818329,
            0.09365126490592957
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.094015121459961,
          "mean_weights": [
            0.10284869372844696,
            0.13057856261730194,
            0.10152992606163025,
            0.10079468786716461,
            0.0796234980225563,
            0.07438632845878601,
            0.2018478661775589,
            0.1111527606844902,
            0.09723767638206482
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.057201385498047,
          "mean_weights": [
            0.09265108406543732,
            0.11481083929538727,
            0.09007447212934494,
            0.08496075868606567,
            0.06770835071802139,
            0.05925413966178894,
            0.24551761150360107,
            0.15022093057632446,
            0.09480181336402893
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.05595625191926956,
        0.047033343464136124,
        0.0632273480296135,
        0.060984931886196136,
        0.11160287261009216,
        0.09079429507255554,
        0.12218338996171951,
        0.08531255275011063,
        0.12365243583917618,
        0.0786692351102829,
        0.0926642045378685,
        0.08076046407222748,
        0.093480683863163,
        0.08904207497835159,
        0.08749273419380188,
        0.08403915911912918,
        0.09927791357040405,
        0.06012389436364174,
        0.07820390164852142,
        0.06841155141592026,
        0.07669360190629959,
        0.07131828367710114,
        0.07840896397829056,
        0.07358616590499878,
        0.08920575678348541,
        0.06597626209259033,
        0.06850729882717133,
        0.07235056161880493,
        0.07440610229969025,
        0.07261744141578674,
        0.07867594808340073,
        0.07352335751056671,
        0.09058429300785065,
        0.07520031183958054,
        0.07234266400337219,
        0.07378771901130676,
        0.07771008461713791,
        0.07715436071157455,
        0.08416424691677094,
        0.08649503439664841,
        0.08947562426328659,
        0.04556623473763466,
        0.056279782205820084,
        0.04546353965997696,
        0.057985786348581314,
        0.05167657509446144,
        0.06238175556063652,
        0.058827146887779236,
        0.06515856087207794,
        0.04798740893602371,
        0.05041429027915001,
        0.046526361256837845,
        0.05083387345075607,
        0.051699329167604446,
        0.05600840970873833,
        0.059578344225883484,
        0.06128782033920288,
        0.05090610682964325,
        0.056301049888134,
        0.05821637064218521,
        0.06424970924854279,
        0.06188955530524254,
        0.06844019144773483,
        0.06796324998140335
      ],
      "loss_nats": 1.4371596574783325,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 1.8435449600219727,
        "mean_weights": [
          0.1112452894449234,
          0.06898629665374756,
          0.05835210159420967,
          0.042265649884939194,
          0.041034020483493805,
          0.04826357960700989,
          0.10493715852499008,
          0.31774088740348816,
          0.20717504620552063
        ],
        "sources": 9
      },
      "step": 2000,
      "stream_state_rms_by_sublayer": [
        0.04804428666830063,
        0.09451264888048172,
        0.18500657379627228,
        0.2031717449426651,
        0.22299833595752716,
        0.22856928408145905,
        0.2331935465335846,
        0.2430499792098999,
        0.0799286738038063,
        0.08243431895971298,
        0.09402555972337723,
        0.09709372371435165,
        0.10843002796173096,
        0.11282631754875183,
        0.12457633018493652,
        0.13645486533641815,
        0.04508088901638985,
        0.05442482605576515,
        0.0760042667388916,
        0.0885990634560585,
        0.10498681664466858,
        0.12215005606412888,
        0.13159531354904175,
        0.15440259873867035,
        0.0497574545443058,
        0.060873307287693024,
        0.07735268771648407,
        0.09471380710601807,
        0.10224150121212006,
        0.1260664463043213,
        0.13362281024456024,
        0.163865864276886,
        0.079553984105587,
        0.08853588253259659,
        0.10536550730466843,
        0.12131853401660919,
        0.14260481297969818,
        0.15660560131072998,
        0.19080305099487305,
        0.2097783088684082,
        0.03440643846988678,
        0.04455159604549408,
        0.060867179185152054,
        0.07411240041255951,
        0.08364810794591904,
        0.10008715838193893,
        0.11119203269481659,
        0.13061410188674927,
        0.025225287303328514,
        0.04960336163640022,
        0.05530283972620964,
        0.07874798029661179,
        0.08037404716014862,
        0.11358420550823212,
        0.11502843350172043,
        0.1580832600593567,
        0.0242468249052763,
        0.06955241411924362,
        0.07164724916219711,
        0.13302887976169586,
        0.1371813863515854,
        0.22457914054393768,
        0.23144805431365967,
        0.32859006524086
      ]
    },
    {
      "activation_grad_rms_by_block": [
        4.638188693206757e-05,
        5.534173033083789e-05,
        4.9559435865376145e-05,
        4.482309304876253e-05,
        4.6912617108318955e-05,
        4.315303158364259e-05,
        3.9931001083459705e-05,
        3.624077362474054e-05,
        2.8622678655665368e-05,
        2.680278521438595e-05,
        2.541001776990015e-05,
        2.4583727281424217e-05,
        3.3810731110861525e-05,
        3.128740718238987e-05,
        2.9118271413608454e-05,
        2.7763266189140268e-05,
        3.4525284718256444e-05,
        3.251996167819016e-05,
        3.314791320008226e-05,
        3.811598799075e-05,
        0.00014694883429910988,
        0.00011575810640351847,
        9.27479995880276e-05,
        7.475013262592256e-05,
        9.14012489374727e-05,
        7.025576633168384e-05,
        4.882529901806265e-05,
        4.170324973529205e-05,
        3.074307824135758e-05,
        2.38938500842778e-05,
        2.2532458388013765e-05,
        2.1951122107566334e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 4.529294619715074e-05,
        "first_to_last_ratio": 1.0314184626645753,
        "imbalance_abs_log_ratio": 0.03093500306067748,
        "last_quartile_mean": 4.391325910546584e-05,
        "mean": 4.717383586694268e-05,
        "normalized": [
          0.9832121149293677,
          1.1731445898725168,
          1.0505704052806353,
          0.9501685038967237,
          0.9944626347672784,
          0.9147662213723499,
          0.8464650022543867,
          0.7682388544141363,
          0.6067490194436969,
          0.5681705700165066,
          0.53864641920515,
          0.5211305553096096,
          0.7167263481864653,
          0.6632364446817158,
          0.6172546895643319,
          0.588531029519173,
          0.7318735922946267,
          0.6893643707481226,
          0.7026758072754231,
          0.8079900073900919,
          3.115049509935762,
          2.4538624912763685,
          1.9660898437352061,
          1.5845676157597377,
          1.937541165727475,
          1.4892951790023068,
          1.0350080319051866,
          0.8840334683174624,
          0.6516976556256888,
          0.50650640646803,
          0.47764736477161285,
          0.4653240870528551
        ],
        "population_cv": 0.6027314599912318
      },
      "activation_output_rms_by_block": [
        0.16101159155368805,
        0.3334391713142395,
        0.3570481836795807,
        0.37708213925361633,
        0.11129721254110336,
        0.12943366169929504,
        0.1512180119752884,
        0.1945808082818985,
        0.07549218833446503,
        0.11765842139720917,
        0.16134388744831085,
        0.21493762731552124,
        0.08329613506793976,
        0.12312135100364685,
        0.16809259355068207,
        0.23101076483726501,
        0.11538279056549072,
        0.16158254444599152,
        0.20080390572547913,
        0.27125677466392517,
        0.0543946772813797,
        0.09335283935070038,
        0.12897421419620514,
        0.17114612460136414,
        0.06173322722315788,
        0.09693212062120438,
        0.13771499693393707,
        0.18387214839458466,
        0.07983565330505371,
        0.15526357293128967,
        0.26127612590789795,
        0.39550676941871643
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.22688884753733873,
        "first_to_last_ratio": 1.3228372499356764,
        "imbalance_abs_log_ratio": 0.2797788616441042,
        "last_quartile_mean": 0.17151682684198022,
        "mean": 0.17372163233812898,
        "normalized": [
          0.9268367409782202,
          1.919387740182172,
          2.055289136269615,
          2.170611306021289,
          0.6406640960204442,
          0.7450635822219737,
          0.8704616111421293,
          1.1200724150644035,
          0.4345583639666029,
          0.677281348405711,
          0.9287495476341928,
          1.2372530952113672,
          0.4794804996179952,
          0.7087278040538177,
          0.9675973641757485,
          1.32977546738468,
          0.6641820538556276,
          0.930123337383164,
          1.1558946518222766,
          1.5614450026347633,
          0.3131140120506511,
          0.5373702635317168,
          0.742418848247821,
          0.9851745133746388,
          0.3553571676266622,
          0.5579738073870804,
          0.7927337262517246,
          1.0584297759573136,
          0.4595608055861626,
          0.8937492173058054,
          1.5039930398498347,
          2.276669658784396
        ],
        "population_cv": 0.5173518438224235
      },
      "bits_per_byte": 1.9433676664104056,
      "branch_output_rms_by_sublayer": [
        0.08528144657611847,
        0.13266965746879578,
        0.2063225358724594,
        0.045500293374061584,
        0.07479097694158554,
        0.018729008734226227,
        0.053203217685222626,
        0.03815881907939911,
        0.1070452407002449,
        0.02209169790148735,
        0.03943018242716789,
        0.01640710048377514,
        0.041509829461574554,
        0.02340921014547348,
        0.06257687509059906,
        0.05481670796871185,
        0.061002109199762344,
        0.037687551230192184,
        0.04113767296075821,
        0.04149388521909714,
        0.040399398654699326,
        0.048693906515836716,
        0.03907898813486099,
        0.06190986558794975,
        0.06714949756860733,
        0.04065176844596863,
        0.04437125846743584,
        0.049332331866025925,
        0.0412614606320858,
        0.0589165985584259,
        0.046277787536382675,
        0.08173932880163193,
        0.10073095560073853,
        0.043543290346860886,
        0.05776162073016167,
        0.05228494107723236,
        0.05185067281126976,
        0.04403071105480194,
        0.07827401161193848,
        0.0687810555100441,
        0.04170985147356987,
        0.030229199677705765,
        0.041845910251140594,
        0.04699210822582245,
        0.038240060210227966,
        0.06082639843225479,
        0.046402644366025925,
        0.07667754590511322,
        0.030182205140590668,
        0.050203632563352585,
        0.025695007294416428,
        0.05539015680551529,
        0.024306239560246468,
        0.06946423649787903,
        0.017157377675175667,
        0.07630576938390732,
        0.030125634744763374,
        0.07239200174808502,
        0.01829664409160614,
        0.09628339111804962,
        0.024464696645736694,
        0.12468792498111725,
        0.032671861350536346,
        0.15056027472019196
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0002458031044227025,
        0.00017276993155813553,
        0.00010939491852011586,
        0.000125258738441463,
        0.00011215544691093282,
        7.31087164016768e-05,
        9.049562402296399e-05,
        0.00013630654451635742,
        9.25970647966133e-05,
        7.580293890923953e-05,
        7.957502622146584e-05,
        8.776523713070475e-05,
        0.00010318467043462154,
        0.00010632361913360551,
        0.00011315585331317908,
        0.00012408116098017353,
        0.00012849590289080713,
        0.00014012462333747852,
        0.00013373629199672077,
        0.00019202792118308513,
        0.0005640243510171648,
        0.0006705621295490249,
        0.000579819507581129,
        0.0005486985299520294,
        0.0004559472160958734,
        0.00038131537611027277,
        0.00030455316817913544,
        0.000254991251417659,
        0.00017923215935365346,
        0.0001463414879582755,
        0.00013146832622599984,
        0.0001387805380572583
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.0001331616280992935,
        "first_to_last_ratio": 0.5346167023449758,
        "imbalance_abs_log_ratio": 0.6262052330699333,
        "last_quartile_mean": 0.000249078690424766,
        "mean": 0.00021243429301935995,
        "normalized": [
          1.1570782707870122,
          0.8132864477883067,
          0.5149588466403882,
          0.5896352074852985,
          0.5279535865742339,
          0.34414743195447317,
          0.4259934812630124,
          0.6416409637964399,
          0.4358856730733892,
          0.3568300476907056,
          0.37458653727914765,
          0.4131406275478619,
          0.4857251104237578,
          0.5005012026185237,
          0.5326628375526298,
          0.584091952464876,
          0.6048736344046712,
          0.6596139509580424,
          0.6295419166835406,
          0.9039403123373536,
          2.655053207279312,
          3.156562530551101,
          2.7294063464992813,
          2.5829093888434693,
          2.146297613325176,
          1.7949803239890372,
          1.433634672869791,
          1.200329983419488,
          0.8437063376453977,
          0.6888788334421074,
          0.6188658354422025,
          0.6532868873699723
        ],
        "population_cv": 0.7868568919954686
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.24809812009334564,
          "mean_weights": [
            0.9288002252578735,
            0.07119979709386826
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6067080497741699,
          "mean_weights": [
            0.5604246854782104,
            0.43957534432411194
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.3847596049308777,
          "mean_weights": [
            0.8648147583007812,
            0.13518527150154114
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6843481659889221,
          "mean_weights": [
            0.49438929557800293,
            0.5056107044219971
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.5728601813316345,
          "mean_weights": [
            0.6954652070999146,
            0.3045347332954407
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.686518132686615,
          "mean_weights": [
            0.5263072848320007,
            0.4736926555633545
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.5044870376586914,
          "mean_weights": [
            0.777843713760376,
            0.2221563160419464
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6724032163619995,
          "mean_weights": [
            0.5694063901901245,
            0.4305936098098755
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.9718273282051086,
          "mean_weights": [
            0.5544795989990234,
            0.20597517490386963,
            0.23954525589942932
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0856361389160156,
          "mean_weights": [
            0.3703838884830475,
            0.26754841208457947,
            0.36206769943237305
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.022249460220337,
          "mean_weights": [
            0.46441033482551575,
            0.21309645473957062,
            0.32249322533607483
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0742855072021484,
          "mean_weights": [
            0.38770636916160583,
            0.24731171131134033,
            0.36498188972473145
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0207322835922241,
          "mean_weights": [
            0.3404029607772827,
            0.22699853777885437,
            0.4325985312461853
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0628657341003418,
          "mean_weights": [
            0.4176652133464813,
            0.22035521268844604,
            0.361979603767395
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.029329538345337,
          "mean_weights": [
            0.42153143882751465,
            0.19259268045425415,
            0.3858758807182312
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0849454402923584,
          "mean_weights": [
            0.37970077991485596,
            0.2838303744792938,
            0.3364688754081726
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.3144235610961914,
          "mean_weights": [
            0.3205832540988922,
            0.13048580288887024,
            0.2396528720855713,
            0.30927807092666626
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3647149801254272,
          "mean_weights": [
            0.3188267648220062,
            0.2379944771528244,
            0.19448311626911163,
            0.24869562685489655
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.2778496742248535,
          "mean_weights": [
            0.37896889448165894,
            0.16353651881217957,
            0.1984466314315796,
            0.2590479254722595
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3567583560943604,
          "mean_weights": [
            0.29770219326019287,
            0.21492166817188263,
            0.17857596278190613,
            0.3088001608848572
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.2773678302764893,
          "mean_weights": [
            0.3856483995914459,
            0.18032220005989075,
            0.1928499937057495,
            0.24117937684059143
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3655540943145752,
          "mean_weights": [
            0.29819539189338684,
            0.20063453912734985,
            0.20929867029190063,
            0.29187142848968506
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.2754871845245361,
          "mean_weights": [
            0.42651012539863586,
            0.1713070571422577,
            0.1726072132587433,
            0.22957557439804077
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3596348762512207,
          "mean_weights": [
            0.2750997543334961,
            0.22746169567108154,
            0.18702520430088043,
            0.31041333079338074
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.5752925872802734,
          "mean_weights": [
            0.28211507201194763,
            0.1675817370414734,
            0.1759152114391327,
            0.16564446687698364,
            0.20874354243278503
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5704567432403564,
          "mean_weights": [
            0.21000899374485016,
            0.18079039454460144,
            0.13544103503227234,
            0.1749160885810852,
            0.29884350299835205
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.579707384109497,
          "mean_weights": [
            0.21335941553115845,
            0.18788449466228485,
            0.20250459015369415,
            0.1511598825454712,
            0.24509160220623016
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5663042068481445,
          "mean_weights": [
            0.21676909923553467,
            0.19467836618423462,
            0.13713446259498596,
            0.15313217043876648,
            0.29828590154647827
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5384478569030762,
          "mean_weights": [
            0.21904908120632172,
            0.15864285826683044,
            0.18178997933864594,
            0.14981383085250854,
            0.29070425033569336
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.552668571472168,
          "mean_weights": [
            0.22710487246513367,
            0.1922147125005722,
            0.13047492504119873,
            0.14686013758182526,
            0.30334532260894775
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5210195779800415,
          "mean_weights": [
            0.2812899947166443,
            0.14511609077453613,
            0.15909329056739807,
            0.13197475671768188,
            0.282525897026062
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5376527309417725,
          "mean_weights": [
            0.23905543982982635,
            0.21673071384429932,
            0.0950920507311821,
            0.16199833154678345,
            0.2871234714984894
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.7209875583648682,
          "mean_weights": [
            0.1703755110502243,
            0.14151470363140106,
            0.17087554931640625,
            0.12742051482200623,
            0.25328749418258667,
            0.1365262269973755
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7200093269348145,
          "mean_weights": [
            0.18992580473423004,
            0.16036251187324524,
            0.08439569175243378,
            0.11250905692577362,
            0.1775888204574585,
            0.27521812915802
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.753357172012329,
          "mean_weights": [
            0.15539541840553284,
            0.1388990879058838,
            0.15745995938777924,
            0.12427005916833878,
            0.17903099954128265,
            0.24494445323944092
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7199265956878662,
          "mean_weights": [
            0.19207912683486938,
            0.17904852330684662,
            0.08410340547561646,
            0.11219910532236099,
            0.15951165556907654,
            0.2730581760406494
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.6756346225738525,
          "mean_weights": [
            0.15863081812858582,
            0.11242729425430298,
            0.15970338881015778,
            0.09774985909461975,
            0.15905189514160156,
            0.3124367296695709
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7271867990493774,
          "mean_weights": [
            0.19560003280639648,
            0.18333327770233154,
            0.08475087583065033,
            0.11598999053239822,
            0.15889793634414673,
            0.2614278793334961
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.6916961669921875,
          "mean_weights": [
            0.19065657258033752,
            0.12983909249305725,
            0.11857748031616211,
            0.09883742034435272,
            0.1529499739408493,
            0.3091394901275635
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7504222393035889,
          "mean_weights": [
            0.20809361338615417,
            0.19103100895881653,
            0.10743936896324158,
            0.1194620355963707,
            0.15873855352401733,
            0.2152354121208191
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.5379457473754883,
          "mean_weights": [
            0.09571269154548645,
            0.07078731805086136,
            0.0824701339006424,
            0.060012347996234894,
            0.08145605027675629,
            0.09452696144580841,
            0.5150344967842102
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.7319592237472534,
          "mean_weights": [
            0.13678354024887085,
            0.11740625649690628,
            0.06668869405984879,
            0.07360264658927917,
            0.09287553280591965,
            0.12232495844364166,
            0.390318363904953
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.4663677215576172,
          "mean_weights": [
            0.09485072642564774,
            0.07167200744152069,
            0.0841417908668518,
            0.05784624442458153,
            0.07052354514598846,
            0.0724712610244751,
            0.5484944581985474
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.749481201171875,
          "mean_weights": [
            0.14530614018440247,
            0.12230368703603745,
            0.0660901889204979,
            0.07630561292171478,
            0.09453228116035461,
            0.12125740200281143,
            0.3742046654224396
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.472255825996399,
          "mean_weights": [
            0.10688216984272003,
            0.07465942203998566,
            0.08327433466911316,
            0.054589346051216125,
            0.0637175589799881,
            0.07299347221851349,
            0.5438836812973022
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.7924723625183105,
          "mean_weights": [
            0.15522946417331696,
            0.13724805414676666,
            0.06889298558235168,
            0.08198928833007812,
            0.09816981852054596,
            0.123870849609375,
            0.3345995247364044
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.5755163431167603,
          "mean_weights": [
            0.12274101376533508,
            0.08900541067123413,
            0.08551642298698425,
            0.06285787373781204,
            0.07429315149784088,
            0.07904880493879318,
            0.4865373373031616
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.8390922546386719,
          "mean_weights": [
            0.15884171426296234,
            0.14726482331752777,
            0.08053631335496902,
            0.09097073972225189,
            0.10738551616668701,
            0.12150256335735321,
            0.29349833726882935
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.6550344228744507,
          "mean_weights": [
            0.0844201147556305,
            0.06695197522640228,
            0.061310961842536926,
            0.0447731651365757,
            0.04748314991593361,
            0.045562684535980225,
            0.2905467748641968,
            0.3589511513710022
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.8679088354110718,
          "mean_weights": [
            0.11674155294895172,
            0.1088850349187851,
            0.05779466778039932,
            0.06273136287927628,
            0.07066237181425095,
            0.08129343390464783,
            0.17753683030605316,
            0.32435476779937744
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.5923268795013428,
          "mean_weights": [
            0.08656083047389984,
            0.05770102143287659,
            0.05105586349964142,
            0.039375390857458115,
            0.04410157352685928,
            0.04500962793827057,
            0.2357216477394104,
            0.4404740333557129
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.8469831943511963,
          "mean_weights": [
            0.10824646055698395,
            0.09116974472999573,
            0.0701109915971756,
            0.0629766583442688,
            0.07049155235290527,
            0.07297641038894653,
            0.18795298039913177,
            0.33607521653175354
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.6590619087219238,
          "mean_weights": [
            0.0944472923874855,
            0.0654030293226242,
            0.060137998312711716,
            0.0449567511677742,
            0.044405773282051086,
            0.05244143307209015,
            0.2202538251876831,
            0.4179539084434509
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.884438157081604,
          "mean_weights": [
            0.11613816022872925,
            0.09775452315807343,
            0.08394932001829147,
            0.07057420164346695,
            0.06920696794986725,
            0.06340071558952332,
            0.21605131030082703,
            0.2829247713088989
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.7424441576004028,
          "mean_weights": [
            0.08763163536787033,
            0.07771545648574829,
            0.07502467930316925,
            0.06006643921136856,
            0.06314834952354431,
            0.054570816457271576,
            0.3030073344707489,
            0.278835266828537
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.8953055143356323,
          "mean_weights": [
            0.1135263741016388,
            0.10274903476238251,
            0.0858360230922699,
            0.08475605398416519,
            0.08078759908676147,
            0.07204630225896835,
            0.2779633104801178,
            0.18233531713485718
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.801485300064087,
          "mean_weights": [
            0.07824120670557022,
            0.06023171544075012,
            0.054100219160318375,
            0.039871927350759506,
            0.04207881540060043,
            0.045095670968294144,
            0.2047339826822281,
            0.15412676334381104,
            0.321519672870636
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.008213520050049,
          "mean_weights": [
            0.09438058733940125,
            0.07153058052062988,
            0.07423336803913116,
            0.06403130292892456,
            0.0645662397146225,
            0.05577655881643295,
            0.21496367454528809,
            0.1609123945236206,
            0.1996053010225296
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.8966764211654663,
          "mean_weights": [
            0.09610478579998016,
            0.07267358899116516,
            0.060228414833545685,
            0.046177469193935394,
            0.04206448048353195,
            0.04096413776278496,
            0.21562573313713074,
            0.21000833809375763,
            0.21615305542945862
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.981720209121704,
          "mean_weights": [
            0.10501760244369507,
            0.06946457922458649,
            0.07816961407661438,
            0.07385893911123276,
            0.06364607810974121,
            0.05328303575515747,
            0.23084522783756256,
            0.15472577512264252,
            0.17098914086818695
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.0028607845306396,
          "mean_weights": [
            0.0842779204249382,
            0.10298711061477661,
            0.09207960218191147,
            0.07771934568881989,
            0.06265503168106079,
            0.0552704893052578,
            0.2620389461517334,
            0.1382768750190735,
            0.12469469010829926
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.0655057430267334,
          "mean_weights": [
            0.10472239553928375,
            0.11792458593845367,
            0.0928182601928711,
            0.10183010995388031,
            0.08287280797958374,
            0.06570011377334595,
            0.21871674060821533,
            0.11616810411214828,
            0.09924688935279846
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.0258567333221436,
          "mean_weights": [
            0.09674200415611267,
            0.09482249617576599,
            0.08996410667896271,
            0.06836152076721191,
            0.06293784081935883,
            0.05016916245222092,
            0.22868779301643372,
            0.1848844438791275,
            0.12343062460422516
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.059412047266960144,
        0.056020382791757584,
        0.08935670554637909,
        0.0742802768945694,
        0.17747226357460022,
        0.1285802572965622,
        0.17946027219295502,
        0.10598119348287582,
        0.17260213196277618,
        0.10133238881826401,
        0.12674333155155182,
        0.10985831171274185,
        0.12412907183170319,
        0.12611564993858337,
        0.11767174303531647,
        0.1125042662024498,
        0.13582420349121094,
        0.07848236709833145,
        0.10855290293693542,
        0.0899009183049202,
        0.10545514523983002,
        0.09790628403425217,
        0.11005345731973648,
        0.09713682532310486,
        0.13049271702766418,
        0.09147685021162033,
        0.09666567295789719,
        0.10080859810113907,
        0.10358717292547226,
        0.09896469116210938,
        0.11041814833879471,
        0.10007023066282272,
        0.13209649920463562,
        0.10974904894828796,
        0.10239993035793304,
        0.10291008651256561,
        0.11153820157051086,
        0.10674569010734558,
        0.11912636458873749,
        0.12148815393447876,
        0.127323716878891,
        0.055056143552064896,
        0.07468672841787338,
        0.05981758236885071,
        0.07975661009550095,
        0.07101839780807495,
        0.08746611326932907,
        0.08339047431945801,
        0.09389390796422958,
        0.06143829599022865,
        0.06575146317481995,
        0.06018613278865814,
        0.0681111291050911,
        0.06816941499710083,
        0.07568862289190292,
        0.08121132105588913,
        0.08424623310565948,
        0.05916989967226982,
        0.0719028189778328,
        0.07458337396383286,
        0.08439797908067703,
        0.08333029597997665,
        0.08961156010627747,
        0.09104464948177338
      ],
      "loss_nats": 1.347039818763733,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 1.7394249439239502,
        "mean_weights": [
          0.10602211207151413,
          0.05902107432484627,
          0.049884356558322906,
          0.034865088760852814,
          0.03282830864191055,
          0.043302543461322784,
          0.08693542331457138,
          0.3314756751060486,
          0.2556654214859009
        ],
        "sources": 9
      },
      "step": 4000,
      "stream_state_rms_by_sublayer": [
        0.08528144657611847,
        0.16101159155368805,
        0.3133305609226227,
        0.3334391713142395,
        0.3510810136795044,
        0.3570481836795807,
        0.36267346143722534,
        0.37708213925361633,
        0.1070452407002449,
        0.11129721254110336,
        0.1260577291250229,
        0.12943366169929504,
        0.14502130448818207,
        0.1512180119752884,
        0.17329032719135284,
        0.1945808082818985,
        0.061002109199762344,
        0.07549218833446503,
        0.0987050011754036,
        0.11765842139720917,
        0.13416865468025208,
        0.16134388744831085,
        0.17569874227046967,
        0.21493762731552124,
        0.06714949756860733,
        0.08329613506793976,
        0.09735818952322006,
        0.12312135100364685,
        0.13312570750713348,
        0.16809259355068207,
        0.17941679060459137,
        0.23101076483726501,
        0.10073095560073853,
        0.11538279056549072,
        0.13780468702316284,
        0.16158254444599152,
        0.17965665459632874,
        0.20080390572547913,
        0.24453195929527283,
        0.27125677466392517,
        0.04170985147356987,
        0.0543946772813797,
        0.07352267950773239,
        0.09335283935070038,
        0.10462787002325058,
        0.12897421419620514,
        0.14256280660629272,
        0.17114612460136414,
        0.030182205140590668,
        0.06173322722315788,
        0.06915032118558884,
        0.09693212062120438,
        0.10093247890472412,
        0.13771499693393707,
        0.1392749696969986,
        0.18387214839458466,
        0.030125634744763374,
        0.07983565330505371,
        0.08267941325902939,
        0.15526357293128967,
        0.15948866307735443,
        0.26127612590789795,
        0.2670789062976837,
        0.39550676941871643
      ]
    },
    {
      "activation_grad_rms_by_block": [
        5.980865898891352e-05,
        6.95241178618744e-05,
        6.203598604770377e-05,
        5.727907409891486e-05,
        7.378464943030849e-05,
        6.77111092954874e-05,
        6.276105705183e-05,
        5.631716703646816e-05,
        4.664637162932195e-05,
        4.3178879423066974e-05,
        4.080804137629457e-05,
        3.917843787348829e-05,
        5.551072536036372e-05,
        5.1256221922812983e-05,
        4.731794251711108e-05,
        4.47981983597856e-05,
        5.737639730796218e-05,
        5.2802144637098536e-05,
        5.3134874178795144e-05,
        6.219849456101656e-05,
        0.0002451144973747432,
        0.00019094848539680243,
        0.00015029680798761547,
        0.00011999815615126863,
        0.0001724466128507629,
        0.00012853756197728217,
        8.318446634802967e-05,
        6.64951658109203e-05,
        6.248620775295421e-05,
        4.157395596848801e-05,
        3.7306774174794555e-05,
        3.4809250792022794e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 6.365272747643758e-05,
        "first_to_last_ratio": 0.8123633197064085,
        "imbalance_abs_log_ratio": 0.2078076008289402,
        "last_quartile_mean": 7.835499945940683e-05,
        "mean": 7.614457786075945e-05,
        "normalized": [
          0.7854618236676242,
          0.9130540849409994,
          0.8147131127464502,
          0.7522410092502885,
          0.9690072688462938,
          0.8892440039434748,
          0.8242354060534288,
          0.7396083689563772,
          0.6126026690255093,
          0.5670644008562938,
          0.5359283946772616,
          0.5145269561429746,
          0.7290174418180176,
          0.6731434248219464,
          0.6214223500409742,
          0.5883307730946398,
          0.7535191463387267,
          0.6934458909688136,
          0.6978155985835179,
          0.8168473226649812,
          3.219066994145898,
          2.5077095541323673,
          1.973834673592298,
          1.5759251613516239,
          2.2647260999477155,
          1.6880724220748888,
          1.092454232265147,
          0.8732751258075867,
          0.8206258345435784,
          0.5459870832104624,
          0.48994656248557816,
          0.45714680900426247
        ],
        "population_cv": 0.6420341358971389
      },
      "activation_output_rms_by_block": [
        0.16915453970432281,
        0.374446302652359,
        0.39966297149658203,
        0.4194087088108063,
        0.09892481565475464,
        0.11664740741252899,
        0.13951337337493896,
        0.18593810498714447,
        0.06607726216316223,
        0.10425819456577301,
        0.14551182091236115,
        0.19865906238555908,
        0.06484340131282806,
        0.10430748760700226,
        0.14720691740512848,
        0.20439592003822327,
        0.10052042454481125,
        0.14085541665554047,
        0.1745898574590683,
        0.2420409768819809,
        0.049456026405096054,
        0.08509506285190582,
        0.11783231794834137,
        0.15649692714214325,
        0.05533855780959129,
        0.08507220447063446,
        0.11942370235919952,
        0.1559232622385025,
        0.06628894805908203,
        0.12384890764951706,
        0.19928614795207977,
        0.30097496509552
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.23796202801167965,
        "first_to_last_ratio": 1.7210004980371292,
        "imbalance_abs_log_ratio": 0.542905806617582,
        "last_quartile_mean": 0.13826958695426583,
        "mean": 0.15974999987520278,
        "normalized": [
          1.0588703589137207,
          2.3439518181213,
          2.5018026404306735,
          2.625406629974646,
          0.6192476728139908,
          0.7301872144203714,
          0.8733231516990753,
          1.1639318005158055,
          0.41362918444307983,
          0.6526334563206251,
          0.9108721190988135,
          1.2435622068278698,
          0.40590548584340497,
          0.6529420199592338,
          0.9214830517691831,
          1.2794736788600816,
          0.6292358348878756,
          0.8817240486108118,
          1.0928942572485663,
          1.5151234871428112,
          0.30958389010160414,
          0.5326764501933169,
          0.737604494775539,
          0.9796364773984299,
          0.3464072479049888,
          0.5325333617345424,
          0.7475662125351719,
          0.9760454607844148,
          0.41495429177381643,
          0.7752670281456541,
          1.247487625087717,
          1.8840373416628649
        ],
        "population_cv": 0.591409330949146
      },
      "bits_per_byte": 1.8740129107275643,
      "branch_output_rms_by_sublayer": [
        0.0933690071105957,
        0.13583384454250336,
        0.24537226557731628,
        0.0415157787501812,
        0.07698935270309448,
        0.018140379339456558,
        0.0523640513420105,
        0.039518751204013824,
        0.09380833059549332,
        0.022877270355820656,
        0.03647654876112938,
        0.015532293356955051,
        0.039120711386203766,
        0.02314678393304348,
        0.06371714919805527,
        0.05432315170764923,
        0.05156813934445381,
        0.03743875026702881,
        0.03728010877966881,
        0.04006272554397583,
        0.03983496502041817,
        0.045583780854940414,
        0.040284499526023865,
        0.05995568633079529,
        0.046871740370988846,
        0.038651175796985626,
        0.04314620420336723,
        0.04559725895524025,
        0.04033593460917473,
        0.05632574483752251,
        0.04471700266003609,
        0.07952121645212173,
        0.08397205173969269,
        0.042337141931056976,
        0.0528092160820961,
        0.05044656619429588,
        0.04443390294909477,
        0.04418695345520973,
        0.07560808956623077,
        0.06310147047042847,
        0.03708760067820549,
        0.029010163620114326,
        0.03599191829562187,
        0.04586305841803551,
        0.033304885029792786,
        0.06033914536237717,
        0.04061548039317131,
        0.07656749337911606,
        0.025487789884209633,
        0.045884523540735245,
        0.021565141156315804,
        0.051604680716991425,
        0.02342291921377182,
        0.06502768397331238,
        0.017244853079319,
        0.06792232394218445,
        0.024166326969861984,
        0.060306549072265625,
        0.016819356009364128,
        0.08025261014699936,
        0.021590016782283783,
        0.09663844108581543,
        0.026201345026493073,
        0.12230555713176727
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 32,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0003065916678101079,
        0.00023586292031870838,
        0.0001322871105193671,
        0.00016391009158296018,
        0.0001640597280926294,
        0.00011452252186811768,
        0.00012670462239247783,
        0.00021513822087305528,
        0.00015124888511926465,
        0.00012797922905715274,
        0.0001317390201682035,
        0.00014536342393434786,
        0.00017054401801644127,
        0.00018163687748006386,
        0.00019727429589294548,
        0.0002118946661868627,
        0.0002209032617907253,
        0.0002444386191807451,
        0.00023590577825104205,
        0.00032383840230939837,
        0.0010316902394974224,
        0.0010821684520459678,
        0.0009765210521856827,
        0.0008761640307174725,
        0.0008326776627851861,
        0.0006767392019523396,
        0.00052552098450176,
        0.0004066875433884613,
        0.0003364842674135882,
        0.00025378360531775436,
        0.000205309661176443,
        0.00020659565641729775
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00018238461043217796,
        "first_to_last_ratio": 0.42368241008054586,
        "imbalance_abs_log_ratio": 0.8587711372443304,
        "last_quartile_mean": 0.0004304748228691038,
        "mean": 0.00035038080369512474,
        "normalized": [
          0.875024158220954,
          0.6731616510701846,
          0.37755239192405493,
          0.46780556998089007,
          0.46823263821091626,
          0.3268515873596963,
          0.3616197607181892,
          0.6140125788975943,
          0.43167001024082974,
          0.3652575361078021,
          0.37598812143496585,
          0.4148726825252455,
          0.486739045683667,
          0.5183984840622454,
          0.5630282647122382,
          0.6047553517550511,
          0.630466222638552,
          0.69763701836085,
          0.6732839692219831,
          0.9242469875466649,
          2.944482770223686,
          3.0885494885375917,
          2.7870278333951726,
          2.500605117282182,
          2.3764933866346176,
          1.9314391508194257,
          1.499856667259172,
          1.160701554136312,
          0.9603387624693381,
          0.7243079604856947,
          0.5859614996348036,
          0.5896317784494322
        ],
        "population_cv": 0.8248093894360681
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.22886337339878082,
          "mean_weights": [
            0.9364937543869019,
            0.06350618600845337
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.603729784488678,
          "mean_weights": [
            0.5866844654083252,
            0.4133155643939972
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.3967982828617096,
          "mean_weights": [
            0.858893632888794,
            0.14110638201236725
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6852496862411499,
          "mean_weights": [
            0.48102498054504395,
            0.518975019454956
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.5890734195709229,
          "mean_weights": [
            0.6876576542854309,
            0.3123423457145691
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6873323321342468,
          "mean_weights": [
            0.5168514847755432,
            0.4831485152244568
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.487301766872406,
          "mean_weights": [
            0.7939615249633789,
            0.2060384750366211
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.674697756767273,
          "mean_weights": [
            0.5704737305641174,
            0.42952626943588257
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.9691112041473389,
          "mean_weights": [
            0.5506306886672974,
            0.18125343322753906,
            0.2681158185005188
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0745112895965576,
          "mean_weights": [
            0.342629998922348,
            0.24457570910453796,
            0.412794291973114
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0074045658111572,
          "mean_weights": [
            0.4399106502532959,
            0.18251104652881622,
            0.3775783181190491
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.060497522354126,
          "mean_weights": [
            0.3786153495311737,
            0.2188822329044342,
            0.4025024175643921
          ],
          "sources": 3
        },
        {
          "entropy_mean": 0.9941019415855408,
          "mean_weights": [
            0.32296958565711975,
            0.19686520099639893,
            0.4801652133464813
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0503522157669067,
          "mean_weights": [
            0.40130966901779175,
            0.19853851199150085,
            0.4001518487930298
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0129402875900269,
          "mean_weights": [
            0.4093267321586609,
            0.16906049847602844,
            0.42161279916763306
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0774019956588745,
          "mean_weights": [
            0.38510820269584656,
            0.25884830951690674,
            0.3560434877872467
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.2740377187728882,
          "mean_weights": [
            0.29550763964653015,
            0.09541245549917221,
            0.24569378793239594,
            0.3633860945701599
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.360302448272705,
          "mean_weights": [
            0.3109527826309204,
            0.20984822511672974,
            0.1935039907693863,
            0.28569501638412476
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.2722704410552979,
          "mean_weights": [
            0.350364625453949,
            0.12631991505622864,
            0.22418734431266785,
            0.29912811517715454
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3558036088943481,
          "mean_weights": [
            0.2989768385887146,
            0.1941717267036438,
            0.1926535815000534,
            0.314197838306427
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.2696614265441895,
          "mean_weights": [
            0.3667730689048767,
            0.15310388803482056,
            0.19384080171585083,
            0.2862822115421295
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3606936931610107,
          "mean_weights": [
            0.2982274293899536,
            0.18611131608486176,
            0.21284276247024536,
            0.3028184771537781
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.269316554069519,
          "mean_weights": [
            0.4081852436065674,
            0.14420126378536224,
            0.18228203058242798,
            0.2653314471244812
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3636136054992676,
          "mean_weights": [
            0.27427661418914795,
            0.21341991424560547,
            0.19867971539497375,
            0.3136237859725952
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.570773959159851,
          "mean_weights": [
            0.26587405800819397,
            0.13419361412525177,
            0.1907704770565033,
            0.17973750829696655,
            0.2294243425130844
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5560038089752197,
          "mean_weights": [
            0.20477107167243958,
            0.15721666812896729,
            0.13811248540878296,
            0.1752161681652069,
            0.32468360662460327
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5659973621368408,
          "mean_weights": [
            0.20531430840492249,
            0.15986156463623047,
            0.21557101607322693,
            0.15222224593162537,
            0.26703086495399475
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5552496910095215,
          "mean_weights": [
            0.2162160575389862,
            0.17982497811317444,
            0.1307673156261444,
            0.1554446816444397,
            0.31774696707725525
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5068919658660889,
          "mean_weights": [
            0.20646695792675018,
            0.1197461187839508,
            0.19215428829193115,
            0.15179987251758575,
            0.3298327624797821
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5435223579406738,
          "mean_weights": [
            0.22410975396633148,
            0.16986176371574402,
            0.13080401718616486,
            0.15279728174209595,
            0.3224271535873413
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.4949920177459717,
          "mean_weights": [
            0.27174079418182373,
            0.12178683280944824,
            0.1529826819896698,
            0.1350194811820984,
            0.31847020983695984
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.544264793395996,
          "mean_weights": [
            0.24016115069389343,
            0.20730282366275787,
            0.09692574292421341,
            0.1695065051317215,
            0.2861037850379944
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.686682939529419,
          "mean_weights": [
            0.16755425930023193,
            0.10962115228176117,
            0.1783454418182373,
            0.13310004770755768,
            0.2885603904724121,
            0.12281867861747742
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7070834636688232,
          "mean_weights": [
            0.181177020072937,
            0.14557164907455444,
            0.08318156003952026,
            0.11424209922552109,
            0.17698991298675537,
            0.2988377511501312
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7419235706329346,
          "mean_weights": [
            0.1532188206911087,
            0.11420352011919022,
            0.1659742295742035,
            0.1250365972518921,
            0.18140509724617004,
            0.26016172766685486
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7127439975738525,
          "mean_weights": [
            0.18731769919395447,
            0.16715484857559204,
            0.08496320992708206,
            0.11362075805664062,
            0.1561521738767624,
            0.2907913327217102
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.6076834201812744,
          "mean_weights": [
            0.15032559633255005,
            0.08624927699565887,
            0.15736976265907288,
            0.08584291487932205,
            0.14632999897003174,
            0.3738824725151062
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.709290862083435,
          "mean_weights": [
            0.18991519510746002,
            0.17091524600982666,
            0.07822999358177185,
            0.116765096783638,
            0.1540035903453827,
            0.2901708781719208
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.6690887212753296,
          "mean_weights": [
            0.19029225409030914,
            0.11002412438392639,
            0.11471512168645859,
            0.0999990925192833,
            0.1507222205400467,
            0.3342471718788147
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7458693981170654,
          "mean_weights": [
            0.20528709888458252,
            0.1722964346408844,
            0.10435444861650467,
            0.12075909972190857,
            0.16116347908973694,
            0.2361394464969635
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.4612125158309937,
          "mean_weights": [
            0.08934466540813446,
            0.054379623383283615,
            0.07567165046930313,
            0.05758609622716904,
            0.07884950935840607,
            0.08982288837432861,
            0.5543456077575684
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.7241814136505127,
          "mean_weights": [
            0.13360008597373962,
            0.10609637945890427,
            0.06363911926746368,
            0.07426095008850098,
            0.09383077174425125,
            0.13373199105262756,
            0.39484071731567383
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.3979300260543823,
          "mean_weights": [
            0.08959861099720001,
            0.05414731800556183,
            0.07668881118297577,
            0.05603286623954773,
            0.06979228556156158,
            0.07073447853326797,
            0.5830056071281433
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.7459560632705688,
          "mean_weights": [
            0.14318783581256866,
            0.1094653457403183,
            0.06349228322505951,
            0.07859592139720917,
            0.09567482024431229,
            0.13266830146312714,
            0.37691545486450195
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.4055393934249878,
          "mean_weights": [
            0.10154954344034195,
            0.05619275942444801,
            0.07791627943515778,
            0.053915880620479584,
            0.06398355215787888,
            0.06915920972824097,
            0.5772827863693237
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.7853620052337646,
          "mean_weights": [
            0.15284112095832825,
            0.12206703424453735,
            0.06508444249629974,
            0.08347312361001968,
            0.09811970591545105,
            0.13767032325267792,
            0.3407442569732666
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.509842872619629,
          "mean_weights": [
            0.11788781732320786,
            0.06822571903467178,
            0.08556114137172699,
            0.06067090481519699,
            0.07228855788707733,
            0.07258446514606476,
            0.5227814316749573
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.839367389678955,
          "mean_weights": [
            0.15906387567520142,
            0.13139380514621735,
            0.07717745006084442,
            0.09607303887605667,
            0.11078508198261261,
            0.12967455387115479,
            0.29583218693733215
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.5841834545135498,
          "mean_weights": [
            0.07896879315376282,
            0.048752427101135254,
            0.055707160383462906,
            0.04139826446771622,
            0.04449019208550453,
            0.04285799711942673,
            0.2801477909088135,
            0.40767738223075867
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.8125314712524414,
          "mean_weights": [
            0.10889512300491333,
            0.08656036853790283,
            0.048123255372047424,
            0.06561170518398285,
            0.06917259097099304,
            0.08286689221858978,
            0.16869932413101196,
            0.3700707256793976
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.50089693069458,
          "mean_weights": [
            0.07974998652935028,
            0.04166398197412491,
            0.04607272148132324,
            0.03586764261126518,
            0.04223340004682541,
            0.04095568507909775,
            0.22249138355255127,
            0.49096518754959106
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.8059929609298706,
          "mean_weights": [
            0.10711923241615295,
            0.07651708275079727,
            0.058284737169742584,
            0.06609588116407394,
            0.07139427959918976,
            0.07493151724338531,
            0.17425751686096191,
            0.37139976024627686
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.5622732639312744,
          "mean_weights": [
            0.08775827288627625,
            0.047765426337718964,
            0.053928621113300323,
            0.04293380677700043,
            0.04174989461898804,
            0.04734005779027939,
            0.2016349583864212,
            0.4768889844417572
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.8757455348968506,
          "mean_weights": [
            0.11563669890165329,
            0.08108697831630707,
            0.07280858606100082,
            0.07553227990865707,
            0.07606013864278793,
            0.06935927271842957,
            0.19856664538383484,
            0.3109493851661682
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.6924852132797241,
          "mean_weights": [
            0.08638577163219452,
            0.059912849217653275,
            0.06609142571687698,
            0.056745607405900955,
            0.06277773529291153,
            0.05352145433425903,
            0.2830069661140442,
            0.3315581977367401
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.8980522155761719,
          "mean_weights": [
            0.11603894829750061,
            0.08804173022508621,
            0.07975024729967117,
            0.09026999771595001,
            0.08430293202400208,
            0.07433462888002396,
            0.26602888107299805,
            0.2012326419353485
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.7512003183364868,
          "mean_weights": [
            0.07744461297988892,
            0.046086352318525314,
            0.05025433003902435,
            0.037484876811504364,
            0.04005061089992523,
            0.041623204946517944,
            0.198423832654953,
            0.17380405962467194,
            0.33482813835144043
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.0160694122314453,
          "mean_weights": [
            0.0938527062535286,
            0.06208505481481552,
            0.06560768187046051,
            0.07029052078723907,
            0.06935097277164459,
            0.056832704693078995,
            0.1942581683397293,
            0.1706070452928543,
            0.21711516380310059
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.7837426662445068,
          "mean_weights": [
            0.08836328238248825,
            0.04953558370471001,
            0.04813014343380928,
            0.0373263955116272,
            0.037015400826931,
            0.03390160948038101,
            0.1840541511774063,
            0.23140482604503632,
            0.29026859998703003
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.9973840713500977,
          "mean_weights": [
            0.10261696577072144,
            0.06039813905954361,
            0.06671924889087677,
            0.08377155661582947,
            0.07002593576908112,
            0.05768897384405136,
            0.20831166207790375,
            0.1594475358724594,
            0.19101998209953308
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.969987154006958,
          "mean_weights": [
            0.08120635896921158,
            0.07894632965326309,
            0.08615730702877045,
            0.07609778642654419,
            0.058728985488414764,
            0.05076971277594566,
            0.24503237009048462,
            0.1544169783592224,
            0.16864418983459473
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.071648359298706,
          "mean_weights": [
            0.10569609701633453,
            0.09922641515731812,
            0.08180510997772217,
            0.10646798461675644,
            0.0826798751950264,
            0.06615597754716873,
            0.20955613255500793,
            0.13320739567279816,
            0.11520501971244812
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.9781701564788818,
          "mean_weights": [
            0.09446848928928375,
            0.07056106626987457,
            0.08033449202775955,
            0.0658978745341301,
            0.058234505355358124,
            0.04316519945859909,
            0.2036968618631363,
            0.22251945734024048,
            0.16112206876277924
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.0626930519938469,
        0.059514161199331284,
        0.09030644595623016,
        0.08157797157764435,
        0.20229491591453552,
        0.14292451739311218,
        0.20300470292568207,
        0.10997853428125381,
        0.19049881398677826,
        0.0991024449467659,
        0.12695962190628052,
        0.1065012738108635,
        0.12132706493139267,
        0.12155740708112717,
        0.11585728824138641,
        0.1095810979604721,
        0.13427142798900604,
        0.06964563578367233,
        0.10485932976007462,
        0.08311934769153595,
        0.10419609397649765,
        0.09363401681184769,
        0.10852954536676407,
        0.0941167026758194,
        0.1286265105009079,
        0.08564268052577972,
        0.09096142649650574,
        0.09493856877088547,
        0.10080574452877045,
        0.09063401818275452,
        0.10566592961549759,
        0.09473779052495956,
        0.1287592649459839,
        0.10505300015211105,
        0.09650235623121262,
        0.09355910867452621,
        0.1065765842795372,
        0.09618847072124481,
        0.11446042358875275,
        0.11416846513748169,
        0.12395420670509338,
        0.047205954790115356,
        0.07221997529268265,
        0.053352247923612595,
        0.07677769660949707,
        0.06512638926506042,
        0.08416415005922318,
        0.07829347252845764,
        0.09094838052988052,
        0.05368548259139061,
        0.059439923614263535,
        0.05310077592730522,
        0.062182098627090454,
        0.061235878616571426,
        0.06896103918552399,
        0.07320034503936768,
        0.0776243805885315,
        0.05304349213838577,
        0.06285123527050018,
        0.0639304369688034,
        0.07231374084949493,
        0.07404901832342148,
        0.08016443997621536,
        0.08157160878181458
      ],
      "loss_nats": 1.2989667654037476,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 1.6036605834960938,
        "mean_weights": [
          0.09759747982025146,
          0.04215940088033676,
          0.041145484894514084,
          0.029554029926657677,
          0.02899969555437565,
          0.03647579252719879,
          0.06642461568117142,
          0.341248482465744,
          0.31639501452445984
        ],
        "sources": 9
      },
      "step": 8000,
      "stream_state_rms_by_sublayer": [
        0.0933690071105957,
        0.16915453970432281,
        0.35791492462158203,
        0.374446302652359,
        0.3940892517566681,
        0.39966297149658203,
        0.4057151973247528,
        0.4194087088108063,
        0.09380833059549332,
        0.09892481565475464,
        0.11286329478025436,
        0.11664740741252899,
        0.13231655955314636,
        0.13951337337493896,
        0.1621972620487213,
        0.18593810498714447,
        0.05156813934445381,
        0.06607726216316223,
        0.08533531427383423,
        0.10425819456577301,
        0.12003090232610703,
        0.14551182091236115,
        0.16063299775123596,
        0.19865906238555908,
        0.046871740370988846,
        0.06484340131282806,
        0.079384446144104,
        0.10430748760700226,
        0.11459959298372269,
        0.14720691740512848,
        0.15625107288360596,
        0.20439592003822327,
        0.08397205173969269,
        0.10052042454481125,
        0.11654024571180344,
        0.14085541665554047,
        0.15294702351093292,
        0.1745898574590683,
        0.21572710573673248,
        0.2420409768819809,
        0.03708760067820549,
        0.049456026405096054,
        0.06553087383508682,
        0.08509506285190582,
        0.09390194714069366,
        0.11783231794834137,
        0.12852425873279572,
        0.15649692714214325,
        0.025487789884209633,
        0.05533855780959129,
        0.06059965491294861,
        0.08507220447063446,
        0.08860157430171967,
        0.11942370235919952,
        0.12100546807050705,
        0.1559232622385025,
        0.024166326969861984,
        0.06628894805908203,
        0.06877554953098297,
        0.12384890764951706,
        0.12682415544986725,
        0.19928614795207977,
        0.2025238573551178,
        0.30097496509552
      ]
    }
  ],
  "environment": {
    "autocast": "cuda-bfloat16-forward-fp32-cross-entropy",
    "compile": false,
    "compute_capability": [
      12,
      0
    ],
    "cublas_workspace_config": ":4096:8",
    "cuda": "12.8",
    "deterministic_algorithms": true,
    "gpu": "NVIDIA GeForce RTX 5090",
    "python": "3.10.14",
    "torch": "2.11.0+cu128"
  },
  "evaluations": [
    {
      "bits_per_byte": 7.9557801867581395,
      "cross_entropy_nats": 5.514526605606079,
      "step": 0
    },
    {
      "bits_per_byte": 5.165135820292823,
      "cross_entropy_nats": 3.5801993310451508,
      "step": 100
    },
    {
      "bits_per_byte": 3.109974443324717,
      "cross_entropy_nats": 2.155670017004013,
      "step": 500
    },
    {
      "bits_per_byte": 1.9592225965086283,
      "cross_entropy_nats": 1.358029618859291,
      "step": 2000
    },
    {
      "bits_per_byte": 1.8027438806514628,
      "cross_entropy_nats": 1.249566838145256,
      "step": 4000
    },
    {
      "bits_per_byte": 1.7123525940760473,
      "cross_entropy_nats": 1.1869123727083206,
      "step": 8000
    }
  ],
  "gradient_gate": null,
  "hashes": {
    "final_mixer_parameters": "f16f089e57699cb35addeee1608f1119edec41037e2b24b645f889b4f23c5f71",
    "final_model_state": "3f0b97ece3a15571ba3d656f589f512ca0bb9e20083c9f58a42ccaee14892f59",
    "final_optimizer_state": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637",
    "final_public_parameters": "1fbfd02c38aae6a81ba07a28d1135ce7d290651ca00bf5fa93595a997937551e",
    "initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d",
    "initial_public_parameter_elements": 18985152,
    "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670",
    "initial_public_parameter_tensors": 227,
    "initial_public_parameters": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58"
  },
  "manifest": {
    "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716",
    "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371",
    "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
    "input_gate_tensor_hashes": {
      "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434",
      "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94",
      "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc"
    },
    "path": "experiments/k3/attnres_gradient/manifest.json",
    "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338"
  },
  "model": {
    "attnres_aggregation_groups": 8,
    "context": 256,
    "d_ff": 768,
    "d_head": 32,
    "d_model": 192,
    "heads": 6,
    "layers": 32,
    "parameters": {
      "core": 18985152,
      "embedding": 98304,
      "mixer": 24960,
      "total": 19010112
    },
    "sublayers": 64,
    "sublayers_per_attnres_group": 8,
    "transformer_blocks_per_attnres_group": 4,
    "vocabulary": 256
  },
  "optimizer": {
    "betas": [
      0.9,
      0.95
    ],
    "epsilon": 1e-08,
    "grad_clip": 1.0,
    "min_lr": 3e-05,
    "name": "AdamW",
    "peak_lr": 0.0003,
    "warmup_steps": 400,
    "weight_decay_ndim_ge_2": 0.1
  },
  "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
  "run_kind": "replay",
  "schema_version": 1,
  "seed": 2026073001,
  "steps": 8000,
  "target_bytes_seen": 65536000,
  "timing": {
    "mean_ms": 110.1352149358973,
    "measured_steps": 7980,
    "median_ms": 110.01293901063036,
    "p95_ms": 110.78795030334732,
    "peak_allocated_bytes": 12815843328,
    "peak_reserved_bytes": 13337886720,
    "warmup_steps_excluded": 20
  },
  "training_history": [
    {
      "bits_per_byte": 7.958777844362678,
      "learning_rate": 7.499999999999999e-07,
      "loss_nats": 5.516604423522949,
      "step": 1,
      "unclipped_grad_norm": 21.323171615600586
    },
    {
      "bits_per_byte": 7.247870358756172,
      "learning_rate": 7.499999999999999e-06,
      "loss_nats": 5.02384090423584,
      "step": 10,
      "unclipped_grad_norm": 11.153014183044434
    },
    {
      "bits_per_byte": 6.7336346654397365,
      "learning_rate": 1.4999999999999999e-05,
      "loss_nats": 4.667399883270264,
      "step": 20,
      "unclipped_grad_norm": 3.4888553619384766
    },
    {
      "bits_per_byte": 6.611855126224988,
      "learning_rate": 2.2499999999999998e-05,
      "loss_nats": 4.582988739013672,
      "step": 30,
      "unclipped_grad_norm": 3.075686454772949
    },
    {
      "bits_per_byte": 6.3915955694037825,
      "learning_rate": 2.9999999999999997e-05,
      "loss_nats": 4.43031644821167,
      "step": 40,
      "unclipped_grad_norm": 3.2318925857543945
    },
    {
      "bits_per_byte": 6.093941755902502,
      "learning_rate": 3.75e-05,
      "loss_nats": 4.223998546600342,
      "step": 50,
      "unclipped_grad_norm": 2.8704335689544678
    },
    {
      "bits_per_byte": 5.8858000966610895,
      "learning_rate": 4.4999999999999996e-05,
      "loss_nats": 4.079725742340088,
      "step": 60,
      "unclipped_grad_norm": 2.669306755065918
    },
    {
      "bits_per_byte": 5.715495996470702,
      "learning_rate": 5.2499999999999995e-05,
      "loss_nats": 3.9616799354553223,
      "step": 70,
      "unclipped_grad_norm": 3.1737496852874756
    },
    {
      "bits_per_byte": 5.565897919513593,
      "learning_rate": 5.9999999999999995e-05,
      "loss_nats": 3.8579864501953125,
      "step": 80,
      "unclipped_grad_norm": 2.5544967651367188
    },
    {
      "bits_per_byte": 5.354669171689194,
      "learning_rate": 6.749999999999999e-05,
      "loss_nats": 3.711573839187622,
      "step": 90,
      "unclipped_grad_norm": 2.425950050354004
    },
    {
      "bits_per_byte": 5.180876231443403,
      "learning_rate": 7.5e-05,
      "loss_nats": 3.5911097526550293,
      "step": 100,
      "unclipped_grad_norm": 2.2783501148223877
    },
    {
      "bits_per_byte": 4.992272489270609,
      "learning_rate": 8.249999999999999e-05,
      "loss_nats": 3.4603796005249023,
      "step": 110,
      "unclipped_grad_norm": 2.1105899810791016
    },
    {
      "bits_per_byte": 4.775783480086919,
      "learning_rate": 8.999999999999999e-05,
      "loss_nats": 3.3103208541870117,
      "step": 120,
      "unclipped_grad_norm": 1.8969556093215942
    },
    {
      "bits_per_byte": 4.582375918510225,
      "learning_rate": 9.75e-05,
      "loss_nats": 3.1762609481811523,
      "step": 130,
      "unclipped_grad_norm": 1.7015109062194824
    },
    {
      "bits_per_byte": 4.427903165297061,
      "learning_rate": 0.00010499999999999999,
      "loss_nats": 3.0691885948181152,
      "step": 140,
      "unclipped_grad_norm": 1.446854829788208
    },
    {
      "bits_per_byte": 4.271120341117507,
      "learning_rate": 0.0001125,
      "loss_nats": 2.960515022277832,
      "step": 150,
      "unclipped_grad_norm": 1.2307006120681763
    },
    {
      "bits_per_byte": 4.166711049639903,
      "learning_rate": 0.00011999999999999999,
      "loss_nats": 2.888144016265869,
      "step": 160,
      "unclipped_grad_norm": 1.0598198175430298
    },
    {
      "bits_per_byte": 4.026318516052877,
      "learning_rate": 0.00012749999999999998,
      "loss_nats": 2.7908313274383545,
      "step": 170,
      "unclipped_grad_norm": 0.9654788374900818
    },
    {
      "bits_per_byte": 3.94755837315965,
      "learning_rate": 0.00013499999999999997,
      "loss_nats": 2.736238956451416,
      "step": 180,
      "unclipped_grad_norm": 0.8419675230979919
    },
    {
      "bits_per_byte": 3.838975750676209,
      "learning_rate": 0.0001425,
      "loss_nats": 2.660975217819214,
      "step": 190,
      "unclipped_grad_norm": 0.817121684551239
    },
    {
      "bits_per_byte": 3.774983758012572,
      "learning_rate": 0.00015,
      "loss_nats": 2.616619348526001,
      "step": 200,
      "unclipped_grad_norm": 0.955824613571167
    },
    {
      "bits_per_byte": 3.7344016998190783,
      "learning_rate": 0.0001575,
      "loss_nats": 2.5884900093078613,
      "step": 210,
      "unclipped_grad_norm": 0.8557147979736328
    },
    {
      "bits_per_byte": 3.63166201591605,
      "learning_rate": 0.00016499999999999997,
      "loss_nats": 2.5172762870788574,
      "step": 220,
      "unclipped_grad_norm": 0.6891153454780579
    },
    {
      "bits_per_byte": 3.6208855830129867,
      "learning_rate": 0.0001725,
      "loss_nats": 2.5098066329956055,
      "step": 230,
      "unclipped_grad_norm": 1.036860466003418
    },
    {
      "bits_per_byte": 3.4987607026353063,
      "learning_rate": 0.00017999999999999998,
      "loss_nats": 2.4251561164855957,
      "step": 240,
      "unclipped_grad_norm": 0.6406477689743042
    },
    {
      "bits_per_byte": 3.5387748741156506,
      "learning_rate": 0.0001875,
      "loss_nats": 2.4528918266296387,
      "step": 250,
      "unclipped_grad_norm": 1.106186866760254
    },
    {
      "bits_per_byte": 3.5260464381252943,
      "learning_rate": 0.000195,
      "loss_nats": 2.4440691471099854,
      "step": 260,
      "unclipped_grad_norm": 0.8556521534919739
    },
    {
      "bits_per_byte": 3.5219934949750615,
      "learning_rate": 0.00020249999999999996,
      "loss_nats": 2.4412598609924316,
      "step": 270,
      "unclipped_grad_norm": 0.9917709827423096
    },
    {
      "bits_per_byte": 3.474740573719901,
      "learning_rate": 0.00020999999999999998,
      "loss_nats": 2.4085066318511963,
      "step": 280,
      "unclipped_grad_norm": 1.2901222705841064
    },
    {
      "bits_per_byte": 3.363479149466621,
      "learning_rate": 0.00021749999999999997,
      "loss_nats": 2.331386089324951,
      "step": 290,
      "unclipped_grad_norm": 0.7895627617835999
    },
    {
      "bits_per_byte": 3.4037392561376993,
      "learning_rate": 0.000225,
      "loss_nats": 2.3592922687530518,
      "step": 300,
      "unclipped_grad_norm": 1.5281883478164673
    },
    {
      "bits_per_byte": 3.3541442751430894,
      "learning_rate": 0.00023249999999999996,
      "loss_nats": 2.324915647506714,
      "step": 310,
      "unclipped_grad_norm": 0.8648361563682556
    },
    {
      "bits_per_byte": 3.3434108379027414,
      "learning_rate": 0.00023999999999999998,
      "loss_nats": 2.3174757957458496,
      "step": 320,
      "unclipped_grad_norm": 1.4613573551177979
    },
    {
      "bits_per_byte": 3.3325662998699364,
      "learning_rate": 0.0002475,
      "loss_nats": 2.3099589347839355,
      "step": 330,
      "unclipped_grad_norm": 1.1215400695800781
    },
    {
      "bits_per_byte": 3.4180712583647006,
      "learning_rate": 0.00025499999999999996,
      "loss_nats": 2.3692264556884766,
      "step": 340,
      "unclipped_grad_norm": 1.1166281700134277
    },
    {
      "bits_per_byte": 3.3577222022116393,
      "learning_rate": 0.0002625,
      "loss_nats": 2.3273956775665283,
      "step": 350,
      "unclipped_grad_norm": 1.0810235738754272
    },
    {
      "bits_per_byte": 3.2923828974611427,
      "learning_rate": 0.00026999999999999995,
      "loss_nats": 2.2821059226989746,
      "step": 360,
      "unclipped_grad_norm": 1.128420114517212
    },
    {
      "bits_per_byte": 3.291989401155969,
      "learning_rate": 0.00027749999999999997,
      "loss_nats": 2.2818331718444824,
      "step": 370,
      "unclipped_grad_norm": 1.0846341848373413
    },
    {
      "bits_per_byte": 3.3389695579268603,
      "learning_rate": 0.000285,
      "loss_nats": 2.3143973350524902,
      "step": 380,
      "unclipped_grad_norm": 1.0868512392044067
    },
    {
      "bits_per_byte": 3.256548592327368,
      "learning_rate": 0.0002925,
      "loss_nats": 2.257267475128174,
      "step": 390,
      "unclipped_grad_norm": 1.1476690769195557
    },
    {
      "bits_per_byte": 3.2058776878689605,
      "learning_rate": 0.0003,
      "loss_nats": 2.2221450805664062,
      "step": 400,
      "unclipped_grad_norm": 1.1263198852539062
    },
    {
      "bits_per_byte": 3.195324144481415,
      "learning_rate": 0.0002999988466112388,
      "loss_nats": 2.214829921722412,
      "step": 410,
      "unclipped_grad_norm": 1.1377208232879639
    },
    {
      "bits_per_byte": 3.227808915506986,
      "learning_rate": 0.00029999538646466355,
      "loss_nats": 2.237346649169922,
      "step": 420,
      "unclipped_grad_norm": 1.2929860353469849
    },
    {
      "bits_per_byte": 3.1876103786249166,
      "learning_rate": 0.0002999896196193985,
      "loss_nats": 2.2094831466674805,
      "step": 430,
      "unclipped_grad_norm": 1.071694016456604
    },
    {
      "bits_per_byte": 3.2374495749837515,
      "learning_rate": 0.00029998154617398326,
      "loss_nats": 2.2440290451049805,
      "step": 440,
      "unclipped_grad_norm": 0.9089367985725403
    },
    {
      "bits_per_byte": 3.155426921203657,
      "learning_rate": 0.0002999711662663707,
      "loss_nats": 2.1871752738952637,
      "step": 450,
      "unclipped_grad_norm": 1.1129212379455566
    },
    {
      "bits_per_byte": 3.188086770567806,
      "learning_rate": 0.0002999584800739247,
      "loss_nats": 2.209813356399536,
      "step": 460,
      "unclipped_grad_norm": 1.3905333280563354
    },
    {
      "bits_per_byte": 3.128408790718412,
      "learning_rate": 0.0002999434878134173,
      "loss_nats": 2.168447732925415,
      "step": 470,
      "unclipped_grad_norm": 1.059533953666687
    },
    {
      "bits_per_byte": 3.136691131218663,
      "learning_rate": 0.000299926189741025,
      "loss_nats": 2.1741886138916016,
      "step": 480,
      "unclipped_grad_norm": 1.0161854028701782
    },
    {
      "bits_per_byte": 3.0887341129908705,
      "learning_rate": 0.000299906586152324,
      "loss_nats": 2.1409473419189453,
      "step": 490,
      "unclipped_grad_norm": 1.385905385017395
    },
    {
      "bits_per_byte": 3.033835874974255,
      "learning_rate": 0.0002998846773822856,
      "loss_nats": 2.1028947830200195,
      "step": 500,
      "unclipped_grad_norm": 1.1716443300247192
    },
    {
      "bits_per_byte": 3.0607071322756636,
      "learning_rate": 0.0002998604638052702,
      "loss_nats": 2.121520519256592,
      "step": 510,
      "unclipped_grad_norm": 0.947066605091095
    },
    {
      "bits_per_byte": 2.997100036784657,
      "learning_rate": 0.0002998339458350211,
      "loss_nats": 2.0774314403533936,
      "step": 520,
      "unclipped_grad_norm": 1.352820634841919
    },
    {
      "bits_per_byte": 3.0643050093317137,
      "learning_rate": 0.0002998051239246572,
      "loss_nats": 2.124014377593994,
      "step": 530,
      "unclipped_grad_norm": 1.2646857500076294
    },
    {
      "bits_per_byte": 3.008989197438803,
      "learning_rate": 0.00029977399856666537,
      "loss_nats": 2.085672378540039,
      "step": 540,
      "unclipped_grad_norm": 1.0451104640960693
    },
    {
      "bits_per_byte": 2.978105584892765,
      "learning_rate": 0.0002997405702928922,
      "loss_nats": 2.064265489578247,
      "step": 550,
      "unclipped_grad_norm": 1.2619587182998657
    },
    {
      "bits_per_byte": 2.968300166036473,
      "learning_rate": 0.0002997048396745345,
      "loss_nats": 2.057468891143799,
      "step": 560,
      "unclipped_grad_norm": 0.9861598610877991
    },
    {
      "bits_per_byte": 2.9221661639078262,
      "learning_rate": 0.0002996668073221302,
      "loss_nats": 2.025491237640381,
      "step": 570,
      "unclipped_grad_norm": 1.1261875629425049
    },
    {
      "bits_per_byte": 2.96999178739036,
      "learning_rate": 0.000299626473885547,
      "loss_nats": 2.0586414337158203,
      "step": 580,
      "unclipped_grad_norm": 0.8640331625938416
    },
    {
      "bits_per_byte": 2.883698804389342,
      "learning_rate": 0.00029958384005397226,
      "loss_nats": 1.9988276958465576,
      "step": 590,
      "unclipped_grad_norm": 1.0034456253051758
    },
    {
      "bits_per_byte": 2.9404706214044567,
      "learning_rate": 0.0002995389065559004,
      "loss_nats": 2.0381789207458496,
      "step": 600,
      "unclipped_grad_norm": 0.8714175224304199
    },
    {
      "bits_per_byte": 2.8380133330141457,
      "learning_rate": 0.000299491674159121,
      "loss_nats": 1.967160940170288,
      "step": 610,
      "unclipped_grad_norm": 0.9427422881126404
    },
    {
      "bits_per_byte": 2.894644124255553,
      "learning_rate": 0.00029944214367070535,
      "loss_nats": 2.0064144134521484,
      "step": 620,
      "unclipped_grad_norm": 1.0238139629364014
    },
    {
      "bits_per_byte": 2.81883675149499,
      "learning_rate": 0.00029939031593699284,
      "loss_nats": 1.9538687467575073,
      "step": 630,
      "unclipped_grad_norm": 0.9875481128692627
    },
    {
      "bits_per_byte": 2.7787291934352267,
      "learning_rate": 0.00029933619184357654,
      "loss_nats": 1.9260683059692383,
      "step": 640,
      "unclipped_grad_norm": 1.0125378370285034
    },
    {
      "bits_per_byte": 2.728423580127252,
      "learning_rate": 0.00029927977231528774,
      "loss_nats": 1.8911991119384766,
      "step": 650,
      "unclipped_grad_norm": 1.5507988929748535
    },
    {
      "bits_per_byte": 2.649404087396524,
      "learning_rate": 0.00029922105831618057,
      "loss_nats": 1.8364269733428955,
      "step": 660,
      "unclipped_grad_norm": 0.9040125012397766
    },
    {
      "bits_per_byte": 2.7024973674923345,
      "learning_rate": 0.0002991600508495153,
      "loss_nats": 1.8732284307479858,
      "step": 670,
      "unclipped_grad_norm": 1.205861210823059
    },
    {
      "bits_per_byte": 2.641403626461037,
      "learning_rate": 0.00029909675095774114,
      "loss_nats": 1.8308814764022827,
      "step": 680,
      "unclipped_grad_norm": 1.3370649814605713
    },
    {
      "bits_per_byte": 2.7325929555321187,
      "learning_rate": 0.0002990311597224787,
      "loss_nats": 1.8940891027450562,
      "step": 690,
      "unclipped_grad_norm": 1.6124995946884155
    },
    {
      "bits_per_byte": 2.6130706886099455,
      "learning_rate": 0.00029896327826450103,
      "loss_nats": 1.8112425804138184,
      "step": 700,
      "unclipped_grad_norm": 1.08383309841156
    },
    {
      "bits_per_byte": 2.628106443844277,
      "learning_rate": 0.0002988931077437152,
      "loss_nats": 1.821664571762085,
      "step": 710,
      "unclipped_grad_norm": 1.0767779350280762
    },
    {
      "bits_per_byte": 2.6725746220166666,
      "learning_rate": 0.00029882064935914164,
      "loss_nats": 1.852487564086914,
      "step": 720,
      "unclipped_grad_norm": 1.296352505683899
    },
    {
      "bits_per_byte": 2.608218370098502,
      "learning_rate": 0.0002987459043488943,
      "loss_nats": 1.8078792095184326,
      "step": 730,
      "unclipped_grad_norm": 1.0864766836166382
    },
    {
      "bits_per_byte": 2.555699340073923,
      "learning_rate": 0.0002986688739901591,
      "loss_nats": 1.7714757919311523,
      "step": 740,
      "unclipped_grad_norm": 1.0339913368225098
    },
    {
      "bits_per_byte": 2.515936435246489,
      "learning_rate": 0.0002985895595991723,
      "loss_nats": 1.743914246559143,
      "step": 750,
      "unclipped_grad_norm": 1.1017143726348877
    },
    {
      "bits_per_byte": 2.4657125136976736,
      "learning_rate": 0.00029850796253119824,
      "loss_nats": 1.709101676940918,
      "step": 760,
      "unclipped_grad_norm": 1.7173939943313599
    },
    {
      "bits_per_byte": 2.601046349592221,
      "learning_rate": 0.00029842408418050544,
      "loss_nats": 1.802907943725586,
      "step": 770,
      "unclipped_grad_norm": 0.9487742781639099
    },
    {
      "bits_per_byte": 2.4794518637098015,
      "learning_rate": 0.0002983379259803436,
      "loss_nats": 1.7186250686645508,
      "step": 780,
      "unclipped_grad_norm": 0.9247332811355591
    },
    {
      "bits_per_byte": 2.466098098800908,
      "learning_rate": 0.0002982494894029186,
      "loss_nats": 1.7093689441680908,
      "step": 790,
      "unclipped_grad_norm": 1.9121034145355225
    },
    {
      "bits_per_byte": 2.4025570446478333,
      "learning_rate": 0.0002981587759593675,
      "loss_nats": 1.66532564163208,
      "step": 800,
      "unclipped_grad_norm": 1.1767630577087402
    },
    {
      "bits_per_byte": 2.455517898102479,
      "learning_rate": 0.00029806578719973283,
      "loss_nats": 1.7020353078842163,
      "step": 810,
      "unclipped_grad_norm": 1.1299240589141846
    },
    {
      "bits_per_byte": 2.400107839716896,
      "learning_rate": 0.00029797052471293596,
      "loss_nats": 1.6636279821395874,
      "step": 820,
      "unclipped_grad_norm": 1.0006046295166016
    },
    {
      "bits_per_byte": 2.4500607166079513,
      "learning_rate": 0.0002978729901267499,
      "loss_nats": 1.6982526779174805,
      "step": 830,
      "unclipped_grad_norm": 1.0554603338241577
    },
    {
      "bits_per_byte": 2.5153210813217015,
      "learning_rate": 0.0002977731851077716,
      "loss_nats": 1.7434877157211304,
      "step": 840,
      "unclipped_grad_norm": 1.0736713409423828
    },
    {
      "bits_per_byte": 2.399556119372928,
      "learning_rate": 0.0002976711113613935,
      "loss_nats": 1.6632455587387085,
      "step": 850,
      "unclipped_grad_norm": 0.8279580473899841
    },
    {
      "bits_per_byte": 2.414819407654351,
      "learning_rate": 0.0002975667706317743,
      "loss_nats": 1.6738252639770508,
      "step": 860,
      "unclipped_grad_norm": 1.2342168092727661
    },
    {
      "bits_per_byte": 2.3222158652452247,
      "learning_rate": 0.00029746016470180915,
      "loss_nats": 1.6096373796463013,
      "step": 870,
      "unclipped_grad_norm": 1.0648512840270996
    },
    {
      "bits_per_byte": 2.3629336177502487,
      "learning_rate": 0.00029735129539309926,
      "loss_nats": 1.6378607749938965,
      "step": 880,
      "unclipped_grad_norm": 0.9258757829666138
    },
    {
      "bits_per_byte": 2.2961121665146167,
      "learning_rate": 0.0002972401645659207,
      "loss_nats": 1.5915436744689941,
      "step": 890,
      "unclipped_grad_norm": 1.0717874765396118
    },
    {
      "bits_per_byte": 2.3158979105831285,
      "learning_rate": 0.00029712677411919283,
      "loss_nats": 1.6052581071853638,
      "step": 900,
      "unclipped_grad_norm": 0.9598665833473206
    },
    {
      "bits_per_byte": 2.3301258633063577,
      "learning_rate": 0.0002970111259904454,
      "loss_nats": 1.6151201725006104,
      "step": 910,
      "unclipped_grad_norm": 1.0995051860809326
    },
    {
      "bits_per_byte": 2.3123376977276173,
      "learning_rate": 0.0002968932221557859,
      "loss_nats": 1.602790355682373,
      "step": 920,
      "unclipped_grad_norm": 1.1837356090545654
    },
    {
      "bits_per_byte": 2.3340250539667196,
      "learning_rate": 0.0002967730646298655,
      "loss_nats": 1.6178228855133057,
      "step": 930,
      "unclipped_grad_norm": 1.1502224206924438
    },
    {
      "bits_per_byte": 2.330418921743428,
      "learning_rate": 0.0002966506554658449,
      "loss_nats": 1.6153233051300049,
      "step": 940,
      "unclipped_grad_norm": 1.0088714361190796
    },
    {
      "bits_per_byte": 2.2995397807463123,
      "learning_rate": 0.00029652599675535876,
      "loss_nats": 1.5939195156097412,
      "step": 950,
      "unclipped_grad_norm": 0.9483612179756165
    },
    {
      "bits_per_byte": 2.3536873144919115,
      "learning_rate": 0.0002963990906284808,
      "loss_nats": 1.6314517259597778,
      "step": 960,
      "unclipped_grad_norm": 0.9582105278968811
    },
    {
      "bits_per_byte": 2.2526236015215417,
      "learning_rate": 0.0002962699392536863,
      "loss_nats": 1.5613996982574463,
      "step": 970,
      "unclipped_grad_norm": 1.2002545595169067
    },
    {
      "bits_per_byte": 2.3230659754884395,
      "learning_rate": 0.0002961385448378161,
      "loss_nats": 1.6102266311645508,
      "step": 980,
      "unclipped_grad_norm": 0.9191399812698364
    },
    {
      "bits_per_byte": 2.285470911975131,
      "learning_rate": 0.00029600490962603817,
      "loss_nats": 1.584167718887329,
      "step": 990,
      "unclipped_grad_norm": 1.053741216659546
    },
    {
      "bits_per_byte": 2.248061417742104,
      "learning_rate": 0.00029586903590180956,
      "loss_nats": 1.5582374334335327,
      "step": 1000,
      "unclipped_grad_norm": 0.9024814963340759
    },
    {
      "bits_per_byte": 2.2402749032242903,
      "learning_rate": 0.00029573092598683735,
      "loss_nats": 1.552840232849121,
      "step": 1010,
      "unclipped_grad_norm": 1.048907995223999
    },
    {
      "bits_per_byte": 2.2041726491204714,
      "learning_rate": 0.00029559058224103895,
      "loss_nats": 1.5278160572052002,
      "step": 1020,
      "unclipped_grad_norm": 1.0486118793487549
    },
    {
      "bits_per_byte": 2.216881995036582,
      "learning_rate": 0.0002954480070625017,
      "loss_nats": 1.5366255044937134,
      "step": 1030,
      "unclipped_grad_norm": 0.9243784546852112
    },
    {
      "bits_per_byte": 2.2610579746851553,
      "learning_rate": 0.00029530320288744205,
      "loss_nats": 1.5672459602355957,
      "step": 1040,
      "unclipped_grad_norm": 1.102870225906372
    },
    {
      "bits_per_byte": 2.2528270569975124,
      "learning_rate": 0.0002951561721901638,
      "loss_nats": 1.5615407228469849,
      "step": 1050,
      "unclipped_grad_norm": 1.0764544010162354
    },
    {
      "bits_per_byte": 2.14889054582713,
      "learning_rate": 0.00029500691748301594,
      "loss_nats": 1.489497423171997,
      "step": 1060,
      "unclipped_grad_norm": 0.971049427986145
    },
    {
      "bits_per_byte": 2.2350196293618754,
      "learning_rate": 0.00029485544131634963,
      "loss_nats": 1.5491975545883179,
      "step": 1070,
      "unclipped_grad_norm": 1.0203380584716797
    },
    {
      "bits_per_byte": 2.217042282867186,
      "learning_rate": 0.0002947017462784745,
      "loss_nats": 1.5367366075515747,
      "step": 1080,
      "unclipped_grad_norm": 1.023221492767334
    },
    {
      "bits_per_byte": 2.186512266651322,
      "learning_rate": 0.00029454583499561476,
      "loss_nats": 1.5155748128890991,
      "step": 1090,
      "unclipped_grad_norm": 1.111006498336792
    },
    {
      "bits_per_byte": 2.2323391077655277,
      "learning_rate": 0.00029438771013186413,
      "loss_nats": 1.5473395586013794,
      "step": 1100,
      "unclipped_grad_norm": 0.9547863602638245
    },
    {
      "bits_per_byte": 2.116467344590568,
      "learning_rate": 0.0002942273743891401,
      "loss_nats": 1.4670233726501465,
      "step": 1110,
      "unclipped_grad_norm": 1.1085364818572998
    },
    {
      "bits_per_byte": 2.143238164023873,
      "learning_rate": 0.00029406483050713824,
      "loss_nats": 1.485579490661621,
      "step": 1120,
      "unclipped_grad_norm": 0.901618242263794
    },
    {
      "bits_per_byte": 2.077939275196329,
      "learning_rate": 0.00029390008126328504,
      "loss_nats": 1.4403177499771118,
      "step": 1130,
      "unclipped_grad_norm": 0.8683149814605713
    },
    {
      "bits_per_byte": 2.224320932617002,
      "learning_rate": 0.00029373312947269045,
      "loss_nats": 1.5417817831039429,
      "step": 1140,
      "unclipped_grad_norm": 1.030465006828308
    },
    {
      "bits_per_byte": 2.0642131678483175,
      "learning_rate": 0.00029356397798809995,
      "loss_nats": 1.4308035373687744,
      "step": 1150,
      "unclipped_grad_norm": 0.827274739742279
    },
    {
      "bits_per_byte": 2.103388407957753,
      "learning_rate": 0.0002933926296998457,
      "loss_nats": 1.4579577445983887,
      "step": 1160,
      "unclipped_grad_norm": 0.9158841967582703
    },
    {
      "bits_per_byte": 2.1566975262803965,
      "learning_rate": 0.0002932190875357972,
      "loss_nats": 1.4949088096618652,
      "step": 1170,
      "unclipped_grad_norm": 0.9397322535514832
    },
    {
      "bits_per_byte": 2.1322636071070153,
      "learning_rate": 0.00029304335446131126,
      "loss_nats": 1.4779725074768066,
      "step": 1180,
      "unclipped_grad_norm": 0.8510220050811768
    },
    {
      "bits_per_byte": 2.068760561119781,
      "learning_rate": 0.00029286543347918116,
      "loss_nats": 1.4339555501937866,
      "step": 1190,
      "unclipped_grad_norm": 0.9270830154418945
    },
    {
      "bits_per_byte": 2.0310528489701576,
      "learning_rate": 0.0002926853276295856,
      "loss_nats": 1.4078185558319092,
      "step": 1200,
      "unclipped_grad_norm": 0.8815417885780334
    },
    {
      "bits_per_byte": 2.1008361854189492,
      "learning_rate": 0.0002925030399900368,
      "loss_nats": 1.456188678741455,
      "step": 1210,
      "unclipped_grad_norm": 1.2037686109542847
    },
    {
      "bits_per_byte": 2.1123913557651197,
      "learning_rate": 0.00029231857367532726,
      "loss_nats": 1.464198112487793,
      "step": 1220,
      "unclipped_grad_norm": 0.8964646458625793
    },
    {
      "bits_per_byte": 2.0711254945517954,
      "learning_rate": 0.0002921319318374776,
      "loss_nats": 1.4355947971343994,
      "step": 1230,
      "unclipped_grad_norm": 0.8661414980888367
    },
    {
      "bits_per_byte": 2.0159510523978943,
      "learning_rate": 0.00029194311766568174,
      "loss_nats": 1.397350788116455,
      "step": 1240,
      "unclipped_grad_norm": 0.9392534494400024
    },
    {
      "bits_per_byte": 2.104896867788471,
      "learning_rate": 0.000291752134386253,
      "loss_nats": 1.4590033292770386,
      "step": 1250,
      "unclipped_grad_norm": 0.8430818915367126
    },
    {
      "bits_per_byte": 2.08180441329182,
      "learning_rate": 0.00029155898526256856,
      "loss_nats": 1.442996859550476,
      "step": 1260,
      "unclipped_grad_norm": 0.8917548656463623
    },
    {
      "bits_per_byte": 2.087352401626003,
      "learning_rate": 0.0002913636735950141,
      "loss_nats": 1.4468424320220947,
      "step": 1270,
      "unclipped_grad_norm": 0.9866762161254883
    },
    {
      "bits_per_byte": 1.9607164163161914,
      "learning_rate": 0.0002911662027209272,
      "loss_nats": 1.359065055847168,
      "step": 1280,
      "unclipped_grad_norm": 0.8596939444541931
    },
    {
      "bits_per_byte": 2.088993460080536,
      "learning_rate": 0.00029096657601454013,
      "loss_nats": 1.4479799270629883,
      "step": 1290,
      "unclipped_grad_norm": 1.007126808166504
    },
    {
      "bits_per_byte": 2.034349412421896,
      "learning_rate": 0.00029076479688692266,
      "loss_nats": 1.4101035594940186,
      "step": 1300,
      "unclipped_grad_norm": 0.8474438786506653
    },
    {
      "bits_per_byte": 2.0247273270714237,
      "learning_rate": 0.00029056086878592323,
      "loss_nats": 1.4034340381622314,
      "step": 1310,
      "unclipped_grad_norm": 0.8226730823516846
    },
    {
      "bits_per_byte": 2.020511000402871,
      "learning_rate": 0.00029035479519611045,
      "loss_nats": 1.4005115032196045,
      "step": 1320,
      "unclipped_grad_norm": 0.9414357542991638
    },
    {
      "bits_per_byte": 2.085571349293668,
      "learning_rate": 0.0002901465796387134,
      "loss_nats": 1.4456079006195068,
      "step": 1330,
      "unclipped_grad_norm": 0.9917200207710266
    },
    {
      "bits_per_byte": 2.0422444479924042,
      "learning_rate": 0.00028993622567156154,
      "loss_nats": 1.4155759811401367,
      "step": 1340,
      "unclipped_grad_norm": 0.8823419213294983
    },
    {
      "bits_per_byte": 2.0447034559344406,
      "learning_rate": 0.0002897237368890237,
      "loss_nats": 1.4172804355621338,
      "step": 1350,
      "unclipped_grad_norm": 0.8861486315727234
    },
    {
      "bits_per_byte": 2.0988803987133373,
      "learning_rate": 0.0002895091169219469,
      "loss_nats": 1.4548330307006836,
      "step": 1360,
      "unclipped_grad_norm": 0.9090827107429504
    },
    {
      "bits_per_byte": 2.082760980795919,
      "learning_rate": 0.00028929236943759425,
      "loss_nats": 1.4436599016189575,
      "step": 1370,
      "unclipped_grad_norm": 0.8366561532020569
    },
    {
      "bits_per_byte": 2.0317923743688677,
      "learning_rate": 0.00028907349813958225,
      "loss_nats": 1.4083311557769775,
      "step": 1380,
      "unclipped_grad_norm": 0.8379060626029968
    },
    {
      "bits_per_byte": 2.0432778917414387,
      "learning_rate": 0.0002888525067678175,
      "loss_nats": 1.4162923097610474,
      "step": 1390,
      "unclipped_grad_norm": 0.8665340542793274
    },
    {
      "bits_per_byte": 1.9746810636009497,
      "learning_rate": 0.00028862939909843273,
      "loss_nats": 1.3687446117401123,
      "step": 1400,
      "unclipped_grad_norm": 0.8112426996231079
    },
    {
      "bits_per_byte": 2.0034977886489185,
      "learning_rate": 0.0002884041789437225,
      "loss_nats": 1.388718843460083,
      "step": 1410,
      "unclipped_grad_norm": 0.8745980262756348
    },
    {
      "bits_per_byte": 2.019574210903621,
      "learning_rate": 0.0002881768501520778,
      "loss_nats": 1.3998621702194214,
      "step": 1420,
      "unclipped_grad_norm": 0.8596643209457397
    },
    {
      "bits_per_byte": 2.0775151659793014,
      "learning_rate": 0.0002879474166079205,
      "loss_nats": 1.4400237798690796,
      "step": 1430,
      "unclipped_grad_norm": 0.913244903087616
    },
    {
      "bits_per_byte": 2.027370700415185,
      "learning_rate": 0.00028771588223163694,
      "loss_nats": 1.405266284942627,
      "step": 1440,
      "unclipped_grad_norm": 1.0067864656448364
    },
    {
      "bits_per_byte": 1.9631205617926026,
      "learning_rate": 0.00028748225097951054,
      "loss_nats": 1.3607314825057983,
      "step": 1450,
      "unclipped_grad_norm": 0.8125852942466736
    },
    {
      "bits_per_byte": 1.9959419028659118,
      "learning_rate": 0.000287246526843655,
      "loss_nats": 1.383481502532959,
      "step": 1460,
      "unclipped_grad_norm": 0.9857028126716614
    },
    {
      "bits_per_byte": 1.9364156117145783,
      "learning_rate": 0.00028700871385194533,
      "loss_nats": 1.3422210216522217,
      "step": 1470,
      "unclipped_grad_norm": 0.8698804974555969
    },
    {
      "bits_per_byte": 1.996789089404062,
      "learning_rate": 0.00028676881606794945,
      "loss_nats": 1.3840687274932861,
      "step": 1480,
      "unclipped_grad_norm": 0.8397032022476196
    },
    {
      "bits_per_byte": 2.017818268038312,
      "learning_rate": 0.00028652683759085854,
      "loss_nats": 1.398645043373108,
      "step": 1490,
      "unclipped_grad_norm": 0.8181726932525635
    },
    {
      "bits_per_byte": 1.996446843928845,
      "learning_rate": 0.0002862827825554171,
      "loss_nats": 1.38383150100708,
      "step": 1500,
      "unclipped_grad_norm": 0.8242432475090027
    },
    {
      "bits_per_byte": 2.0162594172908914,
      "learning_rate": 0.00028603665513185245,
      "loss_nats": 1.3975645303726196,
      "step": 1510,
      "unclipped_grad_norm": 0.9299814701080322
    },
    {
      "bits_per_byte": 1.980743108078562,
      "learning_rate": 0.000285788459525803,
      "loss_nats": 1.3729465007781982,
      "step": 1520,
      "unclipped_grad_norm": 0.8446369171142578
    },
    {
      "bits_per_byte": 2.047233664693934,
      "learning_rate": 0.00028553819997824707,
      "loss_nats": 1.4190342426300049,
      "step": 1530,
      "unclipped_grad_norm": 0.9717140197753906
    },
    {
      "bits_per_byte": 1.993557363411699,
      "learning_rate": 0.00028528588076542964,
      "loss_nats": 1.3818286657333374,
      "step": 1540,
      "unclipped_grad_norm": 0.8417568802833557
    },
    {
      "bits_per_byte": 2.049996737962696,
      "learning_rate": 0.00028503150619878996,
      "loss_nats": 1.4209494590759277,
      "step": 1550,
      "unclipped_grad_norm": 0.8546327352523804
    },
    {
      "bits_per_byte": 1.966114779744124,
      "learning_rate": 0.0002847750806248876,
      "loss_nats": 1.3628069162368774,
      "step": 1560,
      "unclipped_grad_norm": 0.9258830547332764
    },
    {
      "bits_per_byte": 2.0427641795633114,
      "learning_rate": 0.000284516608425328,
      "loss_nats": 1.4159362316131592,
      "step": 1570,
      "unclipped_grad_norm": 0.8567822575569153
    },
    {
      "bits_per_byte": 1.9830621221427962,
      "learning_rate": 0.000284256094016688,
      "loss_nats": 1.374553918838501,
      "step": 1580,
      "unclipped_grad_norm": 0.8522606492042542
    },
    {
      "bits_per_byte": 1.9876789964052082,
      "learning_rate": 0.0002839935418504399,
      "loss_nats": 1.3777540922164917,
      "step": 1590,
      "unclipped_grad_norm": 0.8731296062469482
    },
    {
      "bits_per_byte": 2.0122188568915207,
      "learning_rate": 0.000283728956412876,
      "loss_nats": 1.3947638273239136,
      "step": 1600,
      "unclipped_grad_norm": 0.8627029657363892
    },
    {
      "bits_per_byte": 1.9498011934138815,
      "learning_rate": 0.00028346234222503136,
      "loss_nats": 1.3514991998672485,
      "step": 1610,
      "unclipped_grad_norm": 0.7740172147750854
    },
    {
      "bits_per_byte": 1.9095543294069197,
      "learning_rate": 0.00028319370384260675,
      "loss_nats": 1.3236021995544434,
      "step": 1620,
      "unclipped_grad_norm": 0.7910023331642151
    },
    {
      "bits_per_byte": 1.9407452749500496,
      "learning_rate": 0.00028292304585589105,
      "loss_nats": 1.3452221155166626,
      "step": 1630,
      "unclipped_grad_norm": 0.8484107851982117
    },
    {
      "bits_per_byte": 1.9911346438089947,
      "learning_rate": 0.0002826503728896824,
      "loss_nats": 1.3801493644714355,
      "step": 1640,
      "unclipped_grad_norm": 0.8722788691520691
    },
    {
      "bits_per_byte": 2.0368601871418424,
      "learning_rate": 0.0002823756896032095,
      "loss_nats": 1.4118438959121704,
      "step": 1650,
      "unclipped_grad_norm": 0.8482462763786316
    },
    {
      "bits_per_byte": 1.9665676100638458,
      "learning_rate": 0.0002820990006900518,
      "loss_nats": 1.3631207942962646,
      "step": 1660,
      "unclipped_grad_norm": 0.8444085121154785
    },
    {
      "bits_per_byte": 1.9945896032821775,
      "learning_rate": 0.00028182031087805947,
      "loss_nats": 1.3825441598892212,
      "step": 1670,
      "unclipped_grad_norm": 0.8560684323310852
    },
    {
      "bits_per_byte": 2.0061820938635853,
      "learning_rate": 0.0002815396249292725,
      "loss_nats": 1.3905794620513916,
      "step": 1680,
      "unclipped_grad_norm": 0.8526912927627563
    },
    {
      "bits_per_byte": 2.0514603103215387,
      "learning_rate": 0.0002812569476398391,
      "loss_nats": 1.4219639301300049,
      "step": 1690,
      "unclipped_grad_norm": 0.8917502760887146
    },
    {
      "bits_per_byte": 1.9422809080396035,
      "learning_rate": 0.00028097228383993446,
      "loss_nats": 1.3462865352630615,
      "step": 1700,
      "unclipped_grad_norm": 0.7474871873855591
    },
    {
      "bits_per_byte": 1.878015978909047,
      "learning_rate": 0.0002806856383936774,
      "loss_nats": 1.3017414808273315,
      "step": 1710,
      "unclipped_grad_norm": 0.8072315454483032
    },
    {
      "bits_per_byte": 1.8905051870320533,
      "learning_rate": 0.00028039701619904774,
      "loss_nats": 1.3103983402252197,
      "step": 1720,
      "unclipped_grad_norm": 0.845556914806366
    },
    {
      "bits_per_byte": 1.8762399140735866,
      "learning_rate": 0.0002801064221878024,
      "loss_nats": 1.3005104064941406,
      "step": 1730,
      "unclipped_grad_norm": 0.9344940185546875
    },
    {
      "bits_per_byte": 1.9632232354351677,
      "learning_rate": 0.00027981386132539134,
      "loss_nats": 1.3608026504516602,
      "step": 1740,
      "unclipped_grad_norm": 0.7860697507858276
    },
    {
      "bits_per_byte": 1.989613629244764,
      "learning_rate": 0.0002795193386108724,
      "loss_nats": 1.3790950775146484,
      "step": 1750,
      "unclipped_grad_norm": 0.8348115086555481
    },
    {
      "bits_per_byte": 1.9869124697303127,
      "learning_rate": 0.00027922285907682625,
      "loss_nats": 1.3772227764129639,
      "step": 1760,
      "unclipped_grad_norm": 0.8862095475196838
    },
    {
      "bits_per_byte": 1.9928130224987648,
      "learning_rate": 0.00027892442778927005,
      "loss_nats": 1.3813127279281616,
      "step": 1770,
      "unclipped_grad_norm": 0.816246509552002
    },
    {
      "bits_per_byte": 1.8514405317671223,
      "learning_rate": 0.00027862404984757116,
      "loss_nats": 1.2833207845687866,
      "step": 1780,
      "unclipped_grad_norm": 0.8118871450424194
    },
    {
      "bits_per_byte": 1.9588221993995885,
      "learning_rate": 0.0002783217303843597,
      "loss_nats": 1.3577520847320557,
      "step": 1790,
      "unclipped_grad_norm": 0.839783251285553
    },
    {
      "bits_per_byte": 1.8827697513615458,
      "learning_rate": 0.00027801747456544134,
      "loss_nats": 1.3050365447998047,
      "step": 1800,
      "unclipped_grad_norm": 0.8949951529502869
    },
    {
      "bits_per_byte": 1.92937567388416,
      "learning_rate": 0.00027771128758970843,
      "loss_nats": 1.33734130859375,
      "step": 1810,
      "unclipped_grad_norm": 0.7295374274253845
    },
    {
      "bits_per_byte": 1.8974312723476003,
      "learning_rate": 0.00027740317468905165,
      "loss_nats": 1.3151991367340088,
      "step": 1820,
      "unclipped_grad_norm": 0.7755441069602966
    },
    {
      "bits_per_byte": 1.8886371114783813,
      "learning_rate": 0.0002770931411282704,
      "loss_nats": 1.3091034889221191,
      "step": 1830,
      "unclipped_grad_norm": 0.7666956186294556
    },
    {
      "bits_per_byte": 1.8876932706904463,
      "learning_rate": 0.0002767811922049827,
      "loss_nats": 1.3084492683410645,
      "step": 1840,
      "unclipped_grad_norm": 0.8784642219543457
    },
    {
      "bits_per_byte": 1.8964055678178544,
      "learning_rate": 0.00027646733324953503,
      "loss_nats": 1.314488172531128,
      "step": 1850,
      "unclipped_grad_norm": 0.8603965044021606
    },
    {
      "bits_per_byte": 1.8927124123732264,
      "learning_rate": 0.00027615156962491097,
      "loss_nats": 1.3119282722473145,
      "step": 1860,
      "unclipped_grad_norm": 0.8056252002716064
    },
    {
      "bits_per_byte": 1.8818489562488263,
      "learning_rate": 0.0002758339067266396,
      "loss_nats": 1.3043982982635498,
      "step": 1870,
      "unclipped_grad_norm": 0.8151036500930786
    },
    {
      "bits_per_byte": 1.8811881989042114,
      "learning_rate": 0.0002755143499827035,
      "loss_nats": 1.3039402961730957,
      "step": 1880,
      "unclipped_grad_norm": 0.79389888048172
    },
    {
      "bits_per_byte": 1.8949089747900463,
      "learning_rate": 0.0002751929048534456,
      "loss_nats": 1.313450813293457,
      "step": 1890,
      "unclipped_grad_norm": 0.8192400932312012
    },
    {
      "bits_per_byte": 1.9233574849825177,
      "learning_rate": 0.0002748695768314763,
      "loss_nats": 1.3331698179244995,
      "step": 1900,
      "unclipped_grad_norm": 0.7950363159179688
    },
    {
      "bits_per_byte": 1.8611783614416253,
      "learning_rate": 0.00027454437144157947,
      "loss_nats": 1.2900705337524414,
      "step": 1910,
      "unclipped_grad_norm": 0.8037525415420532
    },
    {
      "bits_per_byte": 1.855479888287937,
      "learning_rate": 0.00027421729424061787,
      "loss_nats": 1.2861206531524658,
      "step": 1920,
      "unclipped_grad_norm": 0.7507341504096985
    },
    {
      "bits_per_byte": 1.9602173226633874,
      "learning_rate": 0.0002738883508174385,
      "loss_nats": 1.3587191104888916,
      "step": 1930,
      "unclipped_grad_norm": 0.8329893350601196
    },
    {
      "bits_per_byte": 1.8066889261861108,
      "learning_rate": 0.0002735575467927768,
      "loss_nats": 1.2523013353347778,
      "step": 1940,
      "unclipped_grad_norm": 0.7595474123954773
    },
    {
      "bits_per_byte": 1.9033319970987033,
      "learning_rate": 0.0002732248878191609,
      "loss_nats": 1.319289207458496,
      "step": 1950,
      "unclipped_grad_norm": 0.7690061330795288
    },
    {
      "bits_per_byte": 1.816397862775269,
      "learning_rate": 0.0002728903795808148,
      "loss_nats": 1.259031057357788,
      "step": 1960,
      "unclipped_grad_norm": 0.7758741974830627
    },
    {
      "bits_per_byte": 1.909067618504978,
      "learning_rate": 0.0002725540277935614,
      "loss_nats": 1.3232648372650146,
      "step": 1970,
      "unclipped_grad_norm": 0.7503845691680908
    },
    {
      "bits_per_byte": 1.9405849871194454,
      "learning_rate": 0.00027221583820472456,
      "loss_nats": 1.3451110124588013,
      "step": 1980,
      "unclipped_grad_norm": 0.8655099868774414
    },
    {
      "bits_per_byte": 1.9436471382180576,
      "learning_rate": 0.0002718758165930314,
      "loss_nats": 1.347233533859253,
      "step": 1990,
      "unclipped_grad_norm": 0.8436980247497559
    },
    {
      "bits_per_byte": 1.9015801818170146,
      "learning_rate": 0.0002715339687685131,
      "loss_nats": 1.3180749416351318,
      "step": 2000,
      "unclipped_grad_norm": 0.8021923303604126
    },
    {
      "bits_per_byte": 1.8746163978494417,
      "learning_rate": 0.00027119030057240576,
      "loss_nats": 1.2993850708007812,
      "step": 2010,
      "unclipped_grad_norm": 0.7083144187927246
    },
    {
      "bits_per_byte": 1.9072580170525995,
      "learning_rate": 0.0002708448178770506,
      "loss_nats": 1.3220105171203613,
      "step": 2020,
      "unclipped_grad_norm": 0.7678324580192566
    },
    {
      "bits_per_byte": 1.983610230851096,
      "learning_rate": 0.00027049752658579376,
      "loss_nats": 1.3749338388442993,
      "step": 2030,
      "unclipped_grad_norm": 0.8317387104034424
    },
    {
      "bits_per_byte": 1.909348466173837,
      "learning_rate": 0.00027014843263288514,
      "loss_nats": 1.323459506034851,
      "step": 2040,
      "unclipped_grad_norm": 0.8302199840545654
    },
    {
      "bits_per_byte": 1.961244747019642,
      "learning_rate": 0.00026979754198337724,
      "loss_nats": 1.359431266784668,
      "step": 2050,
      "unclipped_grad_norm": 0.7990598082542419
    },
    {
      "bits_per_byte": 1.9220292629699043,
      "learning_rate": 0.00026944486063302305,
      "loss_nats": 1.3322491645812988,
      "step": 2060,
      "unclipped_grad_norm": 0.8404503464698792
    },
    {
      "bits_per_byte": 1.8663830724046866,
      "learning_rate": 0.0002690903946081738,
      "loss_nats": 1.2936781644821167,
      "step": 2070,
      "unclipped_grad_norm": 0.7720745801925659
    },
    {
      "bits_per_byte": 1.8672120287818452,
      "learning_rate": 0.00026873414996567596,
      "loss_nats": 1.2942527532577515,
      "step": 2080,
      "unclipped_grad_norm": 0.8028935790061951
    },
    {
      "bits_per_byte": 1.857879218250124,
      "learning_rate": 0.00026837613279276736,
      "loss_nats": 1.2877837419509888,
      "step": 2090,
      "unclipped_grad_norm": 0.7713416814804077
    },
    {
      "bits_per_byte": 1.8603779541845098,
      "learning_rate": 0.0002680163492069739,
      "loss_nats": 1.289515733718872,
      "step": 2100,
      "unclipped_grad_norm": 0.7443708181381226
    },
    {
      "bits_per_byte": 1.897931741861611,
      "learning_rate": 0.00026765480535600414,
      "loss_nats": 1.3155460357666016,
      "step": 2110,
      "unclipped_grad_norm": 0.7534405589103699
    },
    {
      "bits_per_byte": 1.8710701155886507,
      "learning_rate": 0.0002672915074176452,
      "loss_nats": 1.2969269752502441,
      "step": 2120,
      "unclipped_grad_norm": 0.7505779266357422
    },
    {
      "bits_per_byte": 1.8349618420920515,
      "learning_rate": 0.00026692646159965624,
      "loss_nats": 1.271898627281189,
      "step": 2130,
      "unclipped_grad_norm": 0.7835680246353149
    },
    {
      "bits_per_byte": 1.9616896661374241,
      "learning_rate": 0.0002665596741396633,
      "loss_nats": 1.3597396612167358,
      "step": 2140,
      "unclipped_grad_norm": 0.7786996960639954
    },
    {
      "bits_per_byte": 1.8168395142226847,
      "learning_rate": 0.000266191151305052,
      "loss_nats": 1.2593371868133545,
      "step": 2150,
      "unclipped_grad_norm": 0.7194197773933411
    },
    {
      "bits_per_byte": 1.8739360344826286,
      "learning_rate": 0.00026582089939286075,
      "loss_nats": 1.2989134788513184,
      "step": 2160,
      "unclipped_grad_norm": 0.8012002110481262
    },
    {
      "bits_per_byte": 1.9634492206384013,
      "learning_rate": 0.0002654489247296732,
      "loss_nats": 1.3609592914581299,
      "step": 2170,
      "unclipped_grad_norm": 0.8270084261894226
    },
    {
      "bits_per_byte": 1.8175344961148212,
      "learning_rate": 0.00026507523367151007,
      "loss_nats": 1.2598189115524292,
      "step": 2180,
      "unclipped_grad_norm": 0.7824540734291077
    },
    {
      "bits_per_byte": 1.8458584908680682,
      "learning_rate": 0.00026469983260372037,
      "loss_nats": 1.279451608657837,
      "step": 2190,
      "unclipped_grad_norm": 0.7973959445953369
    },
    {
      "bits_per_byte": 1.8557901449900935,
      "learning_rate": 0.0002643227279408727,
      "loss_nats": 1.2863357067108154,
      "step": 2200,
      "unclipped_grad_norm": 0.7451788783073425
    },
    {
      "bits_per_byte": 1.8380872828061812,
      "learning_rate": 0.0002639439261266453,
      "loss_nats": 1.2740650177001953,
      "step": 2210,
      "unclipped_grad_norm": 0.7804975509643555
    },
    {
      "bits_per_byte": 1.8298283319464475,
      "learning_rate": 0.000263563433633716,
      "loss_nats": 1.2683403491973877,
      "step": 2220,
      "unclipped_grad_norm": 0.7694668173789978
    },
    {
      "bits_per_byte": 1.9031531351418058,
      "learning_rate": 0.00026318125696365156,
      "loss_nats": 1.3191652297973633,
      "step": 2230,
      "unclipped_grad_norm": 0.8418469429016113
    },
    {
      "bits_per_byte": 1.8400618156207371,
      "learning_rate": 0.000262797402646797,
      "loss_nats": 1.2754336595535278,
      "step": 2240,
      "unclipped_grad_norm": 0.7987614870071411
    },
    {
      "bits_per_byte": 1.852736936989326,
      "learning_rate": 0.00026241187724216334,
      "loss_nats": 1.2842193841934204,
      "step": 2250,
      "unclipped_grad_norm": 0.7795421481132507
    },
    {
      "bits_per_byte": 1.8339105121473271,
      "learning_rate": 0.0002620246873373162,
      "loss_nats": 1.271169900894165,
      "step": 2260,
      "unclipped_grad_norm": 0.7681040167808533
    },
    {
      "bits_per_byte": 1.825916414569922,
      "learning_rate": 0.00026163583954826264,
      "loss_nats": 1.2656288146972656,
      "step": 2270,
      "unclipped_grad_norm": 0.7933830618858337
    },
    {
      "bits_per_byte": 1.8451832869807807,
      "learning_rate": 0.0002612453405193386,
      "loss_nats": 1.2789835929870605,
      "step": 2280,
      "unclipped_grad_norm": 0.7935850024223328
    },
    {
      "bits_per_byte": 1.8711039961708706,
      "learning_rate": 0.000260853196923095,
      "loss_nats": 1.2969504594802856,
      "step": 2290,
      "unclipped_grad_norm": 0.7818295955657959
    },
    {
      "bits_per_byte": 1.7944778140095496,
      "learning_rate": 0.0002604594154601839,
      "loss_nats": 1.2438372373580933,
      "step": 2300,
      "unclipped_grad_norm": 0.7245032787322998
    },
    {
      "bits_per_byte": 1.8221434591752936,
      "learning_rate": 0.0002600640028592441,
      "loss_nats": 1.2630136013031006,
      "step": 2310,
      "unclipped_grad_norm": 0.7396021485328674
    },
    {
      "bits_per_byte": 1.7928696042413317,
      "learning_rate": 0.0002596669658767859,
      "loss_nats": 1.242722511291504,
      "step": 2320,
      "unclipped_grad_norm": 0.7618010640144348
    },
    {
      "bits_per_byte": 1.8702191454321815,
      "learning_rate": 0.000259268311297076,
      "loss_nats": 1.2963371276855469,
      "step": 2330,
      "unclipped_grad_norm": 0.7263991236686707
    },
    {
      "bits_per_byte": 1.7994940319872652,
      "learning_rate": 0.00025886804593202114,
      "loss_nats": 1.247314214706421,
      "step": 2340,
      "unclipped_grad_norm": 0.7788637280464172
    },
    {
      "bits_per_byte": 1.849551646312696,
      "learning_rate": 0.00025846617662105193,
      "loss_nats": 1.2820115089416504,
      "step": 2350,
      "unclipped_grad_norm": 0.7732409238815308
    },
    {
      "bits_per_byte": 1.8613248906601605,
      "learning_rate": 0.00025806271023100637,
      "loss_nats": 1.2901721000671387,
      "step": 2360,
      "unclipped_grad_norm": 0.8177725672721863
    },
    {
      "bits_per_byte": 1.8822589628884834,
      "learning_rate": 0.00025765765365601165,
      "loss_nats": 1.3046824932098389,
      "step": 2370,
      "unclipped_grad_norm": 0.7931574583053589
    },
    {
      "bits_per_byte": 1.8382422391746087,
      "learning_rate": 0.00025725101381736707,
      "loss_nats": 1.2741724252700806,
      "step": 2380,
      "unclipped_grad_norm": 0.8604201674461365
    },
    {
      "bits_per_byte": 1.8797817247854554,
      "learning_rate": 0.00025684279766342567,
      "loss_nats": 1.3029654026031494,
      "step": 2390,
      "unclipped_grad_norm": 0.7636454105377197
    },
    {
      "bits_per_byte": 1.817860919186159,
      "learning_rate": 0.00025643301216947504,
      "loss_nats": 1.2600451707839966,
      "step": 2400,
      "unclipped_grad_norm": 0.779283881187439
    },
    {
      "bits_per_byte": 1.8354538844561699,
      "learning_rate": 0.00025602166433761866,
      "loss_nats": 1.2722396850585938,
      "step": 2410,
      "unclipped_grad_norm": 0.7781111001968384
    },
    {
      "bits_per_byte": 1.8069649583407457,
      "learning_rate": 0.0002556087611966561,
      "loss_nats": 1.2524926662445068,
      "step": 2420,
      "unclipped_grad_norm": 0.7548633813858032
    },
    {
      "bits_per_byte": 1.8169932667125561,
      "learning_rate": 0.00025519430980196277,
      "loss_nats": 1.259443759918213,
      "step": 2430,
      "unclipped_grad_norm": 0.7562755942344666
    },
    {
      "bits_per_byte": 1.8760644917697067,
      "learning_rate": 0.0002547783172353695,
      "loss_nats": 1.3003888130187988,
      "step": 2440,
      "unclipped_grad_norm": 0.7478408813476562
    },
    {
      "bits_per_byte": 1.838183937055966,
      "learning_rate": 0.0002543607906050415,
      "loss_nats": 1.2741320133209229,
      "step": 2450,
      "unclipped_grad_norm": 0.760445237159729
    },
    {
      "bits_per_byte": 1.8195946762895068,
      "learning_rate": 0.0002539417370453568,
      "loss_nats": 1.261246919631958,
      "step": 2460,
      "unclipped_grad_norm": 0.7331485748291016
    },
    {
      "bits_per_byte": 1.8163858239897086,
      "learning_rate": 0.00025352116371678464,
      "loss_nats": 1.2590227127075195,
      "step": 2470,
      "unclipped_grad_norm": 0.7682369351387024
    },
    {
      "bits_per_byte": 1.8266683227194942,
      "learning_rate": 0.0002530990778057627,
      "loss_nats": 1.2661499977111816,
      "step": 2480,
      "unclipped_grad_norm": 0.7132375240325928
    },
    {
      "bits_per_byte": 1.8042156436840528,
      "learning_rate": 0.00025267548652457475,
      "loss_nats": 1.250586986541748,
      "step": 2490,
      "unclipped_grad_norm": 0.7770085334777832
    },
    {
      "bits_per_byte": 1.8313916542427904,
      "learning_rate": 0.00025225039711122696,
      "loss_nats": 1.2694239616394043,
      "step": 2500,
      "unclipped_grad_norm": 0.7582318782806396
    },
    {
      "bits_per_byte": 1.7788244690759694,
      "learning_rate": 0.00025182381682932464,
      "loss_nats": 1.2329871654510498,
      "step": 2510,
      "unclipped_grad_norm": 0.8006963133811951
    },
    {
      "bits_per_byte": 1.8448090527325032,
      "learning_rate": 0.0002513957529679477,
      "loss_nats": 1.278724193572998,
      "step": 2520,
      "unclipped_grad_norm": 0.7802237272262573
    },
    {
      "bits_per_byte": 1.855150369528884,
      "learning_rate": 0.00025096621284152656,
      "loss_nats": 1.2858922481536865,
      "step": 2530,
      "unclipped_grad_norm": 0.7603861689567566
    },
    {
      "bits_per_byte": 1.7974587892969547,
      "learning_rate": 0.0002505352037897168,
      "loss_nats": 1.245903491973877,
      "step": 2540,
      "unclipped_grad_norm": 0.7829954624176025
    },
    {
      "bits_per_byte": 1.8048358131230642,
      "learning_rate": 0.0002501027331772739,
      "loss_nats": 1.2510168552398682,
      "step": 2550,
      "unclipped_grad_norm": 0.8207260966300964
    },
    {
      "bits_per_byte": 1.8410618947355044,
      "learning_rate": 0.00024966880839392746,
      "loss_nats": 1.2761268615722656,
      "step": 2560,
      "unclipped_grad_norm": 0.8064310550689697
    },
    {
      "bits_per_byte": 1.8072459779922554,
      "learning_rate": 0.0002492334368542548,
      "loss_nats": 1.2526874542236328,
      "step": 2570,
      "unclipped_grad_norm": 0.7300699353218079
    },
    {
      "bits_per_byte": 1.8352736466380657,
      "learning_rate": 0.00024879662599755416,
      "loss_nats": 1.2721147537231445,
      "step": 2580,
      "unclipped_grad_norm": 0.7476873993873596
    },
    {
      "bits_per_byte": 1.793617556789934,
      "learning_rate": 0.00024835838328771803,
      "loss_nats": 1.2432409524917603,
      "step": 2590,
      "unclipped_grad_norm": 0.7210070490837097
    },
    {
      "bits_per_byte": 1.797604974550188,
      "learning_rate": 0.0002479187162131051,
      "loss_nats": 1.2460048198699951,
      "step": 2600,
      "unclipped_grad_norm": 0.7853913903236389
    },
    {
      "bits_per_byte": 1.7385629865436842,
      "learning_rate": 0.00024747763228641264,
      "loss_nats": 1.2050800323486328,
      "step": 2610,
      "unclipped_grad_norm": 0.7092626094818115
    },
    {
      "bits_per_byte": 1.8079149905041119,
      "learning_rate": 0.00024703513904454795,
      "loss_nats": 1.2531511783599854,
      "step": 2620,
      "unclipped_grad_norm": 0.7522015571594238
    },
    {
      "bits_per_byte": 1.8198622812942493,
      "learning_rate": 0.00024659124404849956,
      "loss_nats": 1.261432409286499,
      "step": 2630,
      "unclipped_grad_norm": 0.7675716876983643
    },
    {
      "bits_per_byte": 1.680096624469618,
      "learning_rate": 0.00024614595488320833,
      "loss_nats": 1.164554238319397,
      "step": 2640,
      "unclipped_grad_norm": 0.6825466156005859
    },
    {
      "bits_per_byte": 1.7638260340768748,
      "learning_rate": 0.00024569927915743745,
      "loss_nats": 1.2225910425186157,
      "step": 2650,
      "unclipped_grad_norm": 0.7630164623260498
    },
    {
      "bits_per_byte": 1.8201589513669878,
      "learning_rate": 0.0002452512245036427,
      "loss_nats": 1.2616380453109741,
      "step": 2660,
      "unclipped_grad_norm": 0.7772946953773499
    },
    {
      "bits_per_byte": 1.811244230659514,
      "learning_rate": 0.00024480179857784177,
      "loss_nats": 1.2554588317871094,
      "step": 2670,
      "unclipped_grad_norm": 0.7582975625991821
    },
    {
      "bits_per_byte": 1.74257860945868,
      "learning_rate": 0.00024435100905948387,
      "loss_nats": 1.207863450050354,
      "step": 2680,
      "unclipped_grad_norm": 0.7818799614906311
    },
    {
      "bits_per_byte": 1.9030688636428832,
      "learning_rate": 0.00024389886365131793,
      "loss_nats": 1.3191068172454834,
      "step": 2690,
      "unclipped_grad_norm": 0.7075229287147522
    },
    {
      "bits_per_byte": 1.8200497423836899,
      "learning_rate": 0.00024344537007926163,
      "loss_nats": 1.2615623474121094,
      "step": 2700,
      "unclipped_grad_norm": 0.8272333741188049
    },
    {
      "bits_per_byte": 1.7354167359288,
      "learning_rate": 0.00024299053609226878,
      "loss_nats": 1.2028992176055908,
      "step": 2710,
      "unclipped_grad_norm": 0.7310962677001953
    },
    {
      "bits_per_byte": 1.728331394678554,
      "learning_rate": 0.00024253436946219733,
      "loss_nats": 1.1979880332946777,
      "step": 2720,
      "unclipped_grad_norm": 0.7193791270256042
    },
    {
      "bits_per_byte": 1.76859700479446,
      "learning_rate": 0.0002420768779836764,
      "loss_nats": 1.225898027420044,
      "step": 2730,
      "unclipped_grad_norm": 0.773657500743866
    },
    {
      "bits_per_byte": 1.7790022991369616,
      "learning_rate": 0.00024161806947397305,
      "loss_nats": 1.2331104278564453,
      "step": 2740,
      "unclipped_grad_norm": 0.7759174704551697
    },
    {
      "bits_per_byte": 1.7909771071512373,
      "learning_rate": 0.00024115795177285886,
      "loss_nats": 1.241410732269287,
      "step": 2750,
      "unclipped_grad_norm": 0.770866870880127
    },
    {
      "bits_per_byte": 1.81886375002334,
      "learning_rate": 0.0002406965327424758,
      "loss_nats": 1.2607402801513672,
      "step": 2760,
      "unclipped_grad_norm": 0.7793042659759521
    },
    {
      "bits_per_byte": 1.704546881994831,
      "learning_rate": 0.00024023382026720197,
      "loss_nats": 1.181501865386963,
      "step": 2770,
      "unclipped_grad_norm": 0.7179723978042603
    },
    {
      "bits_per_byte": 1.7222445846992174,
      "learning_rate": 0.00023976982225351698,
      "loss_nats": 1.1937689781188965,
      "step": 2780,
      "unclipped_grad_norm": 0.711125910282135
    },
    {
      "bits_per_byte": 1.8073385046584196,
      "learning_rate": 0.0002393045466298666,
      "loss_nats": 1.2527515888214111,
      "step": 2790,
      "unclipped_grad_norm": 0.7564381957054138
    },
    {
      "bits_per_byte": 1.8564130661515186,
      "learning_rate": 0.00023883800134652763,
      "loss_nats": 1.2867674827575684,
      "step": 2800,
      "unclipped_grad_norm": 0.7982874512672424
    },
    {
      "bits_per_byte": 1.7924368958917607,
      "learning_rate": 0.0002383701943754715,
      "loss_nats": 1.2424225807189941,
      "step": 2810,
      "unclipped_grad_norm": 0.827074408531189
    },
    {
      "bits_per_byte": 1.781587886310033,
      "learning_rate": 0.00023790113371022884,
      "loss_nats": 1.2349026203155518,
      "step": 2820,
      "unclipped_grad_norm": 0.6945767998695374
    },
    {
      "bits_per_byte": 1.862601517877515,
      "learning_rate": 0.0002374308273657521,
      "loss_nats": 1.2910569906234741,
      "step": 2830,
      "unclipped_grad_norm": 0.7473636269569397
    },
    {
      "bits_per_byte": 1.7520161574247766,
      "learning_rate": 0.00023695928337827911,
      "loss_nats": 1.2144050598144531,
      "step": 2840,
      "unclipped_grad_norm": 0.7269325852394104
    },
    {
      "bits_per_byte": 1.7726036846116109,
      "learning_rate": 0.00023648650980519564,
      "loss_nats": 1.2286752462387085,
      "step": 2850,
      "unclipped_grad_norm": 0.7537488341331482
    },
    {
      "bits_per_byte": 1.7375121725469125,
      "learning_rate": 0.00023601251472489757,
      "loss_nats": 1.2043516635894775,
      "step": 2860,
      "unclipped_grad_norm": 0.7839256525039673
    },
    {
      "bits_per_byte": 1.836773851301542,
      "learning_rate": 0.00023553730623665305,
      "loss_nats": 1.273154616355896,
      "step": 2870,
      "unclipped_grad_norm": 0.7615643739700317
    },
    {
      "bits_per_byte": 1.8235200083127996,
      "learning_rate": 0.00023506089246046403,
      "loss_nats": 1.263967752456665,
      "step": 2880,
      "unclipped_grad_norm": 0.7500733137130737
    },
    {
      "bits_per_byte": 1.7346435019305209,
      "learning_rate": 0.00023458328153692736,
      "loss_nats": 1.2023632526397705,
      "step": 2890,
      "unclipped_grad_norm": 0.7335168123245239
    },
    {
      "bits_per_byte": 1.710140445731493,
      "learning_rate": 0.0002341044816270961,
      "loss_nats": 1.1853790283203125,
      "step": 2900,
      "unclipped_grad_norm": 0.7237004041671753
    },
    {
      "bits_per_byte": 1.7640040361205178,
      "learning_rate": 0.00023362450091233953,
      "loss_nats": 1.2227144241333008,
      "step": 2910,
      "unclipped_grad_norm": 0.7653093338012695
    },
    {
      "bits_per_byte": 1.7372325287566095,
      "learning_rate": 0.00023314334759420388,
      "loss_nats": 1.204157829284668,
      "step": 2920,
      "unclipped_grad_norm": 0.7592934966087341
    },
    {
      "bits_per_byte": 1.7605243111456106,
      "learning_rate": 0.0002326610298942717,
      "loss_nats": 1.2203024625778198,
      "step": 2930,
      "unclipped_grad_norm": 0.7871323227882385
    },
    {
      "bits_per_byte": 1.7773966691085064,
      "learning_rate": 0.00023217755605402175,
      "loss_nats": 1.2319974899291992,
      "step": 2940,
      "unclipped_grad_norm": 0.7994735240936279
    },
    {
      "bits_per_byte": 1.761262116717812,
      "learning_rate": 0.00023169293433468802,
      "loss_nats": 1.2208138704299927,
      "step": 2950,
      "unclipped_grad_norm": 0.7672474384307861
    },
    {
      "bits_per_byte": 1.8009139207527887,
      "learning_rate": 0.00023120717301711857,
      "loss_nats": 1.2482984066009521,
      "step": 2960,
      "unclipped_grad_norm": 0.7721540927886963
    },
    {
      "bits_per_byte": 1.7577087831683356,
      "learning_rate": 0.00023072028040163408,
      "loss_nats": 1.218350887298584,
      "step": 2970,
      "unclipped_grad_norm": 0.7659915685653687
    },
    {
      "bits_per_byte": 1.7511909846659368,
      "learning_rate": 0.00023023226480788593,
      "loss_nats": 1.2138330936431885,
      "step": 2980,
      "unclipped_grad_norm": 0.7457980513572693
    },
    {
      "bits_per_byte": 1.7677185174138528,
      "learning_rate": 0.00022974313457471416,
      "loss_nats": 1.2252891063690186,
      "step": 2990,
      "unclipped_grad_norm": 0.7551211714744568
    },
    {
      "bits_per_byte": 1.7804380103063644,
      "learning_rate": 0.0002292528980600049,
      "loss_nats": 1.2341055870056152,
      "step": 3000,
      "unclipped_grad_norm": 0.7369211316108704
    },
    {
      "bits_per_byte": 1.77081558099059,
      "learning_rate": 0.00022876156364054758,
      "loss_nats": 1.227435827255249,
      "step": 3010,
      "unclipped_grad_norm": 0.7526711225509644
    },
    {
      "bits_per_byte": 1.726040929734363,
      "learning_rate": 0.0002282691397118918,
      "loss_nats": 1.1964004039764404,
      "step": 3020,
      "unclipped_grad_norm": 0.8113403916358948
    },
    {
      "bits_per_byte": 1.8045116258261877,
      "learning_rate": 0.00022777563468820376,
      "loss_nats": 1.250792145729065,
      "step": 3030,
      "unclipped_grad_norm": 0.7493385672569275
    },
    {
      "bits_per_byte": 1.7229166928987893,
      "learning_rate": 0.00022728105700212282,
      "loss_nats": 1.194234848022461,
      "step": 3040,
      "unclipped_grad_norm": 0.7606903314590454
    },
    {
      "bits_per_byte": 1.7403560776615798,
      "learning_rate": 0.00022678541510461698,
      "loss_nats": 1.2063229084014893,
      "step": 3050,
      "unclipped_grad_norm": 0.7690193057060242
    },
    {
      "bits_per_byte": 1.759874388708,
      "learning_rate": 0.00022628871746483876,
      "loss_nats": 1.2198519706726074,
      "step": 3060,
      "unclipped_grad_norm": 0.7480750679969788
    },
    {
      "bits_per_byte": 1.6873952242069339,
      "learning_rate": 0.0002257909725699804,
      "loss_nats": 1.169613242149353,
      "step": 3070,
      "unclipped_grad_norm": 0.7001752853393555
    },
    {
      "bits_per_byte": 1.737272084766308,
      "learning_rate": 0.00022529218892512886,
      "loss_nats": 1.2041852474212646,
      "step": 3080,
      "unclipped_grad_norm": 0.8322204351425171
    },
    {
      "bits_per_byte": 1.8015143121869506,
      "learning_rate": 0.00022479237505312034,
      "loss_nats": 1.248714566230774,
      "step": 3090,
      "unclipped_grad_norm": 0.8183287382125854
    },
    {
      "bits_per_byte": 1.7001176408045102,
      "learning_rate": 0.00022429153949439496,
      "loss_nats": 1.178431749343872,
      "step": 3100,
      "unclipped_grad_norm": 0.7750749588012695
    },
    {
      "bits_per_byte": 1.7770231227908324,
      "learning_rate": 0.00022378969080685053,
      "loss_nats": 1.231738567352295,
      "step": 3110,
      "unclipped_grad_norm": 0.7082787752151489
    },
    {
      "bits_per_byte": 1.749435729731231,
      "learning_rate": 0.0002232868375656965,
      "loss_nats": 1.2126164436340332,
      "step": 3120,
      "unclipped_grad_norm": 0.736611545085907
    },
    {
      "bits_per_byte": 1.7966217497352055,
      "learning_rate": 0.00022278298836330733,
      "loss_nats": 1.2453233003616333,
      "step": 3130,
      "unclipped_grad_norm": 0.7366204857826233
    },
    {
      "bits_per_byte": 1.7636881039908827,
      "learning_rate": 0.00022227815180907572,
      "loss_nats": 1.222495436668396,
      "step": 3140,
      "unclipped_grad_norm": 0.7583194375038147
    },
    {
      "bits_per_byte": 1.7359698321339747,
      "learning_rate": 0.00022177233652926546,
      "loss_nats": 1.2032825946807861,
      "step": 3150,
      "unclipped_grad_norm": 0.7600930333137512
    },
    {
      "bits_per_byte": 1.7415977924008093,
      "learning_rate": 0.00022126555116686416,
      "loss_nats": 1.207183599472046,
      "step": 3160,
      "unclipped_grad_norm": 0.7639689445495605
    },
    {
      "bits_per_byte": 1.654884311818432,
      "learning_rate": 0.00022075780438143532,
      "loss_nats": 1.1470783948898315,
      "step": 3170,
      "unclipped_grad_norm": 0.7011478543281555
    },
    {
      "bits_per_byte": 1.7584833930278216,
      "learning_rate": 0.00022024910484897065,
      "loss_nats": 1.2188878059387207,
      "step": 3180,
      "unclipped_grad_norm": 0.7645173072814941
    },
    {
      "bits_per_byte": 1.7305350083840592,
      "learning_rate": 0.00021973946126174158,
      "loss_nats": 1.199515461921692,
      "step": 3190,
      "unclipped_grad_norm": 0.7665988206863403
    },
    {
      "bits_per_byte": 1.7003714871971838,
      "learning_rate": 0.00021922888232815086,
      "loss_nats": 1.178607702255249,
      "step": 3200,
      "unclipped_grad_norm": 0.7379894852638245
    },
    {
      "bits_per_byte": 1.6959066455981344,
      "learning_rate": 0.0002187173767725837,
      "loss_nats": 1.1755129098892212,
      "step": 3210,
      "unclipped_grad_norm": 0.8013631701469421
    },
    {
      "bits_per_byte": 1.7098007799960389,
      "learning_rate": 0.0002182049533352588,
      "loss_nats": 1.1851435899734497,
      "step": 3220,
      "unclipped_grad_norm": 0.7296364903450012
    },
    {
      "bits_per_byte": 1.7987765203678656,
      "learning_rate": 0.00021769162077207884,
      "loss_nats": 1.246816873550415,
      "step": 3230,
      "unclipped_grad_norm": 0.7610434889793396
    },
    {
      "bits_per_byte": 1.6862874839527267,
      "learning_rate": 0.00021717738785448102,
      "loss_nats": 1.1688454151153564,
      "step": 3240,
      "unclipped_grad_norm": 0.723595917224884
    },
    {
      "bits_per_byte": 1.8345631863073515,
      "learning_rate": 0.00021666226336928708,
      "loss_nats": 1.2716223001480103,
      "step": 3250,
      "unclipped_grad_norm": 0.8353017568588257
    },
    {
      "bits_per_byte": 1.6872093109613513,
      "learning_rate": 0.0002161462561185532,
      "loss_nats": 1.1694843769073486,
      "step": 3260,
      "unclipped_grad_norm": 0.7602720260620117
    },
    {
      "bits_per_byte": 1.7198954736710823,
      "learning_rate": 0.00021562937491941957,
      "loss_nats": 1.1921406984329224,
      "step": 3270,
      "unclipped_grad_norm": 0.7516167163848877
    },
    {
      "bits_per_byte": 1.7692164863028679,
      "learning_rate": 0.00021511162860395975,
      "loss_nats": 1.2263274192810059,
      "step": 3280,
      "unclipped_grad_norm": 0.7487898468971252
    },
    {
      "bits_per_byte": 1.7624494849393688,
      "learning_rate": 0.0002145930260190298,
      "loss_nats": 1.2216368913650513,
      "step": 3290,
      "unclipped_grad_norm": 0.7708044052124023
    },
    {
      "bits_per_byte": 1.7002387165907176,
      "learning_rate": 0.00021407357602611698,
      "loss_nats": 1.1785156726837158,
      "step": 3300,
      "unclipped_grad_norm": 0.7703046202659607
    },
    {
      "bits_per_byte": 1.7332599015043295,
      "learning_rate": 0.0002135532875011885,
      "loss_nats": 1.2014042139053345,
      "step": 3310,
      "unclipped_grad_norm": 0.8132672905921936
    },
    {
      "bits_per_byte": 1.7262779218272521,
      "learning_rate": 0.0002130321693345397,
      "loss_nats": 1.1965646743774414,
      "step": 3320,
      "unclipped_grad_norm": 0.7545798420906067
    },
    {
      "bits_per_byte": 1.7272033604715449,
      "learning_rate": 0.00021251023043064224,
      "loss_nats": 1.1972061395645142,
      "step": 3330,
      "unclipped_grad_norm": 0.7775130867958069
    },
    {
      "bits_per_byte": 1.6975052243379043,
      "learning_rate": 0.00021198747970799188,
      "loss_nats": 1.1766209602355957,
      "step": 3340,
      "unclipped_grad_norm": 0.7105540633201599
    },
    {
      "bits_per_byte": 1.7083408192728646,
      "learning_rate": 0.00021146392609895618,
      "loss_nats": 1.1841316223144531,
      "step": 3350,
      "unclipped_grad_norm": 0.7163253426551819
    },
    {
      "bits_per_byte": 1.732425613664994,
      "learning_rate": 0.00021093957854962178,
      "loss_nats": 1.2008259296417236,
      "step": 3360,
      "unclipped_grad_norm": 0.7626740336418152
    },
    {
      "bits_per_byte": 1.6756070173688444,
      "learning_rate": 0.00021041444601964157,
      "loss_nats": 1.1614422798156738,
      "step": 3370,
      "unclipped_grad_norm": 0.8061453700065613
    },
    {
      "bits_per_byte": 1.7092863798873084,
      "learning_rate": 0.00020988853748208164,
      "loss_nats": 1.1847870349884033,
      "step": 3380,
      "unclipped_grad_norm": 0.7406735420227051
    },
    {
      "bits_per_byte": 1.7539413312185348,
      "learning_rate": 0.00020936186192326782,
      "loss_nats": 1.2157394886016846,
      "step": 3390,
      "unclipped_grad_norm": 0.7695116400718689
    },
    {
      "bits_per_byte": 1.8092177590843976,
      "learning_rate": 0.00020883442834263225,
      "loss_nats": 1.2540541887283325,
      "step": 3400,
      "unclipped_grad_norm": 0.7858016490936279
    },
    {
      "bits_per_byte": 1.8255297975707825,
      "learning_rate": 0.0002083062457525596,
      "loss_nats": 1.2653608322143555,
      "step": 3410,
      "unclipped_grad_norm": 0.7603964805603027
    },
    {
      "bits_per_byte": 1.6857877023693193,
      "learning_rate": 0.00020777732317823306,
      "loss_nats": 1.1684989929199219,
      "step": 3420,
      "unclipped_grad_norm": 0.7168062329292297
    },
    {
      "bits_per_byte": 1.6176447364444573,
      "learning_rate": 0.00020724766965748005,
      "loss_nats": 1.1212658882141113,
      "step": 3430,
      "unclipped_grad_norm": 0.7431134581565857
    },
    {
      "bits_per_byte": 1.711189711884407,
      "learning_rate": 0.00020671729424061788,
      "loss_nats": 1.1861063241958618,
      "step": 3440,
      "unclipped_grad_norm": 0.7753933668136597
    },
    {
      "bits_per_byte": 1.77535626693867,
      "learning_rate": 0.00020618620599029906,
      "loss_nats": 1.2305831909179688,
      "step": 3450,
      "unclipped_grad_norm": 0.8042635321617126
    },
    {
      "bits_per_byte": 1.7083845028661837,
      "learning_rate": 0.00020565441398135649,
      "loss_nats": 1.184161901473999,
      "step": 3460,
      "unclipped_grad_norm": 0.7639697194099426
    },
    {
      "bits_per_byte": 1.6937178224006033,
      "learning_rate": 0.00020512192730064827,
      "loss_nats": 1.1739957332611084,
      "step": 3470,
      "unclipped_grad_norm": 0.748127281665802
    },
    {
      "bits_per_byte": 1.6501064617948125,
      "learning_rate": 0.00020458875504690273,
      "loss_nats": 1.1437666416168213,
      "step": 3480,
      "unclipped_grad_norm": 0.7506517767906189
    },
    {
      "bits_per_byte": 1.8358463488654388,
      "learning_rate": 0.00020405490633056245,
      "loss_nats": 1.2725117206573486,
      "step": 3490,
      "unclipped_grad_norm": 0.836082398891449
    },
    {
      "bits_per_byte": 1.714104301868579,
      "learning_rate": 0.00020352039027362917,
      "loss_nats": 1.188126564025879,
      "step": 3500,
      "unclipped_grad_norm": 0.7544166445732117
    },
    {
      "bits_per_byte": 1.744907598516664,
      "learning_rate": 0.0002029852160095075,
      "loss_nats": 1.2094777822494507,
      "step": 3510,
      "unclipped_grad_norm": 0.7650617361068726
    },
    {
      "bits_per_byte": 1.738600994709525,
      "learning_rate": 0.000202449392682849,
      "loss_nats": 1.2051063776016235,
      "step": 3520,
      "unclipped_grad_norm": 0.7794367671012878
    },
    {
      "bits_per_byte": 1.7051804660806096,
      "learning_rate": 0.0002019129294493959,
      "loss_nats": 1.181941032409668,
      "step": 3530,
      "unclipped_grad_norm": 0.7445787787437439
    },
    {
      "bits_per_byte": 1.7158172490711725,
      "learning_rate": 0.00020137583547582478,
      "loss_nats": 1.1893138885498047,
      "step": 3540,
      "unclipped_grad_norm": 0.7629107236862183
    },
    {
      "bits_per_byte": 1.7173026632266746,
      "learning_rate": 0.00020083811993958973,
      "loss_nats": 1.1903434991836548,
      "step": 3550,
      "unclipped_grad_norm": 0.7763495445251465
    },
    {
      "bits_per_byte": 1.6560372835098163,
      "learning_rate": 0.00020029979202876563,
      "loss_nats": 1.14787757396698,
      "step": 3560,
      "unclipped_grad_norm": 0.7450886964797974
    },
    {
      "bits_per_byte": 1.6838996768281473,
      "learning_rate": 0.00019976086094189118,
      "loss_nats": 1.1671903133392334,
      "step": 3570,
      "unclipped_grad_norm": 0.7487124800682068
    },
    {
      "bits_per_byte": 1.6946102403759304,
      "learning_rate": 0.00019922133588781175,
      "loss_nats": 1.1746143102645874,
      "step": 3580,
      "unclipped_grad_norm": 0.757170557975769
    },
    {
      "bits_per_byte": 1.7307594457434352,
      "learning_rate": 0.0001986812260855219,
      "loss_nats": 1.1996710300445557,
      "step": 3590,
      "unclipped_grad_norm": 0.7700981497764587
    },
    {
      "bits_per_byte": 1.7491381997452384,
      "learning_rate": 0.00019814054076400787,
      "loss_nats": 1.2124102115631104,
      "step": 3600,
      "unclipped_grad_norm": 0.7877120971679688
    },
    {
      "bits_per_byte": 1.688025540622346,
      "learning_rate": 0.00019759928916209004,
      "loss_nats": 1.1700501441955566,
      "step": 3610,
      "unclipped_grad_norm": 0.780867874622345
    },
    {
      "bits_per_byte": 1.6436277033541595,
      "learning_rate": 0.00019705748052826493,
      "loss_nats": 1.1392759084701538,
      "step": 3620,
      "unclipped_grad_norm": 0.7423334717750549
    },
    {
      "bits_per_byte": 1.6950637586262558,
      "learning_rate": 0.00019651512412054723,
      "loss_nats": 1.1749286651611328,
      "step": 3630,
      "unclipped_grad_norm": 0.7513464689254761
    },
    {
      "bits_per_byte": 1.6758522646289749,
      "learning_rate": 0.00019597222920631144,
      "loss_nats": 1.1616122722625732,
      "step": 3640,
      "unclipped_grad_norm": 0.7453336715698242
    },
    {
      "bits_per_byte": 1.736063734661346,
      "learning_rate": 0.0001954288050621338,
      "loss_nats": 1.2033476829528809,
      "step": 3650,
      "unclipped_grad_norm": 0.7682862877845764
    },
    {
      "bits_per_byte": 1.6360068081291268,
      "learning_rate": 0.00019488486097363363,
      "loss_nats": 1.1339935064315796,
      "step": 3660,
      "unclipped_grad_norm": 0.7531437277793884
    },
    {
      "bits_per_byte": 1.7183529612754938,
      "learning_rate": 0.00019434040623531452,
      "loss_nats": 1.1910715103149414,
      "step": 3670,
      "unclipped_grad_norm": 0.7578480243682861
    },
    {
      "bits_per_byte": 1.6583936178092877,
      "learning_rate": 0.0001937954501504058,
      "loss_nats": 1.1495108604431152,
      "step": 3680,
      "unclipped_grad_norm": 0.7224898934364319
    },
    {
      "bits_per_byte": 1.6372521345040243,
      "learning_rate": 0.00019325000203070333,
      "loss_nats": 1.1348567008972168,
      "step": 3690,
      "unclipped_grad_norm": 0.7246464490890503
    },
    {
      "bits_per_byte": 1.6659130432703113,
      "learning_rate": 0.0001927040711964106,
      "loss_nats": 1.1547229290008545,
      "step": 3700,
      "unclipped_grad_norm": 0.7878638505935669
    },
    {
      "bits_per_byte": 1.694639821391879,
      "learning_rate": 0.00019215766697597923,
      "loss_nats": 1.1746348142623901,
      "step": 3710,
      "unclipped_grad_norm": 0.7455682754516602
    },
    {
      "bits_per_byte": 1.6891824679147003,
      "learning_rate": 0.00019161079870594978,
      "loss_nats": 1.1708520650863647,
      "step": 3720,
      "unclipped_grad_norm": 0.7515487670898438
    },
    {
      "bits_per_byte": 1.6091924770851538,
      "learning_rate": 0.0001910634757307922,
      "loss_nats": 1.1154072284698486,
      "step": 3730,
      "unclipped_grad_norm": 0.7353127598762512
    },
    {
      "bits_per_byte": 1.6633680440028435,
      "learning_rate": 0.0001905157074027459,
      "loss_nats": 1.152958869934082,
      "step": 3740,
      "unclipped_grad_norm": 0.7253232002258301
    },
    {
      "bits_per_byte": 1.6922951819126664,
      "learning_rate": 0.00018996750308166037,
      "loss_nats": 1.1730096340179443,
      "step": 3750,
      "unclipped_grad_norm": 0.7686370611190796
    },
    {
      "bits_per_byte": 1.7713081393026613,
      "learning_rate": 0.00018941887213483482,
      "loss_nats": 1.2277772426605225,
      "step": 3760,
      "unclipped_grad_norm": 0.7601476907730103
    },
    {
      "bits_per_byte": 1.7305630415561497,
      "learning_rate": 0.00018886982393685845,
      "loss_nats": 1.1995348930358887,
      "step": 3770,
      "unclipped_grad_norm": 0.7877833247184753
    },
    {
      "bits_per_byte": 1.5644979697442287,
      "learning_rate": 0.00018832036786945008,
      "loss_nats": 1.0844273567199707,
      "step": 3780,
      "unclipped_grad_norm": 0.7386898994445801
    },
    {
      "bits_per_byte": 1.6644766441703052,
      "learning_rate": 0.0001877705133212979,
      "loss_nats": 1.1537272930145264,
      "step": 3790,
      "unclipped_grad_norm": 0.7420179843902588
    },
    {
      "bits_per_byte": 1.6653939996300076,
      "learning_rate": 0.00018722026968789906,
      "loss_nats": 1.1543631553649902,
      "step": 3800,
      "unclipped_grad_norm": 0.7838358283042908
    },
    {
      "bits_per_byte": 1.6952983429620325,
      "learning_rate": 0.00018666964637139907,
      "loss_nats": 1.17509126663208,
      "step": 3810,
      "unclipped_grad_norm": 0.8048396110534668
    },
    {
      "bits_per_byte": 1.6744323758634516,
      "learning_rate": 0.00018611865278043115,
      "loss_nats": 1.160628080368042,
      "step": 3820,
      "unclipped_grad_norm": 0.7709245681762695
    },
    {
      "bits_per_byte": 1.720191799778519,
      "learning_rate": 0.00018556729832995555,
      "loss_nats": 1.1923460960388184,
      "step": 3830,
      "unclipped_grad_norm": 0.7672807574272156
    },
    {
      "bits_per_byte": 1.6467005173771254,
      "learning_rate": 0.00018501559244109864,
      "loss_nats": 1.1414058208465576,
      "step": 3840,
      "unclipped_grad_norm": 0.7319741249084473
    },
    {
      "bits_per_byte": 1.5964396195583748,
      "learning_rate": 0.00018446354454099178,
      "loss_nats": 1.106567621231079,
      "step": 3850,
      "unclipped_grad_norm": 0.7756319642066956
    },
    {
      "bits_per_byte": 1.6360811046342996,
      "learning_rate": 0.00018391116406261048,
      "loss_nats": 1.1340450048446655,
      "step": 3860,
      "unclipped_grad_norm": 0.766754150390625
    },
    {
      "bits_per_byte": 1.7433616464680584,
      "learning_rate": 0.00018335846044461305,
      "loss_nats": 1.2084062099456787,
      "step": 3870,
      "unclipped_grad_norm": 0.7916337847709656
    },
    {
      "bits_per_byte": 1.6358671582166262,
      "learning_rate": 0.0001828054431311793,
      "loss_nats": 1.1338967084884644,
      "step": 3880,
      "unclipped_grad_norm": 0.7497678399085999
    },
    {
      "bits_per_byte": 1.600699113872294,
      "learning_rate": 0.00018225212157184934,
      "loss_nats": 1.1095200777053833,
      "step": 3890,
      "unclipped_grad_norm": 0.7555440664291382
    },
    {
      "bits_per_byte": 1.6778517349279058,
      "learning_rate": 0.00018169850522136192,
      "loss_nats": 1.1629981994628906,
      "step": 3900,
      "unclipped_grad_norm": 0.826478123664856
    },
    {
      "bits_per_byte": 1.6805711246033486,
      "learning_rate": 0.000181144603539493,
      "loss_nats": 1.1648831367492676,
      "step": 3910,
      "unclipped_grad_norm": 0.7849931120872498
    },
    {
      "bits_per_byte": 1.6375147520118918,
      "learning_rate": 0.00018059042599089412,
      "loss_nats": 1.1350387334823608,
      "step": 3920,
      "unclipped_grad_norm": 0.7588120102882385
    },
    {
      "bits_per_byte": 1.673005091843941,
      "learning_rate": 0.00018003598204493048,
      "loss_nats": 1.15963876247406,
      "step": 3930,
      "unclipped_grad_norm": 0.742434024810791
    },
    {
      "bits_per_byte": 1.6996766772876981,
      "learning_rate": 0.00017948128117551936,
      "loss_nats": 1.1781260967254639,
      "step": 3940,
      "unclipped_grad_norm": 0.7588658928871155
    },
    {
      "bits_per_byte": 1.7339400928884918,
      "learning_rate": 0.00017892633286096816,
      "loss_nats": 1.2018756866455078,
      "step": 3950,
      "unclipped_grad_norm": 0.7937244176864624
    },
    {
      "bits_per_byte": 1.713467450112434,
      "learning_rate": 0.00017837114658381247,
      "loss_nats": 1.1876851320266724,
      "step": 3960,
      "unclipped_grad_norm": 0.7884036302566528
    },
    {
      "bits_per_byte": 1.6516962694193882,
      "learning_rate": 0.00017781573183065386,
      "loss_nats": 1.1448686122894287,
      "step": 3970,
      "unclipped_grad_norm": 0.8004661798477173
    },
    {
      "bits_per_byte": 1.6740500584305835,
      "learning_rate": 0.0001772600980919981,
      "loss_nats": 1.1603630781173706,
      "step": 3980,
      "unclipped_grad_norm": 0.7692103385925293
    },
    {
      "bits_per_byte": 1.691941929547794,
      "learning_rate": 0.0001767042548620927,
      "loss_nats": 1.172764778137207,
      "step": 3990,
      "unclipped_grad_norm": 0.7891059517860413
    },
    {
      "bits_per_byte": 1.5813677479673616,
      "learning_rate": 0.00017614821163876486,
      "loss_nats": 1.0961205959320068,
      "step": 4000,
      "unclipped_grad_norm": 0.7436221837997437
    },
    {
      "bits_per_byte": 1.6793110077204767,
      "learning_rate": 0.00017559197792325906,
      "loss_nats": 1.164009690284729,
      "step": 4010,
      "unclipped_grad_norm": 0.8044730424880981
    },
    {
      "bits_per_byte": 1.6584589712166156,
      "learning_rate": 0.0001750355632200748,
      "loss_nats": 1.1495561599731445,
      "step": 4020,
      "unclipped_grad_norm": 0.8183668851852417
    },
    {
      "bits_per_byte": 1.6850787898824628,
      "learning_rate": 0.00017447897703680418,
      "loss_nats": 1.1680076122283936,
      "step": 4030,
      "unclipped_grad_norm": 0.793128252029419
    },
    {
      "bits_per_byte": 1.617821878574846,
      "learning_rate": 0.0001739222288839694,
      "loss_nats": 1.1213886737823486,
      "step": 4040,
      "unclipped_grad_norm": 0.7582541108131409
    },
    {
      "bits_per_byte": 1.6450140555027646,
      "learning_rate": 0.00017336532827486026,
      "loss_nats": 1.1402368545532227,
      "step": 4050,
      "unclipped_grad_norm": 0.7559582591056824
    },
    {
      "bits_per_byte": 1.6581645369183384,
      "learning_rate": 0.00017280828472537163,
      "loss_nats": 1.1493520736694336,
      "step": 4060,
      "unclipped_grad_norm": 0.743245005607605
    },
    {
      "bits_per_byte": 1.680131880913045,
      "learning_rate": 0.0001722511077538408,
      "loss_nats": 1.1645786762237549,
      "step": 4070,
      "unclipped_grad_norm": 0.7624887824058533
    },
    {
      "bits_per_byte": 1.7564567494700543,
      "learning_rate": 0.00017169380688088497,
      "loss_nats": 1.2174830436706543,
      "step": 4080,
      "unclipped_grad_norm": 0.8132302761077881
    },
    {
      "bits_per_byte": 1.6669108866106175,
      "learning_rate": 0.0001711363916292384,
      "loss_nats": 1.1554145812988281,
      "step": 4090,
      "unclipped_grad_norm": 0.8276432752609253
    },
    {
      "bits_per_byte": 1.638365894151012,
      "learning_rate": 0.00017057887152358974,
      "loss_nats": 1.1356287002563477,
      "step": 4100,
      "unclipped_grad_norm": 0.7751885652542114
    },
    {
      "bits_per_byte": 1.6303674970073352,
      "learning_rate": 0.0001700212560904194,
      "loss_nats": 1.1300846338272095,
      "step": 4110,
      "unclipped_grad_norm": 0.7898787260055542
    },
    {
      "bits_per_byte": 1.6896222275529564,
      "learning_rate": 0.00016946355485783656,
      "loss_nats": 1.171156883239746,
      "step": 4120,
      "unclipped_grad_norm": 0.8148247599601746
    },
    {
      "bits_per_byte": 1.6213521664491068,
      "learning_rate": 0.0001689057773554166,
      "loss_nats": 1.1238356828689575,
      "step": 4130,
      "unclipped_grad_norm": 0.8133904933929443
    },
    {
      "bits_per_byte": 1.6881359534842002,
      "learning_rate": 0.00016834793311403806,
      "loss_nats": 1.1701266765594482,
      "step": 4140,
      "unclipped_grad_norm": 0.8217440247535706
    },
    {
      "bits_per_byte": 1.6129334437067215,
      "learning_rate": 0.00016779003166571981,
      "loss_nats": 1.1180002689361572,
      "step": 4150,
      "unclipped_grad_norm": 0.7947918176651001
    },
    {
      "bits_per_byte": 1.651558683298698,
      "learning_rate": 0.00016723208254345838,
      "loss_nats": 1.144773244857788,
      "step": 4160,
      "unclipped_grad_norm": 0.7860805988311768
    },
    {
      "bits_per_byte": 1.6828089628563754,
      "learning_rate": 0.00016667409528106481,
      "loss_nats": 1.1664342880249023,
      "step": 4170,
      "unclipped_grad_norm": 0.8207855224609375
    },
    {
      "bits_per_byte": 1.7311515661874024,
      "learning_rate": 0.00016611607941300184,
      "loss_nats": 1.1999428272247314,
      "step": 4180,
      "unclipped_grad_norm": 0.8373809456825256
    },
    {
      "bits_per_byte": 1.6433404923272186,
      "learning_rate": 0.0001655580444742211,
      "loss_nats": 1.139076828956604,
      "step": 4190,
      "unclipped_grad_norm": 0.7621071338653564
    },
    {
      "bits_per_byte": 1.682679287937625,
      "learning_rate": 0.00016499999999999997,
      "loss_nats": 1.166344404220581,
      "step": 4200,
      "unclipped_grad_norm": 0.7672834396362305
    },
    {
      "bits_per_byte": 1.661693964879345,
      "learning_rate": 0.0001644419555257789,
      "loss_nats": 1.1517984867095947,
      "step": 4210,
      "unclipped_grad_norm": 0.7330533862113953
    },
    {
      "bits_per_byte": 1.6972419188994332,
      "learning_rate": 0.0001638839205869981,
      "loss_nats": 1.1764384508132935,
      "step": 4220,
      "unclipped_grad_norm": 0.7866858243942261
    },
    {
      "bits_per_byte": 1.6670219874030747,
      "learning_rate": 0.0001633259047189352,
      "loss_nats": 1.155491590499878,
      "step": 4230,
      "unclipped_grad_norm": 0.7902593612670898
    },
    {
      "bits_per_byte": 1.6114956687455084,
      "learning_rate": 0.0001627679174565416,
      "loss_nats": 1.1170036792755127,
      "step": 4240,
      "unclipped_grad_norm": 0.740251362323761
    },
    {
      "bits_per_byte": 1.6335233786506678,
      "learning_rate": 0.00016220996833428016,
      "loss_nats": 1.1322721242904663,
      "step": 4250,
      "unclipped_grad_norm": 0.7777352333068848
    },
    {
      "bits_per_byte": 1.6115895712728796,
      "learning_rate": 0.00016165206688596191,
      "loss_nats": 1.1170687675476074,
      "step": 4260,
      "unclipped_grad_norm": 0.774424135684967
    },
    {
      "bits_per_byte": 1.619611874005026,
      "learning_rate": 0.00016109422264458335,
      "loss_nats": 1.1226294040679932,
      "step": 4270,
      "unclipped_grad_norm": 0.7803477048873901
    },
    {
      "bits_per_byte": 1.6299772683725275,
      "learning_rate": 0.00016053644514216336,
      "loss_nats": 1.1298141479492188,
      "step": 4280,
      "unclipped_grad_norm": 0.7580373287200928
    },
    {
      "bits_per_byte": 1.6795242662075467,
      "learning_rate": 0.00015997874390958058,
      "loss_nats": 1.164157509803772,
      "step": 4290,
      "unclipped_grad_norm": 0.7638103365898132
    },
    {
      "bits_per_byte": 1.600082040120998,
      "learning_rate": 0.00015942112847641018,
      "loss_nats": 1.109092354774475,
      "step": 4300,
      "unclipped_grad_norm": 0.7825598120689392
    },
    {
      "bits_per_byte": 1.6771565810531184,
      "learning_rate": 0.00015886360837076156,
      "loss_nats": 1.1625163555145264,
      "step": 4310,
      "unclipped_grad_norm": 0.7940689921379089
    },
    {
      "bits_per_byte": 1.6577068910843924,
      "learning_rate": 0.00015830619311911495,
      "loss_nats": 1.149034857749939,
      "step": 4320,
      "unclipped_grad_norm": 0.7976187467575073
    },
    {
      "bits_per_byte": 1.6195226150092283,
      "learning_rate": 0.00015774889224615915,
      "loss_nats": 1.1225675344467163,
      "step": 4330,
      "unclipped_grad_norm": 0.7892199158668518
    },
    {
      "bits_per_byte": 1.655279183984813,
      "learning_rate": 0.00015719171527462834,
      "loss_nats": 1.1473520994186401,
      "step": 4340,
      "unclipped_grad_norm": 0.8144169449806213
    },
    {
      "bits_per_byte": 1.6394932404274176,
      "learning_rate": 0.0001566346717251397,
      "loss_nats": 1.136410117149353,
      "step": 4350,
      "unclipped_grad_norm": 0.8119622468948364
    },
    {
      "bits_per_byte": 1.6287904160989233,
      "learning_rate": 0.00015607777111603056,
      "loss_nats": 1.1289914846420288,
      "step": 4360,
      "unclipped_grad_norm": 0.7554351091384888
    },
    {
      "bits_per_byte": 1.6321829458698431,
      "learning_rate": 0.00015552102296319576,
      "loss_nats": 1.1313430070877075,
      "step": 4370,
      "unclipped_grad_norm": 0.808085024356842
    },
    {
      "bits_per_byte": 1.6820371047193032,
      "learning_rate": 0.00015496443677992516,
      "loss_nats": 1.1658992767333984,
      "step": 4380,
      "unclipped_grad_norm": 0.7708722949028015
    },
    {
      "bits_per_byte": 1.6522423143358775,
      "learning_rate": 0.0001544080220767409,
      "loss_nats": 1.1452471017837524,
      "step": 4390,
      "unclipped_grad_norm": 0.8123121857643127
    },
    {
      "bits_per_byte": 1.6025224739367414,
      "learning_rate": 0.00015385178836123511,
      "loss_nats": 1.1107839345932007,
      "step": 4400,
      "unclipped_grad_norm": 0.7717524766921997
    },
    {
      "bits_per_byte": 1.6853694405624209,
      "learning_rate": 0.00015329574513790726,
      "loss_nats": 1.1682090759277344,
      "step": 4410,
      "unclipped_grad_norm": 0.81525057554245
    },
    {
      "bits_per_byte": 1.6142167782474597,
      "learning_rate": 0.00015273990190800186,
      "loss_nats": 1.1188898086547852,
      "step": 4420,
      "unclipped_grad_norm": 0.7956154346466064
    },
    {
      "bits_per_byte": 1.6755413199962148,
      "learning_rate": 0.0001521842681693461,
      "loss_nats": 1.1613967418670654,
      "step": 4430,
      "unclipped_grad_norm": 0.7907513380050659
    },
    {
      "bits_per_byte": 1.6011154838700328,
      "learning_rate": 0.00015162885341618752,
      "loss_nats": 1.1098086833953857,
      "step": 4440,
      "unclipped_grad_norm": 0.7607184648513794
    },
    {
      "bits_per_byte": 1.5990663105850023,
      "learning_rate": 0.00015107366713903178,
      "loss_nats": 1.1083883047103882,
      "step": 4450,
      "unclipped_grad_norm": 0.7442869544029236
    },
    {
      "bits_per_byte": 1.5859970049806358,
      "learning_rate": 0.0001505187188244806,
      "loss_nats": 1.0993293523788452,
      "step": 4460,
      "unclipped_grad_norm": 0.7537500262260437
    },
    {
      "bits_per_byte": 1.6451918855637568,
      "learning_rate": 0.00014996401795506952,
      "loss_nats": 1.1403601169586182,
      "step": 4470,
      "unclipped_grad_norm": 0.7931647896766663
    },
    {
      "bits_per_byte": 1.6505943765753102,
      "learning_rate": 0.00014940957400910585,
      "loss_nats": 1.1441048383712769,
      "step": 4480,
      "unclipped_grad_norm": 0.9106901288032532
    },
    {
      "bits_per_byte": 1.6301040195862135,
      "learning_rate": 0.00014885539646050699,
      "loss_nats": 1.1299020051956177,
      "step": 4490,
      "unclipped_grad_norm": 0.7690432667732239
    },
    {
      "bits_per_byte": 1.6520481459230536,
      "learning_rate": 0.00014830149477863805,
      "loss_nats": 1.1451125144958496,
      "step": 4500,
      "unclipped_grad_norm": 0.7990603446960449
    },
    {
      "bits_per_byte": 1.5968599451570833,
      "learning_rate": 0.00014774787842815066,
      "loss_nats": 1.1068589687347412,
      "step": 4510,
      "unclipped_grad_norm": 0.8130239248275757
    },
    {
      "bits_per_byte": 1.6430763269754933,
      "learning_rate": 0.00014719455686882066,
      "loss_nats": 1.138893723487854,
      "step": 4520,
      "unclipped_grad_norm": 0.7745712399482727
    },
    {
      "bits_per_byte": 1.660709880151108,
      "learning_rate": 0.00014664153955538695,
      "loss_nats": 1.1511163711547852,
      "step": 4530,
      "unclipped_grad_norm": 0.7695059776306152
    },
    {
      "bits_per_byte": 1.707295852686222,
      "learning_rate": 0.00014608883593738946,
      "loss_nats": 1.1834073066711426,
      "step": 4540,
      "unclipped_grad_norm": 0.8436031937599182
    },
    {
      "bits_per_byte": 1.5623325361872151,
      "learning_rate": 0.00014553645545900817,
      "loss_nats": 1.0829263925552368,
      "step": 4550,
      "unclipped_grad_norm": 0.7549967169761658
    },
    {
      "bits_per_byte": 1.6257084869954619,
      "learning_rate": 0.0001449844075589013,
      "loss_nats": 1.1268552541732788,
      "step": 4560,
      "unclipped_grad_norm": 0.8449703454971313
    },
    {
      "bits_per_byte": 1.6175644205465043,
      "learning_rate": 0.0001444327016700444,
      "loss_nats": 1.1212102174758911,
      "step": 4570,
      "unclipped_grad_norm": 0.7528937458992004
    },
    {
      "bits_per_byte": 1.570274179056107,
      "learning_rate": 0.0001438813472195688,
      "loss_nats": 1.0884311199188232,
      "step": 4580,
      "unclipped_grad_norm": 0.790579617023468
    },
    {
      "bits_per_byte": 1.609637052237634,
      "learning_rate": 0.00014333035362860093,
      "loss_nats": 1.1157153844833374,
      "step": 4590,
      "unclipped_grad_norm": 0.7935193181037903
    },
    {
      "bits_per_byte": 1.6088013885370918,
      "learning_rate": 0.00014277973031210088,
      "loss_nats": 1.1151361465454102,
      "step": 4600,
      "unclipped_grad_norm": 0.7915687561035156
    },
    {
      "bits_per_byte": 1.6615916352020816,
      "learning_rate": 0.00014222948667870207,
      "loss_nats": 1.151727557182312,
      "step": 4610,
      "unclipped_grad_norm": 0.8493216633796692
    },
    {
      "bits_per_byte": 1.6675298521710726,
      "learning_rate": 0.00014167963213054991,
      "loss_nats": 1.1558436155319214,
      "step": 4620,
      "unclipped_grad_norm": 0.8179065585136414
    },
    {
      "bits_per_byte": 1.6286588493710132,
      "learning_rate": 0.00014113017606314152,
      "loss_nats": 1.1289002895355225,
      "step": 4630,
      "unclipped_grad_norm": 0.8027536273002625
    },
    {
      "bits_per_byte": 1.7031861552612044,
      "learning_rate": 0.00014058112786516518,
      "loss_nats": 1.180558681488037,
      "step": 4640,
      "unclipped_grad_norm": 0.8179929256439209
    },
    {
      "bits_per_byte": 1.6059254946493637,
      "learning_rate": 0.0001400324969183396,
      "loss_nats": 1.113142728805542,
      "step": 4650,
      "unclipped_grad_norm": 0.8228299021720886
    },
    {
      "bits_per_byte": 1.5739184914278899,
      "learning_rate": 0.00013948429259725406,
      "loss_nats": 1.0909571647644043,
      "step": 4660,
      "unclipped_grad_norm": 0.820624589920044
    },
    {
      "bits_per_byte": 1.7237126286069824,
      "learning_rate": 0.00013893652426920776,
      "loss_nats": 1.194786548614502,
      "step": 4670,
      "unclipped_grad_norm": 0.8582949638366699
    },
    {
      "bits_per_byte": 1.7009104808249877,
      "learning_rate": 0.00013838920129405016,
      "loss_nats": 1.1789813041687012,
      "step": 4680,
      "unclipped_grad_norm": 0.8158925175666809
    },
    {
      "bits_per_byte": 1.6679322915740917,
      "learning_rate": 0.0001378423330240207,
      "loss_nats": 1.1561225652694702,
      "step": 4690,
      "unclipped_grad_norm": 0.849236249923706
    },
    {
      "bits_per_byte": 1.589747430648001,
      "learning_rate": 0.00013729592880358936,
      "loss_nats": 1.101928949356079,
      "step": 4700,
      "unclipped_grad_norm": 0.7403190732002258
    },
    {
      "bits_per_byte": 1.6742342518496576,
      "learning_rate": 0.00013674999796929659,
      "loss_nats": 1.1604907512664795,
      "step": 4710,
      "unclipped_grad_norm": 0.8582251071929932
    },
    {
      "bits_per_byte": 1.607032031025015,
      "learning_rate": 0.00013620454984959417,
      "loss_nats": 1.1139097213745117,
      "step": 4720,
      "unclipped_grad_norm": 0.8400667905807495
    },
    {
      "bits_per_byte": 1.5176990826873464,
      "learning_rate": 0.00013565959376468542,
      "loss_nats": 1.0519888401031494,
      "step": 4730,
      "unclipped_grad_norm": 0.7827285528182983
    },
    {
      "bits_per_byte": 1.5868581221135059,
      "learning_rate": 0.00013511513902636634,
      "loss_nats": 1.099926233291626,
      "step": 4740,
      "unclipped_grad_norm": 0.7815871238708496
    },
    {
      "bits_per_byte": 1.6377919880450826,
      "learning_rate": 0.0001345711949378662,
      "loss_nats": 1.1352308988571167,
      "step": 4750,
      "unclipped_grad_norm": 0.7846289873123169
    },
    {
      "bits_per_byte": 1.6125860387519786,
      "learning_rate": 0.00013402777079368853,
      "loss_nats": 1.1177594661712646,
      "step": 4760,
      "unclipped_grad_norm": 0.8307337164878845
    },
    {
      "bits_per_byte": 1.5936545325103024,
      "learning_rate": 0.0001334848758794528,
      "loss_nats": 1.1046371459960938,
      "step": 4770,
      "unclipped_grad_norm": 0.8213374614715576
    },
    {
      "bits_per_byte": 1.6121588338472355,
      "learning_rate": 0.00013294251947173504,
      "loss_nats": 1.1174633502960205,
      "step": 4780,
      "unclipped_grad_norm": 0.8224621415138245
    },
    {
      "bits_per_byte": 1.5705015401205475,
      "learning_rate": 0.00013240071083790996,
      "loss_nats": 1.0885887145996094,
      "step": 4790,
      "unclipped_grad_norm": 0.8122685551643372
    },
    {
      "bits_per_byte": 1.5722774329733569,
      "learning_rate": 0.0001318594592359921,
      "loss_nats": 1.0898196697235107,
      "step": 4800,
      "unclipped_grad_norm": 0.8455460667610168
    },
    {
      "bits_per_byte": 1.6480244398234671,
      "learning_rate": 0.0001313187739144781,
      "loss_nats": 1.1423234939575195,
      "step": 4810,
      "unclipped_grad_norm": 0.8817458748817444
    },
    {
      "bits_per_byte": 1.574972745077679,
      "learning_rate": 0.0001307786641121882,
      "loss_nats": 1.0916879177093506,
      "step": 4820,
      "unclipped_grad_norm": 0.7954387664794922
    },
    {
      "bits_per_byte": 1.608869665649484,
      "learning_rate": 0.00013023913905810876,
      "loss_nats": 1.1151834726333618,
      "step": 4830,
      "unclipped_grad_norm": 0.8687117695808411
    },
    {
      "bits_per_byte": 1.5985519104762718,
      "learning_rate": 0.00012970020797123432,
      "loss_nats": 1.1080317497253418,
      "step": 4840,
      "unclipped_grad_norm": 0.8505465388298035
    },
    {
      "bits_per_byte": 1.6228387844831649,
      "learning_rate": 0.00012916188006041027,
      "loss_nats": 1.1248661279678345,
      "step": 4850,
      "unclipped_grad_norm": 0.7912812232971191
    },
    {
      "bits_per_byte": 1.5432041098103009,
      "learning_rate": 0.00012862416452417516,
      "loss_nats": 1.0696675777435303,
      "step": 4860,
      "unclipped_grad_norm": 0.7933070063591003
    },
    {
      "bits_per_byte": 1.635918581029234,
      "learning_rate": 0.00012808707055060405,
      "loss_nats": 1.13393235206604,
      "step": 4870,
      "unclipped_grad_norm": 0.8805916905403137
    },
    {
      "bits_per_byte": 1.4766798445953597,
      "learning_rate": 0.000127550607317151,
      "loss_nats": 1.0235564708709717,
      "step": 4880,
      "unclipped_grad_norm": 0.8066675066947937
    },
    {
      "bits_per_byte": 1.5344280071194227,
      "learning_rate": 0.00012701478399049248,
      "loss_nats": 1.0635844469070435,
      "step": 4890,
      "unclipped_grad_norm": 0.8088182806968689
    },
    {
      "bits_per_byte": 1.6101516243290155,
      "learning_rate": 0.0001264796097263708,
      "loss_nats": 1.1160720586776733,
      "step": 4900,
      "unclipped_grad_norm": 0.8653903603553772
    },
    {
      "bits_per_byte": 1.6779198400576476,
      "learning_rate": 0.0001259450936694375,
      "loss_nats": 1.1630454063415527,
      "step": 4910,
      "unclipped_grad_norm": 0.8391038179397583
    },
    {
      "bits_per_byte": 1.6284904783558185,
      "learning_rate": 0.00012541124495309727,
      "loss_nats": 1.1287835836410522,
      "step": 4920,
      "unclipped_grad_norm": 0.8723531365394592
    },
    {
      "bits_per_byte": 1.5831391692712484,
      "learning_rate": 0.00012487807269935167,
      "loss_nats": 1.0973484516143799,
      "step": 4930,
      "unclipped_grad_norm": 0.7553638815879822
    },
    {
      "bits_per_byte": 1.5755332365368409,
      "learning_rate": 0.00012434558601864349,
      "loss_nats": 1.0920764207839966,
      "step": 4940,
      "unclipped_grad_norm": 0.8158645033836365
    },
    {
      "bits_per_byte": 1.530142715407874,
      "learning_rate": 0.0001238137940097009,
      "loss_nats": 1.0606141090393066,
      "step": 4950,
      "unclipped_grad_norm": 0.7773033380508423
    },
    {
      "bits_per_byte": 1.5220387209165676,
      "learning_rate": 0.0001232827057593821,
      "loss_nats": 1.0549968481063843,
      "step": 4960,
      "unclipped_grad_norm": 0.8325244784355164
    },
    {
      "bits_per_byte": 1.6031579498316795,
      "learning_rate": 0.00012275233034251992,
      "loss_nats": 1.1112244129180908,
      "step": 4970,
      "unclipped_grad_norm": 0.8591333627700806
    },
    {
      "bits_per_byte": 1.6532000857185325,
      "learning_rate": 0.0001222226768217669,
      "loss_nats": 1.1459109783172607,
      "step": 4980,
      "unclipped_grad_norm": 0.8351151347160339
    },
    {
      "bits_per_byte": 1.585272098107249,
      "learning_rate": 0.00012169375424744034,
      "loss_nats": 1.0988268852233887,
      "step": 4990,
      "unclipped_grad_norm": 0.8779996037483215
    },
    {
      "bits_per_byte": 1.5394700224947675,
      "learning_rate": 0.00012116557165736772,
      "loss_nats": 1.0670793056488037,
      "step": 5000,
      "unclipped_grad_norm": 0.7928086519241333
    },
    {
      "bits_per_byte": 1.5552827953456976,
      "learning_rate": 0.00012063813807673214,
      "loss_nats": 1.0780398845672607,
      "step": 5010,
      "unclipped_grad_norm": 0.8201308846473694
    },
    {
      "bits_per_byte": 1.5898205232746179,
      "learning_rate": 0.00012011146251791832,
      "loss_nats": 1.1019796133041382,
      "step": 5020,
      "unclipped_grad_norm": 0.7774789929389954
    },
    {
      "bits_per_byte": 1.5697566832596608,
      "learning_rate": 0.0001195855539803584,
      "loss_nats": 1.088072419166565,
      "step": 5030,
      "unclipped_grad_norm": 0.7855319380760193
    },
    {
      "bits_per_byte": 1.539816567536256,
      "learning_rate": 0.00011906042145037817,
      "loss_nats": 1.0673195123672485,
      "step": 5040,
      "unclipped_grad_norm": 0.7920348048210144
    },
    {
      "bits_per_byte": 1.5092143186070297,
      "learning_rate": 0.0001185360739010438,
      "loss_nats": 1.0461076498031616,
      "step": 5050,
      "unclipped_grad_norm": 0.8045132160186768
    },
    {
      "bits_per_byte": 1.5339026001210367,
      "learning_rate": 0.00011801252029200808,
      "loss_nats": 1.0632202625274658,
      "step": 5060,
      "unclipped_grad_norm": 0.8044583797454834
    },
    {
      "bits_per_byte": 1.6138325690054323,
      "learning_rate": 0.00011748976956935777,
      "loss_nats": 1.1186234951019287,
      "step": 5070,
      "unclipped_grad_norm": 0.8723475337028503
    },
    {
      "bits_per_byte": 1.574335549356232,
      "learning_rate": 0.00011696783066546024,
      "loss_nats": 1.091246247291565,
      "step": 5080,
      "unclipped_grad_norm": 0.8741123080253601
    },
    {
      "bits_per_byte": 1.5594206979254568,
      "learning_rate": 0.00011644671249881147,
      "loss_nats": 1.0809080600738525,
      "step": 5090,
      "unclipped_grad_norm": 0.8465037941932678
    },
    {
      "bits_per_byte": 1.5640659493252613,
      "learning_rate": 0.00011592642397388293,
      "loss_nats": 1.0841279029846191,
      "step": 5100,
      "unclipped_grad_norm": 0.783299446105957
    },
    {
      "bits_per_byte": 1.488738924108558,
      "learning_rate": 0.00011540697398097015,
      "loss_nats": 1.0319151878356934,
      "step": 5110,
      "unclipped_grad_norm": 0.8348572254180908
    },
    {
      "bits_per_byte": 1.5847128105266433,
      "learning_rate": 0.00011488837139604016,
      "loss_nats": 1.0984392166137695,
      "step": 5120,
      "unclipped_grad_norm": 0.8110998868942261
    },
    {
      "bits_per_byte": 1.590178763136365,
      "learning_rate": 0.00011437062508058039,
      "loss_nats": 1.1022279262542725,
      "step": 5130,
      "unclipped_grad_norm": 0.8208310604095459
    },
    {
      "bits_per_byte": 1.5494388248693793,
      "learning_rate": 0.00011385374388144673,
      "loss_nats": 1.0739891529083252,
      "step": 5140,
      "unclipped_grad_norm": 0.8678956627845764
    },
    {
      "bits_per_byte": 1.6273239200350162,
      "learning_rate": 0.00011333773663071286,
      "loss_nats": 1.1279749870300293,
      "step": 5150,
      "unclipped_grad_norm": 0.848126232624054
    },
    {
      "bits_per_byte": 1.5085432423033631,
      "learning_rate": 0.00011282261214551897,
      "loss_nats": 1.0456424951553345,
      "step": 5160,
      "unclipped_grad_norm": 0.7922812104225159
    },
    {
      "bits_per_byte": 1.6700107019097687,
      "learning_rate": 0.00011230837922792112,
      "loss_nats": 1.1575632095336914,
      "step": 5170,
      "unclipped_grad_norm": 0.8759046196937561
    },
    {
      "bits_per_byte": 1.5869253673299932,
      "learning_rate": 0.0001117950466647412,
      "loss_nats": 1.0999728441238403,
      "step": 5180,
      "unclipped_grad_norm": 0.8647724390029907
    },
    {
      "bits_per_byte": 1.5237559676854329,
      "learning_rate": 0.00011128262322741627,
      "loss_nats": 1.0561871528625488,
      "step": 5190,
      "unclipped_grad_norm": 0.813337504863739
    },
    {
      "bits_per_byte": 1.556985939537192,
      "learning_rate": 0.00011077111767184913,
      "loss_nats": 1.0792204141616821,
      "step": 5200,
      "unclipped_grad_norm": 0.8388134241104126
    },
    {
      "bits_per_byte": 1.5602121620847276,
      "learning_rate": 0.00011026053873825837,
      "loss_nats": 1.0814566612243652,
      "step": 5210,
      "unclipped_grad_norm": 0.8162683248519897
    },
    {
      "bits_per_byte": 1.6432956048553435,
      "learning_rate": 0.0001097508951510293,
      "loss_nats": 1.1390457153320312,
      "step": 5220,
      "unclipped_grad_norm": 0.8667425513267517
    },
    {
      "bits_per_byte": 1.5501931407760636,
      "learning_rate": 0.0001092421956185646,
      "loss_nats": 1.074512004852295,
      "step": 5230,
      "unclipped_grad_norm": 0.8668702840805054
    },
    {
      "bits_per_byte": 1.5510165937083948,
      "learning_rate": 0.00010873444883313581,
      "loss_nats": 1.075082778930664,
      "step": 5240,
      "unclipped_grad_norm": 0.8049055337905884
    },
    {
      "bits_per_byte": 1.5393878147876552,
      "learning_rate": 0.00010822766347073447,
      "loss_nats": 1.0670223236083984,
      "step": 5250,
      "unclipped_grad_norm": 0.8209428191184998
    },
    {
      "bits_per_byte": 1.540701762240247,
      "learning_rate": 0.00010772184819092424,
      "loss_nats": 1.0679330825805664,
      "step": 5260,
      "unclipped_grad_norm": 0.8524174690246582
    },
    {
      "bits_per_byte": 1.5376201771020872,
      "learning_rate": 0.0001072170116366926,
      "loss_nats": 1.0657970905303955,
      "step": 5270,
      "unclipped_grad_norm": 0.8064137697219849
    },
    {
      "bits_per_byte": 1.5550475230793173,
      "learning_rate": 0.00010671316243430345,
      "loss_nats": 1.0778768062591553,
      "step": 5280,
      "unclipped_grad_norm": 0.8169121146202087
    },
    {
      "bits_per_byte": 1.5423493560355128,
      "learning_rate": 0.00010621030919314944,
      "loss_nats": 1.069075107574463,
      "step": 5290,
      "unclipped_grad_norm": 0.8263237476348877
    },
    {
      "bits_per_byte": 1.5542627662434303,
      "learning_rate": 0.000105708460505605,
      "loss_nats": 1.077332854270935,
      "step": 5300,
      "unclipped_grad_norm": 0.8492859601974487
    },
    {
      "bits_per_byte": 1.5509104804128124,
      "learning_rate": 0.00010520762494687964,
      "loss_nats": 1.0750092267990112,
      "step": 5310,
      "unclipped_grad_norm": 0.8885292410850525
    },
    {
      "bits_per_byte": 1.5517654061702515,
      "learning_rate": 0.00010470781107487111,
      "loss_nats": 1.0756018161773682,
      "step": 5320,
      "unclipped_grad_norm": 0.8326203227043152
    },
    {
      "bits_per_byte": 1.5477100552629064,
      "learning_rate": 0.00010420902743001955,
      "loss_nats": 1.0727908611297607,
      "step": 5330,
      "unclipped_grad_norm": 0.8806997537612915
    },
    {
      "bits_per_byte": 1.503639328996661,
      "learning_rate": 0.00010371128253516117,
      "loss_nats": 1.0422433614730835,
      "step": 5340,
      "unclipped_grad_norm": 0.8234722018241882
    },
    {
      "bits_per_byte": 1.5869049013945407,
      "learning_rate": 0.000103214584895383,
      "loss_nats": 1.0999586582183838,
      "step": 5350,
      "unclipped_grad_norm": 0.8706242442131042
    },
    {
      "bits_per_byte": 1.5072631754329913,
      "learning_rate": 0.00010271894299787715,
      "loss_nats": 1.044755220413208,
      "step": 5360,
      "unclipped_grad_norm": 0.8708282709121704
    },
    {
      "bits_per_byte": 1.471025742965594,
      "learning_rate": 0.00010222436531179622,
      "loss_nats": 1.0196373462677002,
      "step": 5370,
      "unclipped_grad_norm": 0.8303158283233643
    },
    {
      "bits_per_byte": 1.5968417149960918,
      "learning_rate": 0.0001017308602881082,
      "loss_nats": 1.1068463325500488,
      "step": 5380,
      "unclipped_grad_norm": 0.883955180644989
    },
    {
      "bits_per_byte": 1.531507741707772,
      "learning_rate": 0.00010123843635945242,
      "loss_nats": 1.0615602731704712,
      "step": 5390,
      "unclipped_grad_norm": 0.8507125377655029
    },
    {
      "bits_per_byte": 1.5483677169198058,
      "learning_rate": 0.00010074710193999505,
      "loss_nats": 1.073246717453003,
      "step": 5400,
      "unclipped_grad_norm": 0.8522453904151917
    },
    {
      "bits_per_byte": 1.571479089508052,
      "learning_rate": 0.0001002568654252858,
      "loss_nats": 1.089266300201416,
      "step": 5410,
      "unclipped_grad_norm": 0.8313266038894653
    },
    {
      "bits_per_byte": 1.5408727129952047,
      "learning_rate": 9.976773519211404e-05,
      "loss_nats": 1.0680515766143799,
      "step": 5420,
      "unclipped_grad_norm": 0.8902517557144165
    },
    {
      "bits_per_byte": 1.5778336764747818,
      "learning_rate": 9.92797195983659e-05,
      "loss_nats": 1.0936709642410278,
      "step": 5430,
      "unclipped_grad_norm": 0.8956695795059204
    },
    {
      "bits_per_byte": 1.513927503153925,
      "learning_rate": 9.879282698288142e-05,
      "loss_nats": 1.0493745803833008,
      "step": 5440,
      "unclipped_grad_norm": 0.897573709487915
    },
    {
      "bits_per_byte": 1.5078418970531444,
      "learning_rate": 9.830706566531197e-05,
      "loss_nats": 1.0451563596725464,
      "step": 5450,
      "unclipped_grad_norm": 0.8440007567405701
    },
    {
      "bits_per_byte": 1.5030907043404085,
      "learning_rate": 9.782244394597826e-05,
      "loss_nats": 1.0418630838394165,
      "step": 5460,
      "unclipped_grad_norm": 0.8436890244483948
    },
    {
      "bits_per_byte": 1.5464133060754008,
      "learning_rate": 9.733897010572829e-05,
      "loss_nats": 1.0718920230865479,
      "step": 5470,
      "unclipped_grad_norm": 0.8681005239486694
    },
    {
      "bits_per_byte": 1.553998600891705,
      "learning_rate": 9.685665240579612e-05,
      "loss_nats": 1.077149748802185,
      "step": 5480,
      "unclipped_grad_norm": 0.8523774147033691
    },
    {
      "bits_per_byte": 1.543775436176467,
      "learning_rate": 9.637549908766041e-05,
      "loss_nats": 1.070063591003418,
      "step": 5490,
      "unclipped_grad_norm": 0.8906207084655762
    },
    {
      "bits_per_byte": 1.5444891641775478,
      "learning_rate": 9.589551837290389e-05,
      "loss_nats": 1.0705583095550537,
      "step": 5500,
      "unclipped_grad_norm": 0.8658390045166016
    },
    {
      "bits_per_byte": 1.6370650173798855,
      "learning_rate": 9.541671846307257e-05,
      "loss_nats": 1.1347270011901855,
      "step": 5510,
      "unclipped_grad_norm": 0.9161565899848938
    },
    {
      "bits_per_byte": 1.5520985365649729,
      "learning_rate": 9.493910753953597e-05,
      "loss_nats": 1.075832724571228,
      "step": 5520,
      "unclipped_grad_norm": 0.845907986164093
    },
    {
      "bits_per_byte": 1.5215033389244317,
      "learning_rate": 9.446269376334689e-05,
      "loss_nats": 1.0546257495880127,
      "step": 5530,
      "unclipped_grad_norm": 0.8585748672485352
    },
    {
      "bits_per_byte": 1.5943249208833656,
      "learning_rate": 9.39874852751024e-05,
      "loss_nats": 1.1051018238067627,
      "step": 5540,
      "unclipped_grad_norm": 0.9360165596008301
    },
    {
      "bits_per_byte": 1.5176287417831436,
      "learning_rate": 9.351349019480433e-05,
      "loss_nats": 1.0519400835037231,
      "step": 5550,
      "unclipped_grad_norm": 0.8308430910110474
    },
    {
      "bits_per_byte": 1.5164790377621256,
      "learning_rate": 9.304071662172086e-05,
      "loss_nats": 1.0511431694030762,
      "step": 5560,
      "unclipped_grad_norm": 0.894058346748352
    },
    {
      "bits_per_byte": 1.4905304673825959,
      "learning_rate": 9.256917263424787e-05,
      "loss_nats": 1.0331569910049438,
      "step": 5570,
      "unclipped_grad_norm": 0.8399901390075684
    },
    {
      "bits_per_byte": 1.5655759569998369,
      "learning_rate": 9.209886628977114e-05,
      "loss_nats": 1.085174560546875,
      "step": 5580,
      "unclipped_grad_norm": 0.8579773902893066
    },
    {
      "bits_per_byte": 1.482571282283316,
      "learning_rate": 9.162980562452843e-05,
      "loss_nats": 1.0276401042938232,
      "step": 5590,
      "unclipped_grad_norm": 0.8161733746528625
    },
    {
      "bits_per_byte": 1.5496305855250914,
      "learning_rate": 9.116199865347238e-05,
      "loss_nats": 1.0741220712661743,
      "step": 5600,
      "unclipped_grad_norm": 0.8440633416175842
    },
    {
      "bits_per_byte": 1.5466313800766949,
      "learning_rate": 9.069545337013333e-05,
      "loss_nats": 1.0720431804656982,
      "step": 5610,
      "unclipped_grad_norm": 0.8384022116661072
    },
    {
      "bits_per_byte": 1.55366271877457,
      "learning_rate": 9.0230177746483e-05,
      "loss_nats": 1.0769169330596924,
      "step": 5620,
      "unclipped_grad_norm": 0.8643615245819092
    },
    {
      "bits_per_byte": 1.5359743031333302,
      "learning_rate": 8.976617973279797e-05,
      "loss_nats": 1.0646562576293945,
      "step": 5630,
      "unclipped_grad_norm": 0.8559057712554932
    },
    {
      "bits_per_byte": 1.5154779267514533,
      "learning_rate": 8.930346725752419e-05,
      "loss_nats": 1.050449252128601,
      "step": 5640,
      "unclipped_grad_norm": 0.8564271330833435
    },
    {
      "bits_per_byte": 1.5399061704973558,
      "learning_rate": 8.88420482271411e-05,
      "loss_nats": 1.0673816204071045,
      "step": 5650,
      "unclipped_grad_norm": 0.899230420589447
    },
    {
      "bits_per_byte": 1.6017691899259623,
      "learning_rate": 8.838193052602691e-05,
      "loss_nats": 1.1102617979049683,
      "step": 5660,
      "unclipped_grad_norm": 0.8796955943107605
    },
    {
      "bits_per_byte": 1.5203328250026595,
      "learning_rate": 8.792312201632352e-05,
      "loss_nats": 1.05381441116333,
      "step": 5670,
      "unclipped_grad_norm": 0.8509314060211182
    },
    {
      "bits_per_byte": 1.5301421994599214,
      "learning_rate": 8.746563053780261e-05,
      "loss_nats": 1.060613751411438,
      "step": 5680,
      "unclipped_grad_norm": 0.8624072074890137
    },
    {
      "bits_per_byte": 1.6028342784827556,
      "learning_rate": 8.700946390773114e-05,
      "loss_nats": 1.1110000610351562,
      "step": 5690,
      "unclipped_grad_norm": 0.8914096355438232
    },
    {
      "bits_per_byte": 1.526354281574668,
      "learning_rate": 8.655462992073833e-05,
      "loss_nats": 1.057988166809082,
      "step": 5700,
      "unclipped_grad_norm": 0.8579604625701904
    },
    {
      "bits_per_byte": 1.5347007716036911,
      "learning_rate": 8.610113634868199e-05,
      "loss_nats": 1.063773512840271,
      "step": 5710,
      "unclipped_grad_norm": 0.8810272216796875
    },
    {
      "bits_per_byte": 1.5491378552303696,
      "learning_rate": 8.564899094051613e-05,
      "loss_nats": 1.0737805366516113,
      "step": 5720,
      "unclipped_grad_norm": 0.8394830226898193
    },
    {
      "bits_per_byte": 1.5456280332915613,
      "learning_rate": 8.519820142215816e-05,
      "loss_nats": 1.071347713470459,
      "step": 5730,
      "unclipped_grad_norm": 0.865943968296051
    },
    {
      "bits_per_byte": 1.5433466834278662,
      "learning_rate": 8.474877549635728e-05,
      "loss_nats": 1.0697664022445679,
      "step": 5740,
      "unclipped_grad_norm": 0.886944591999054
    },
    {
      "bits_per_byte": 1.5488365416260579,
      "learning_rate": 8.430072084256247e-05,
      "loss_nats": 1.0735716819763184,
      "step": 5750,
      "unclipped_grad_norm": 0.9145897626876831
    },
    {
      "bits_per_byte": 1.5636783004302166,
      "learning_rate": 8.385404511679161e-05,
      "loss_nats": 1.0838592052459717,
      "step": 5760,
      "unclipped_grad_norm": 0.9009037017822266
    },
    {
      "bits_per_byte": 1.511110599315443,
      "learning_rate": 8.340875595150034e-05,
      "loss_nats": 1.0474220514297485,
      "step": 5770,
      "unclipped_grad_norm": 0.844070315361023
    },
    {
      "bits_per_byte": 1.494012772097266,
      "learning_rate": 8.296486095545203e-05,
      "loss_nats": 1.035570740699768,
      "step": 5780,
      "unclipped_grad_norm": 0.8943845629692078
    },
    {
      "bits_per_byte": 1.5135893852623288,
      "learning_rate": 8.252236771358729e-05,
      "loss_nats": 1.049140214920044,
      "step": 5790,
      "unclipped_grad_norm": 0.8841454386711121
    },
    {
      "bits_per_byte": 1.4905120652389536,
      "learning_rate": 8.208128378689484e-05,
      "loss_nats": 1.033144235610962,
      "step": 5800,
      "unclipped_grad_norm": 0.8580894470214844
    },
    {
      "bits_per_byte": 1.5735937881830608,
      "learning_rate": 8.16416167122819e-05,
      "loss_nats": 1.0907320976257324,
      "step": 5810,
      "unclipped_grad_norm": 0.933175265789032
    },
    {
      "bits_per_byte": 1.5042643139498963,
      "learning_rate": 8.120337400244578e-05,
      "loss_nats": 1.042676568031311,
      "step": 5820,
      "unclipped_grad_norm": 0.828717052936554
    },
    {
      "bits_per_byte": 1.5156406223391694,
      "learning_rate": 8.076656314574521e-05,
      "loss_nats": 1.0505620241165161,
      "step": 5830,
      "unclipped_grad_norm": 0.9511673450469971
    },
    {
      "bits_per_byte": 1.5092205099824607,
      "learning_rate": 8.03311916060725e-05,
      "loss_nats": 1.0461119413375854,
      "step": 5840,
      "unclipped_grad_norm": 0.8395604491233826
    },
    {
      "bits_per_byte": 1.477936693807865,
      "learning_rate": 7.989726682272606e-05,
      "loss_nats": 1.0244276523590088,
      "step": 5850,
      "unclipped_grad_norm": 0.9143134355545044
    },
    {
      "bits_per_byte": 1.5336375748560571,
      "learning_rate": 7.946479621028318e-05,
      "loss_nats": 1.063036561012268,
      "step": 5860,
      "unclipped_grad_norm": 0.9014923572540283
    },
    {
      "bits_per_byte": 1.5402322496033916,
      "learning_rate": 7.903378715847343e-05,
      "loss_nats": 1.0676076412200928,
      "step": 5870,
      "unclipped_grad_norm": 0.8728002905845642
    },
    {
      "bits_per_byte": 1.555285031120159,
      "learning_rate": 7.860424703205224e-05,
      "loss_nats": 1.078041434288025,
      "step": 5880,
      "unclipped_grad_norm": 0.8936865925788879
    },
    {
      "bits_per_byte": 1.564054942435606,
      "learning_rate": 7.817618317067534e-05,
      "loss_nats": 1.084120273590088,
      "step": 5890,
      "unclipped_grad_norm": 0.8752698302268982
    },
    {
      "bits_per_byte": 1.5458715607251832,
      "learning_rate": 7.774960288877295e-05,
      "loss_nats": 1.071516513824463,
      "step": 5900,
      "unclipped_grad_norm": 0.8957662582397461
    },
    {
      "bits_per_byte": 1.5568008862048637,
      "learning_rate": 7.732451347542522e-05,
      "loss_nats": 1.0790921449661255,
      "step": 5910,
      "unclipped_grad_norm": 0.8808028101921082
    },
    {
      "bits_per_byte": 1.4896764015384112,
      "learning_rate": 7.690092219423722e-05,
      "loss_nats": 1.0325649976730347,
      "step": 5920,
      "unclipped_grad_norm": 0.8660398721694946
    },
    {
      "bits_per_byte": 1.5908711652887386,
      "learning_rate": 7.647883628321536e-05,
      "loss_nats": 1.102707862854004,
      "step": 5930,
      "unclipped_grad_norm": 0.9604227542877197
    },
    {
      "bits_per_byte": 1.4574508083676907,
      "learning_rate": 7.605826295464316e-05,
      "loss_nats": 1.010227918624878,
      "step": 5940,
      "unclipped_grad_norm": 0.8380697965621948
    },
    {
      "bits_per_byte": 1.5131755950043528,
      "learning_rate": 7.563920939495851e-05,
      "loss_nats": 1.0488533973693848,
      "step": 5950,
      "unclipped_grad_norm": 0.8392192125320435
    },
    {
      "bits_per_byte": 1.5452194025131114,
      "learning_rate": 7.522168276463047e-05,
      "loss_nats": 1.0710644721984863,
      "step": 5960,
      "unclipped_grad_norm": 0.932554304599762
    },
    {
      "bits_per_byte": 1.5213482105733536,
      "learning_rate": 7.480569019803719e-05,
      "loss_nats": 1.054518222808838,
      "step": 5970,
      "unclipped_grad_norm": 0.927780270576477
    },
    {
      "bits_per_byte": 1.5601101763727658,
      "learning_rate": 7.439123880334388e-05,
      "loss_nats": 1.0813859701156616,
      "step": 5980,
      "unclipped_grad_norm": 0.8920019268989563
    },
    {
      "bits_per_byte": 1.5382338112003657,
      "learning_rate": 7.39783356623813e-05,
      "loss_nats": 1.0662224292755127,
      "step": 5990,
      "unclipped_grad_norm": 0.9537093639373779
    },
    {
      "bits_per_byte": 1.4675097296513546,
      "learning_rate": 7.356698783052497e-05,
      "loss_nats": 1.017200231552124,
      "step": 6000,
      "unclipped_grad_norm": 0.8773758411407471
    },
    {
      "bits_per_byte": 1.506333093257125,
      "learning_rate": 7.315720233657431e-05,
      "loss_nats": 1.0441105365753174,
      "step": 6010,
      "unclipped_grad_norm": 0.8785988092422485
    },
    {
      "bits_per_byte": 1.4794742187065786,
      "learning_rate": 7.274898618263286e-05,
      "loss_nats": 1.0254933834075928,
      "step": 6020,
      "unclipped_grad_norm": 0.8875186443328857
    },
    {
      "bits_per_byte": 1.5245675538148544,
      "learning_rate": 7.234234634398831e-05,
      "loss_nats": 1.056749701499939,
      "step": 6030,
      "unclipped_grad_norm": 0.88789963722229
    },
    {
      "bits_per_byte": 1.5628784091210537,
      "learning_rate": 7.193728976899362e-05,
      "loss_nats": 1.083304762840271,
      "step": 6040,
      "unclipped_grad_norm": 0.9149079322814941
    },
    {
      "bits_per_byte": 1.4881958028971334,
      "learning_rate": 7.153382337894797e-05,
      "loss_nats": 1.031538724899292,
      "step": 6050,
      "unclipped_grad_norm": 0.9299342632293701
    },
    {
      "bits_per_byte": 1.4785307218839452,
      "learning_rate": 7.113195406797887e-05,
      "loss_nats": 1.0248394012451172,
      "step": 6060,
      "unclipped_grad_norm": 0.8783867359161377
    },
    {
      "bits_per_byte": 1.5509557118499895,
      "learning_rate": 7.073168870292397e-05,
      "loss_nats": 1.075040578842163,
      "step": 6070,
      "unclipped_grad_norm": 0.9337034821510315
    },
    {
      "bits_per_byte": 1.4914187577743023,
      "learning_rate": 7.033303412321404e-05,
      "loss_nats": 1.0337727069854736,
      "step": 6080,
      "unclipped_grad_norm": 0.9420101046562195
    },
    {
      "bits_per_byte": 1.4968499698885498,
      "learning_rate": 6.993599714075586e-05,
      "loss_nats": 1.0375373363494873,
      "step": 6090,
      "unclipped_grad_norm": 0.8664546012878418
    },
    {
      "bits_per_byte": 1.4906561867003767,
      "learning_rate": 6.954058453981609e-05,
      "loss_nats": 1.0332441329956055,
      "step": 6100,
      "unclipped_grad_norm": 0.8687640428543091
    },
    {
      "bits_per_byte": 1.5538661742505406,
      "learning_rate": 6.914680307690504e-05,
      "loss_nats": 1.077057957649231,
      "step": 6110,
      "unclipped_grad_norm": 0.8986173868179321
    },
    {
      "bits_per_byte": 1.5476213122150613,
      "learning_rate": 6.875465948066139e-05,
      "loss_nats": 1.0727293491363525,
      "step": 6120,
      "unclipped_grad_norm": 0.9189077019691467
    },
    {
      "bits_per_byte": 1.4789527673091625,
      "learning_rate": 6.836416045173733e-05,
      "loss_nats": 1.0251319408416748,
      "step": 6130,
      "unclipped_grad_norm": 0.8851786851882935
    },
    {
      "bits_per_byte": 1.4768556108645414,
      "learning_rate": 6.797531266268377e-05,
      "loss_nats": 1.0236783027648926,
      "step": 6140,
      "unclipped_grad_norm": 0.9043483138084412
    },
    {
      "bits_per_byte": 1.552988202817886,
      "learning_rate": 6.758812275783663e-05,
      "loss_nats": 1.0764493942260742,
      "step": 6150,
      "unclipped_grad_norm": 0.9005390405654907
    },
    {
      "bits_per_byte": 1.3994871515820966,
      "learning_rate": 6.720259735320298e-05,
      "loss_nats": 0.970050573348999,
      "step": 6160,
      "unclipped_grad_norm": 0.8554620742797852
    },
    {
      "bits_per_byte": 1.5302097886417105,
      "learning_rate": 6.681874303634842e-05,
      "loss_nats": 1.0606606006622314,
      "step": 6170,
      "unclipped_grad_norm": 0.8954014778137207
    },
    {
      "bits_per_byte": 1.4938230752333643,
      "learning_rate": 6.643656636628402e-05,
      "loss_nats": 1.0354392528533936,
      "step": 6180,
      "unclipped_grad_norm": 0.892278790473938
    },
    {
      "bits_per_byte": 1.516749394489282,
      "learning_rate": 6.605607387335466e-05,
      "loss_nats": 1.05133056640625,
      "step": 6190,
      "unclipped_grad_norm": 0.9185703992843628
    },
    {
      "bits_per_byte": 1.5086576107661869,
      "learning_rate": 6.567727205912724e-05,
      "loss_nats": 1.0457217693328857,
      "step": 6200,
      "unclipped_grad_norm": 0.8927735090255737
    },
    {
      "bits_per_byte": 1.5088584865023946,
      "learning_rate": 6.530016739627959e-05,
      "loss_nats": 1.045861005783081,
      "step": 6210,
      "unclipped_grad_norm": 0.8906724452972412
    },
    {
      "bits_per_byte": 1.5118191678369979,
      "learning_rate": 6.492476632848993e-05,
      "loss_nats": 1.0479131937026978,
      "step": 6220,
      "unclipped_grad_norm": 0.9077256917953491
    },
    {
      "bits_per_byte": 1.4471287536282067,
      "learning_rate": 6.455107527032678e-05,
      "loss_nats": 1.0030732154846191,
      "step": 6230,
      "unclipped_grad_norm": 0.8839235305786133
    },
    {
      "bits_per_byte": 1.5076786855174757,
      "learning_rate": 6.417910060713926e-05,
      "loss_nats": 1.0450432300567627,
      "step": 6240,
      "unclipped_grad_norm": 0.9515005946159363
    },
    {
      "bits_per_byte": 1.5408861276419719,
      "learning_rate": 6.380884869494799e-05,
      "loss_nats": 1.0680608749389648,
      "step": 6250,
      "unclipped_grad_norm": 0.901561975479126
    },
    {
      "bits_per_byte": 1.4200397662908057,
      "learning_rate": 6.34403258603367e-05,
      "loss_nats": 0.9842965602874756,
      "step": 6260,
      "unclipped_grad_norm": 0.889621376991272
    },
    {
      "bits_per_byte": 1.5216213190229235,
      "learning_rate": 6.30735384003437e-05,
      "loss_nats": 1.0547075271606445,
      "step": 6270,
      "unclipped_grad_norm": 0.9411186575889587
    },
    {
      "bits_per_byte": 1.5224643779774532,
      "learning_rate": 6.270849258235484e-05,
      "loss_nats": 1.0552918910980225,
      "step": 6280,
      "unclipped_grad_norm": 0.9406285881996155
    },
    {
      "bits_per_byte": 1.452789390598705,
      "learning_rate": 6.234519464399581e-05,
      "loss_nats": 1.0069968700408936,
      "step": 6290,
      "unclipped_grad_norm": 0.8839805126190186
    },
    {
      "bits_per_byte": 1.5074908804627336,
      "learning_rate": 6.198365079302614e-05,
      "loss_nats": 1.0449130535125732,
      "step": 6300,
      "unclipped_grad_norm": 0.9184122681617737
    },
    {
      "bits_per_byte": 1.478450577968643,
      "learning_rate": 6.16238672072326e-05,
      "loss_nats": 1.0247838497161865,
      "step": 6310,
      "unclipped_grad_norm": 0.8948619365692139
    },
    {
      "bits_per_byte": 1.4214500240278807,
      "learning_rate": 6.126585003432406e-05,
      "loss_nats": 0.985274076461792,
      "step": 6320,
      "unclipped_grad_norm": 0.8749227523803711
    },
    {
      "bits_per_byte": 1.5185149683830395,
      "learning_rate": 6.090960539182614e-05,
      "loss_nats": 1.0525543689727783,
      "step": 6330,
      "unclipped_grad_norm": 0.9816636443138123
    },
    {
      "bits_per_byte": 1.4759477144506365,
      "learning_rate": 6.0555139366976935e-05,
      "loss_nats": 1.023048996925354,
      "step": 6340,
      "unclipped_grad_norm": 0.8950188159942627
    },
    {
      "bits_per_byte": 1.587707716408768,
      "learning_rate": 6.020245801662274e-05,
      "loss_nats": 1.1005151271820068,
      "step": 6350,
      "unclipped_grad_norm": 1.01009202003479
    },
    {
      "bits_per_byte": 1.4638088349874379,
      "learning_rate": 5.985156736711483e-05,
      "loss_nats": 1.0146349668502808,
      "step": 6360,
      "unclipped_grad_norm": 0.9816907644271851
    },
    {
      "bits_per_byte": 1.4382974445064014,
      "learning_rate": 5.950247341420618e-05,
      "loss_nats": 0.9969518184661865,
      "step": 6370,
      "unclipped_grad_norm": 0.8970034122467041
    },
    {
      "bits_per_byte": 1.4818757844432269,
      "learning_rate": 5.915518212294935e-05,
      "loss_nats": 1.0271580219268799,
      "step": 6380,
      "unclipped_grad_norm": 0.9058489799499512
    },
    {
      "bits_per_byte": 1.527050123380059,
      "learning_rate": 5.880969942759425e-05,
      "loss_nats": 1.0584704875946045,
      "step": 6390,
      "unclipped_grad_norm": 0.929339587688446
    },
    {
      "bits_per_byte": 1.5156593684481134,
      "learning_rate": 5.846603123148688e-05,
      "loss_nats": 1.0505750179290771,
      "step": 6400,
      "unclipped_grad_norm": 0.9009133577346802
    },
    {
      "bits_per_byte": 1.4969940913499729,
      "learning_rate": 5.812418340696858e-05,
      "loss_nats": 1.0376372337341309,
      "step": 6410,
      "unclipped_grad_norm": 0.957128643989563
    },
    {
      "bits_per_byte": 1.4783625228514012,
      "learning_rate": 5.778416179527541e-05,
      "loss_nats": 1.0247228145599365,
      "step": 6420,
      "unclipped_grad_norm": 0.9414587616920471
    },
    {
      "bits_per_byte": 1.523227980947284,
      "learning_rate": 5.7445972206438635e-05,
      "loss_nats": 1.055821180343628,
      "step": 6430,
      "unclipped_grad_norm": 0.9234302639961243
    },
    {
      "bits_per_byte": 1.4623229048839832,
      "learning_rate": 5.710962041918516e-05,
      "loss_nats": 1.013604998588562,
      "step": 6440,
      "unclipped_grad_norm": 0.9177728295326233
    },
    {
      "bits_per_byte": 1.5306256426914968,
      "learning_rate": 5.677511218083908e-05,
      "loss_nats": 1.0609488487243652,
      "step": 6450,
      "unclipped_grad_norm": 0.9453822374343872
    },
    {
      "bits_per_byte": 1.5847898587542297,
      "learning_rate": 5.644245320722316e-05,
      "loss_nats": 1.0984926223754883,
      "step": 6460,
      "unclipped_grad_norm": 0.9647909998893738
    },
    {
      "bits_per_byte": 1.5690988496201106,
      "learning_rate": 5.61116491825615e-05,
      "loss_nats": 1.0876164436340332,
      "step": 6470,
      "unclipped_grad_norm": 0.9592546820640564
    },
    {
      "bits_per_byte": 1.4544314809491432,
      "learning_rate": 5.5782705759382104e-05,
      "loss_nats": 1.0081350803375244,
      "step": 6480,
      "unclipped_grad_norm": 0.9334172010421753
    },
    {
      "bits_per_byte": 1.5573938823850386,
      "learning_rate": 5.5455628558420546e-05,
      "loss_nats": 1.0795031785964966,
      "step": 6490,
      "unclipped_grad_norm": 0.967822790145874
    },
    {
      "bits_per_byte": 1.5280063469188563,
      "learning_rate": 5.513042316852368e-05,
      "loss_nats": 1.0591332912445068,
      "step": 6500,
      "unclipped_grad_norm": 0.9625112414360046
    },
    {
      "bits_per_byte": 1.4687796495453256,
      "learning_rate": 5.480709514655443e-05,
      "loss_nats": 1.018080472946167,
      "step": 6510,
      "unclipped_grad_norm": 0.9510544538497925
    },
    {
      "bits_per_byte": 1.4284500618832987,
      "learning_rate": 5.448565001729653e-05,
      "loss_nats": 0.9901261329650879,
      "step": 6520,
      "unclipped_grad_norm": 0.8985839486122131
    },
    {
      "bits_per_byte": 1.4546447394362132,
      "learning_rate": 5.41660932733604e-05,
      "loss_nats": 1.0082828998565674,
      "step": 6530,
      "unclipped_grad_norm": 0.907886803150177
    },
    {
      "bits_per_byte": 1.4970680438898438,
      "learning_rate": 5.384843037508902e-05,
      "loss_nats": 1.0376884937286377,
      "step": 6540,
      "unclipped_grad_norm": 0.9381445050239563
    },
    {
      "bits_per_byte": 1.4280130539674563,
      "learning_rate": 5.353266675046498e-05,
      "loss_nats": 0.9898232221603394,
      "step": 6550,
      "unclipped_grad_norm": 0.9031082391738892
    },
    {
      "bits_per_byte": 1.5210671909218436,
      "learning_rate": 5.321880779501729e-05,
      "loss_nats": 1.054323434829712,
      "step": 6560,
      "unclipped_grad_norm": 0.9433532357215881
    },
    {
      "bits_per_byte": 1.479243074023819,
      "learning_rate": 5.290685887172962e-05,
      "loss_nats": 1.0253331661224365,
      "step": 6570,
      "unclipped_grad_norm": 0.9311387538909912
    },
    {
      "bits_per_byte": 1.5051876888023787,
      "learning_rate": 5.2596825310948295e-05,
      "loss_nats": 1.0433166027069092,
      "step": 6580,
      "unclipped_grad_norm": 0.9569925665855408
    },
    {
      "bits_per_byte": 1.494823842278735,
      "learning_rate": 5.228871241029154e-05,
      "loss_nats": 1.0361329317092896,
      "step": 6590,
      "unclipped_grad_norm": 0.8958868384361267
    },
    {
      "bits_per_byte": 1.4302109062541577,
      "learning_rate": 5.198252543455865e-05,
      "loss_nats": 0.9913466572761536,
      "step": 6600,
      "unclipped_grad_norm": 0.9136191010475159
    },
    {
      "bits_per_byte": 1.436879103584736,
      "learning_rate": 5.167826961564027e-05,
      "loss_nats": 0.9959686994552612,
      "step": 6610,
      "unclipped_grad_norm": 0.8710504770278931
    },
    {
      "bits_per_byte": 1.4278971376607748,
      "learning_rate": 5.137595015242884e-05,
      "loss_nats": 0.9897428750991821,
      "step": 6620,
      "unclipped_grad_norm": 0.9145034551620483
    },
    {
      "bits_per_byte": 1.4582168190946336,
      "learning_rate": 5.1075572210729925e-05,
      "loss_nats": 1.010758876800537,
      "step": 6630,
      "unclipped_grad_norm": 0.8999127149581909
    },
    {
      "bits_per_byte": 1.5240701799885592,
      "learning_rate": 5.07771409231737e-05,
      "loss_nats": 1.056404948234558,
      "step": 6640,
      "unclipped_grad_norm": 0.9801992774009705
    },
    {
      "bits_per_byte": 1.5169175935218258,
      "learning_rate": 5.048066138912756e-05,
      "loss_nats": 1.0514471530914307,
      "step": 6650,
      "unclipped_grad_norm": 0.8882671594619751
    },
    {
      "bits_per_byte": 1.5152221885496202,
      "learning_rate": 5.018613867460863e-05,
      "loss_nats": 1.050271987915039,
      "step": 6660,
      "unclipped_grad_norm": 0.9029646515846252
    },
    {
      "bits_per_byte": 1.4896082964086694,
      "learning_rate": 4.989357781219755e-05,
      "loss_nats": 1.0325177907943726,
      "step": 6670,
      "unclipped_grad_norm": 0.9797500371932983
    },
    {
      "bits_per_byte": 1.4764526555135697,
      "learning_rate": 4.960298380095222e-05,
      "loss_nats": 1.023398995399475,
      "step": 6680,
      "unclipped_grad_norm": 0.9356769919395447
    },
    {
      "bits_per_byte": 1.4958203097578342,
      "learning_rate": 4.931436160632256e-05,
      "loss_nats": 1.0368236303329468,
      "step": 6690,
      "unclipped_grad_norm": 0.939789891242981
    },
    {
      "bits_per_byte": 1.5480882451121538,
      "learning_rate": 4.9027716160065485e-05,
      "loss_nats": 1.073053002357483,
      "step": 6700,
      "unclipped_grad_norm": 1.0080333948135376
    },
    {
      "bits_per_byte": 1.525887864625528,
      "learning_rate": 4.874305236016084e-05,
      "loss_nats": 1.0576648712158203,
      "step": 6710,
      "unclipped_grad_norm": 0.9197862148284912
    },
    {
      "bits_per_byte": 1.534906462854123,
      "learning_rate": 4.846037507072752e-05,
      "loss_nats": 1.0639160871505737,
      "step": 6720,
      "unclipped_grad_norm": 0.9543269872665405
    },
    {
      "bits_per_byte": 1.4820493149379474,
      "learning_rate": 4.817968912194049e-05,
      "loss_nats": 1.0272783041000366,
      "step": 6730,
      "unclipped_grad_norm": 0.9817740321159363
    },
    {
      "bits_per_byte": 1.4384499931177168,
      "learning_rate": 4.790099930994813e-05,
      "loss_nats": 0.9970575571060181,
      "step": 6740,
      "unclipped_grad_norm": 0.9710240364074707
    },
    {
      "bits_per_byte": 1.511069151496585,
      "learning_rate": 4.762431039679047e-05,
      "loss_nats": 1.0473933219909668,
      "step": 6750,
      "unclipped_grad_norm": 0.9612263441085815
    },
    {
      "bits_per_byte": 1.4780942299160553,
      "learning_rate": 4.734962711031755e-05,
      "loss_nats": 1.0245368480682373,
      "step": 6760,
      "unclipped_grad_norm": 0.9238441586494446
    },
    {
      "bits_per_byte": 1.3767874194686909,
      "learning_rate": 4.707695414410892e-05,
      "loss_nats": 0.9543163180351257,
      "step": 6770,
      "unclipped_grad_norm": 0.9215719699859619
    },
    {
      "bits_per_byte": 1.4785716537548506,
      "learning_rate": 4.680629615739322e-05,
      "loss_nats": 1.0248677730560303,
      "step": 6780,
      "unclipped_grad_norm": 0.9403936266899109
    },
    {
      "bits_per_byte": 1.4459957319243224,
      "learning_rate": 4.653765777496863e-05,
      "loss_nats": 1.0022878646850586,
      "step": 6790,
      "unclipped_grad_norm": 0.9651554822921753
    },
    {
      "bits_per_byte": 1.5175859181030786,
      "learning_rate": 4.6271043587123986e-05,
      "loss_nats": 1.051910400390625,
      "step": 6800,
      "unclipped_grad_norm": 0.9926899075508118
    },
    {
      "bits_per_byte": 1.5110634760691066,
      "learning_rate": 4.600645814956006e-05,
      "loss_nats": 1.0473893880844116,
      "step": 6810,
      "unclipped_grad_norm": 0.9793192148208618
    },
    {
      "bits_per_byte": 1.5052492585913877,
      "learning_rate": 4.574390598331204e-05,
      "loss_nats": 1.0433592796325684,
      "step": 6820,
      "unclipped_grad_norm": 1.0081095695495605
    },
    {
      "bits_per_byte": 1.466343515295854,
      "learning_rate": 4.548339157467199e-05,
      "loss_nats": 1.0163918733596802,
      "step": 6830,
      "unclipped_grad_norm": 0.960361659526825
    },
    {
      "bits_per_byte": 1.4376153613130795,
      "learning_rate": 4.52249193751124e-05,
      "loss_nats": 0.9964790344238281,
      "step": 6840,
      "unclipped_grad_norm": 0.9702624678611755
    },
    {
      "bits_per_byte": 1.4922177891702109,
      "learning_rate": 4.4968493801209985e-05,
      "loss_nats": 1.0343265533447266,
      "step": 6850,
      "unclipped_grad_norm": 0.9893983006477356
    },
    {
      "bits_per_byte": 1.4659078832412185,
      "learning_rate": 4.471411923457034e-05,
      "loss_nats": 1.016089916229248,
      "step": 6860,
      "unclipped_grad_norm": 0.9693014621734619
    },
    {
      "bits_per_byte": 1.413224953750367,
      "learning_rate": 4.446180002175291e-05,
      "loss_nats": 0.9795728921890259,
      "step": 6870,
      "unclipped_grad_norm": 0.9252057075500488
    },
    {
      "bits_per_byte": 1.4200107012228098,
      "learning_rate": 4.421154047419693e-05,
      "loss_nats": 0.9842764139175415,
      "step": 6880,
      "unclipped_grad_norm": 0.8884434700012207
    },
    {
      "bits_per_byte": 1.4366857090938405,
      "learning_rate": 4.396334486814753e-05,
      "loss_nats": 0.9958346486091614,
      "step": 6890,
      "unclipped_grad_norm": 0.9679457545280457
    },
    {
      "bits_per_byte": 1.5416700245646047,
      "learning_rate": 4.371721744458285e-05,
      "loss_nats": 1.0686042308807373,
      "step": 6900,
      "unclipped_grad_norm": 0.9985603094100952
    },
    {
      "bits_per_byte": 1.4893800754309745,
      "learning_rate": 4.347316240914145e-05,
      "loss_nats": 1.0323596000671387,
      "step": 6910,
      "unclipped_grad_norm": 0.9936915636062622
    },
    {
      "bits_per_byte": 1.4499423897963222,
      "learning_rate": 4.3231183932050546e-05,
      "loss_nats": 1.00502347946167,
      "step": 6920,
      "unclipped_grad_norm": 0.923648476600647
    },
    {
      "bits_per_byte": 1.4406846496917027,
      "learning_rate": 4.299128614805464e-05,
      "loss_nats": 0.9986065030097961,
      "step": 6930,
      "unclipped_grad_norm": 0.9748247861862183
    },
    {
      "bits_per_byte": 1.408974058568991,
      "learning_rate": 4.2753473156344965e-05,
      "loss_nats": 0.9766263961791992,
      "step": 6940,
      "unclipped_grad_norm": 0.9357670545578003
    },
    {
      "bits_per_byte": 1.434796565665438,
      "learning_rate": 4.251774902048942e-05,
      "loss_nats": 0.9945251941680908,
      "step": 6950,
      "unclipped_grad_norm": 0.9624595642089844
    },
    {
      "bits_per_byte": 1.49675830313564,
      "learning_rate": 4.228411776836306e-05,
      "loss_nats": 1.0374737977981567,
      "step": 6960,
      "unclipped_grad_norm": 0.9387544989585876
    },
    {
      "bits_per_byte": 1.4929531869853003,
      "learning_rate": 4.2052583392079427e-05,
      "loss_nats": 1.0348362922668457,
      "step": 6970,
      "unclipped_grad_norm": 0.9343914985656738
    },
    {
      "bits_per_byte": 1.4660633555575984,
      "learning_rate": 4.182314984792214e-05,
      "loss_nats": 1.016197681427002,
      "step": 6980,
      "unclipped_grad_norm": 0.9326976537704468
    },
    {
      "bits_per_byte": 1.492015881538098,
      "learning_rate": 4.159582105627748e-05,
      "loss_nats": 1.034186601638794,
      "step": 6990,
      "unclipped_grad_norm": 1.0069916248321533
    },
    {
      "bits_per_byte": 1.52302985693349,
      "learning_rate": 4.137060090156724e-05,
      "loss_nats": 1.0556838512420654,
      "step": 7000,
      "unclipped_grad_norm": 0.9719612002372742
    },
    {
      "bits_per_byte": 1.512785538352196,
      "learning_rate": 4.114749323218249e-05,
      "loss_nats": 1.0485830307006836,
      "step": 7010,
      "unclipped_grad_norm": 0.9873834848403931
    },
    {
      "bits_per_byte": 1.4473323810868282,
      "learning_rate": 4.0926501860417706e-05,
      "loss_nats": 1.0032143592834473,
      "step": 7020,
      "unclipped_grad_norm": 0.939332127571106
    },
    {
      "bits_per_byte": 1.5166639191118032,
      "learning_rate": 4.070763056240571e-05,
      "loss_nats": 1.0512713193893433,
      "step": 7030,
      "unclipped_grad_norm": 0.9837853312492371
    },
    {
      "bits_per_byte": 1.5072451172546506,
      "learning_rate": 4.049088307805306e-05,
      "loss_nats": 1.0447427034378052,
      "step": 7040,
      "unclipped_grad_norm": 0.9734936952590942
    },
    {
      "bits_per_byte": 1.4796651194490362,
      "learning_rate": 4.027626311097629e-05,
      "loss_nats": 1.0256257057189941,
      "step": 7050,
      "unclipped_grad_norm": 1.005554437637329
    },
    {
      "bits_per_byte": 1.4927048440374544,
      "learning_rate": 4.006377432843846e-05,
      "loss_nats": 1.0346641540527344,
      "step": 7060,
      "unclipped_grad_norm": 0.9903765320777893
    },
    {
      "bits_per_byte": 1.4561590466770602,
      "learning_rate": 3.985342036128656e-05,
      "loss_nats": 1.009332537651062,
      "step": 7070,
      "unclipped_grad_norm": 1.009922981262207
    },
    {
      "bits_per_byte": 1.4353391709289103,
      "learning_rate": 3.964520480388956e-05,
      "loss_nats": 0.9949012994766235,
      "step": 7080,
      "unclipped_grad_norm": 0.9552373290061951
    },
    {
      "bits_per_byte": 1.4780041110070032,
      "learning_rate": 3.943913121407678e-05,
      "loss_nats": 1.0244743824005127,
      "step": 7090,
      "unclipped_grad_norm": 1.0046122074127197
    },
    {
      "bits_per_byte": 1.4728850474040718,
      "learning_rate": 3.923520311307735e-05,
      "loss_nats": 1.0209261178970337,
      "step": 7100,
      "unclipped_grad_norm": 0.9868813157081604
    },
    {
      "bits_per_byte": 1.4862149067244954,
      "learning_rate": 3.9033423985459816e-05,
      "loss_nats": 1.030165672302246,
      "step": 7110,
      "unclipped_grad_norm": 0.9593554735183716
    },
    {
      "bits_per_byte": 1.5328834309320238,
      "learning_rate": 3.88337972790728e-05,
      "loss_nats": 1.062513828277588,
      "step": 7120,
      "unclipped_grad_norm": 1.0129024982452393
    },
    {
      "bits_per_byte": 1.4811555211014134,
      "learning_rate": 3.8636326404985845e-05,
      "loss_nats": 1.0266587734222412,
      "step": 7130,
      "unclipped_grad_norm": 1.0546875
    },
    {
      "bits_per_byte": 1.4724865636020226,
      "learning_rate": 3.844101473743143e-05,
      "loss_nats": 1.0206499099731445,
      "step": 7140,
      "unclipped_grad_norm": 0.9776664972305298
    },
    {
      "bits_per_byte": 1.4357171887955067,
      "learning_rate": 3.824786561374701e-05,
      "loss_nats": 0.9951633214950562,
      "step": 7150,
      "unclipped_grad_norm": 0.9291411638259888
    },
    {
      "bits_per_byte": 1.5232358921492237,
      "learning_rate": 3.805688233431824e-05,
      "loss_nats": 1.0558266639709473,
      "step": 7160,
      "unclipped_grad_norm": 1.0189123153686523
    },
    {
      "bits_per_byte": 1.4618840051589814,
      "learning_rate": 3.786806816252236e-05,
      "loss_nats": 1.0133007764816284,
      "step": 7170,
      "unclipped_grad_norm": 0.9592419266700745
    },
    {
      "bits_per_byte": 1.4762450724539784,
      "learning_rate": 3.768142632467269e-05,
      "loss_nats": 1.0232551097869873,
      "step": 7180,
      "unclipped_grad_norm": 0.9680334329605103
    },
    {
      "bits_per_byte": 1.4441101141402626,
      "learning_rate": 3.7496960009963236e-05,
      "loss_nats": 1.0009808540344238,
      "step": 7190,
      "unclipped_grad_norm": 1.0148857831954956
    },
    {
      "bits_per_byte": 1.472608843266786,
      "learning_rate": 3.7314672370414325e-05,
      "loss_nats": 1.0207346677780151,
      "step": 7200,
      "unclipped_grad_norm": 0.9935786128044128
    },
    {
      "bits_per_byte": 1.5299951542934338,
      "learning_rate": 3.713456652081882e-05,
      "loss_nats": 1.060511827468872,
      "step": 7210,
      "unclipped_grad_norm": 0.9725626111030579
    },
    {
      "bits_per_byte": 1.5238232129019202,
      "learning_rate": 3.695664553868873e-05,
      "loss_nats": 1.0562337636947632,
      "step": 7220,
      "unclipped_grad_norm": 1.0141370296478271
    },
    {
      "bits_per_byte": 1.4390708504873315,
      "learning_rate": 3.678091246420277e-05,
      "loss_nats": 0.9974879026412964,
      "step": 7230,
      "unclipped_grad_norm": 1.0297694206237793
    },
    {
      "bits_per_byte": 1.4632025961431465,
      "learning_rate": 3.660737030015427e-05,
      "loss_nats": 1.0142147541046143,
      "step": 7240,
      "unclipped_grad_norm": 0.9345177412033081
    },
    {
      "bits_per_byte": 1.5031187375124992,
      "learning_rate": 3.6436022011900056e-05,
      "loss_nats": 1.0418825149536133,
      "step": 7250,
      "unclipped_grad_norm": 1.0408201217651367
    },
    {
      "bits_per_byte": 1.3820238612308362,
      "learning_rate": 3.626687052730954e-05,
      "loss_nats": 0.9579459428787231,
      "step": 7260,
      "unclipped_grad_norm": 0.9544695615768433
    },
    {
      "bits_per_byte": 1.4274823155068936,
      "learning_rate": 3.6099918736714955e-05,
      "loss_nats": 0.9894553422927856,
      "step": 7270,
      "unclipped_grad_norm": 0.9940797686576843
    },
    {
      "bits_per_byte": 1.4990891840027836,
      "learning_rate": 3.5935169492861716e-05,
      "loss_nats": 1.0390894412994385,
      "step": 7280,
      "unclipped_grad_norm": 1.013985514640808
    },
    {
      "bits_per_byte": 1.4370805812602216,
      "learning_rate": 3.5772625610859886e-05,
      "loss_nats": 0.99610835313797,
      "step": 7290,
      "unclipped_grad_norm": 0.9820091724395752
    },
    {
      "bits_per_byte": 1.4451605841717328,
      "learning_rate": 3.5612289868135887e-05,
      "loss_nats": 1.001708984375,
      "step": 7300,
      "unclipped_grad_norm": 0.9996907114982605
    },
    {
      "bits_per_byte": 1.4502488628801598,
      "learning_rate": 3.545416500438522e-05,
      "loss_nats": 1.0052359104156494,
      "step": 7310,
      "unclipped_grad_norm": 0.9723894000053406
    },
    {
      "bits_per_byte": 1.4141343980081293,
      "learning_rate": 3.52982537215255e-05,
      "loss_nats": 0.9802032709121704,
      "step": 7320,
      "unclipped_grad_norm": 0.9815637469291687
    },
    {
      "bits_per_byte": 1.429772436485783,
      "learning_rate": 3.5144558683650396e-05,
      "loss_nats": 0.9910427331924438,
      "step": 7330,
      "unclipped_grad_norm": 1.0089058876037598
    },
    {
      "bits_per_byte": 1.4033304478809279,
      "learning_rate": 3.499308251698404e-05,
      "loss_nats": 0.9727145433425903,
      "step": 7340,
      "unclipped_grad_norm": 0.9830491542816162
    },
    {
      "bits_per_byte": 1.4800763299672492,
      "learning_rate": 3.484382780983616e-05,
      "loss_nats": 1.02591073513031,
      "step": 7350,
      "unclipped_grad_norm": 0.9842677712440491
    },
    {
      "bits_per_byte": 1.4409101189469837,
      "learning_rate": 3.469679711255795e-05,
      "loss_nats": 0.9987627863883972,
      "step": 7360,
      "unclipped_grad_norm": 1.0266367197036743
    },
    {
      "bits_per_byte": 1.4700960047550296,
      "learning_rate": 3.4551992937498284e-05,
      "loss_nats": 1.0189929008483887,
      "step": 7370,
      "unclipped_grad_norm": 0.9935671091079712
    },
    {
      "bits_per_byte": 1.4111584102175994,
      "learning_rate": 3.440941775896104e-05,
      "loss_nats": 0.9781404733657837,
      "step": 7380,
      "unclipped_grad_norm": 0.9791932702064514
    },
    {
      "bits_per_byte": 1.4805810990475317,
      "learning_rate": 3.4269074013162596e-05,
      "loss_nats": 1.0262606143951416,
      "step": 7390,
      "unclipped_grad_norm": 0.9806758761405945
    },
    {
      "bits_per_byte": 1.422500752033327,
      "learning_rate": 3.41309640981904e-05,
      "loss_nats": 0.9860023856163025,
      "step": 7400,
      "unclipped_grad_norm": 1.014562726020813
    },
    {
      "bits_per_byte": 1.419007526420327,
      "learning_rate": 3.3995090373961795e-05,
      "loss_nats": 0.9835810661315918,
      "step": 7410,
      "unclipped_grad_norm": 0.9609370231628418
    },
    {
      "bits_per_byte": 1.483305992167802,
      "learning_rate": 3.3861455162183876e-05,
      "loss_nats": 1.0281493663787842,
      "step": 7420,
      "unclipped_grad_norm": 0.9924190640449524
    },
    {
      "bits_per_byte": 1.4892452410326982,
      "learning_rate": 3.3730060746313664e-05,
      "loss_nats": 1.0322661399841309,
      "step": 7430,
      "unclipped_grad_norm": 1.0244429111480713
    },
    {
      "bits_per_byte": 1.50888548777858,
      "learning_rate": 3.3600909371519225e-05,
      "loss_nats": 1.0458797216415405,
      "step": 7440,
      "unclipped_grad_norm": 1.0394790172576904
    },
    {
      "bits_per_byte": 1.4553271665948369,
      "learning_rate": 3.3474003244641194e-05,
      "loss_nats": 1.0087559223175049,
      "step": 7450,
      "unclipped_grad_norm": 0.99248868227005
    },
    {
      "bits_per_byte": 1.4425173828106221,
      "learning_rate": 3.334934453415514e-05,
      "loss_nats": 0.999876856803894,
      "step": 7460,
      "unclipped_grad_norm": 1.0004469156265259
    },
    {
      "bits_per_byte": 1.437790267669007,
      "learning_rate": 3.3226935370134486e-05,
      "loss_nats": 0.9966002702713013,
      "step": 7470,
      "unclipped_grad_norm": 1.0109968185424805
    },
    {
      "bits_per_byte": 1.4682612938356252,
      "learning_rate": 3.31067778442141e-05,
      "loss_nats": 1.017721176147461,
      "step": 7480,
      "unclipped_grad_norm": 1.0299010276794434
    },
    {
      "bits_per_byte": 1.5222762289574092,
      "learning_rate": 3.298887400955457e-05,
      "loss_nats": 1.055161476135254,
      "step": 7490,
      "unclipped_grad_norm": 1.0327476263046265
    },
    {
      "bits_per_byte": 1.4747042798848986,
      "learning_rate": 3.287322588080714e-05,
      "loss_nats": 1.0221871137619019,
      "step": 7500,
      "unclipped_grad_norm": 0.9886637926101685
    },
    {
      "bits_per_byte": 1.4352613487793948,
      "learning_rate": 3.275983543407924e-05,
      "loss_nats": 0.9948473572731018,
      "step": 7510,
      "unclipped_grad_norm": 0.9710773825645447
    },
    {
      "bits_per_byte": 1.4570311706995855,
      "learning_rate": 3.2648704606900735e-05,
      "loss_nats": 1.009937047958374,
      "step": 7520,
      "unclipped_grad_norm": 1.0144232511520386
    },
    {
      "bits_per_byte": 1.502727820947088,
      "learning_rate": 3.253983529819086e-05,
      "loss_nats": 1.0416115522384644,
      "step": 7530,
      "unclipped_grad_norm": 1.0307852029800415
    },
    {
      "bits_per_byte": 1.4572943041554056,
      "learning_rate": 3.243322936822571e-05,
      "loss_nats": 1.0101194381713867,
      "step": 7540,
      "unclipped_grad_norm": 0.9732069969177246
    },
    {
      "bits_per_byte": 1.4747278415080667,
      "learning_rate": 3.232888863860648e-05,
      "loss_nats": 1.0222034454345703,
      "step": 7550,
      "unclipped_grad_norm": 1.0250486135482788
    },
    {
      "bits_per_byte": 1.4513507557242378,
      "learning_rate": 3.222681489222838e-05,
      "loss_nats": 1.0059996843338013,
      "step": 7560,
      "unclipped_grad_norm": 1.0031380653381348
    },
    {
      "bits_per_byte": 1.4127096077370564,
      "learning_rate": 3.212700987325009e-05,
      "loss_nats": 0.979215681552887,
      "step": 7570,
      "unclipped_grad_norm": 0.9720339775085449
    },
    {
      "bits_per_byte": 1.587541409185384,
      "learning_rate": 3.202947528706403e-05,
      "loss_nats": 1.1003998517990112,
      "step": 7580,
      "unclipped_grad_norm": 1.080464482307434
    },
    {
      "bits_per_byte": 1.4421111597892842,
      "learning_rate": 3.1934212800267116e-05,
      "loss_nats": 0.9995952844619751,
      "step": 7590,
      "unclipped_grad_norm": 0.9815735220909119
    },
    {
      "bits_per_byte": 1.4267507013101233,
      "learning_rate": 3.1841224040632484e-05,
      "loss_nats": 0.9889482259750366,
      "step": 7600,
      "unclipped_grad_norm": 1.029758334159851
    },
    {
      "bits_per_byte": 1.4539179407536669,
      "learning_rate": 3.175051059708143e-05,
      "loss_nats": 1.0077791213989258,
      "step": 7610,
      "unclipped_grad_norm": 1.031221628189087
    },
    {
      "bits_per_byte": 1.4028547438686412,
      "learning_rate": 3.1662074019656414e-05,
      "loss_nats": 0.9723848104476929,
      "step": 7620,
      "unclipped_grad_norm": 0.9562389850616455
    },
    {
      "bits_per_byte": 1.3938100042771153,
      "learning_rate": 3.157591581949455e-05,
      "loss_nats": 0.9661154747009277,
      "step": 7630,
      "unclipped_grad_norm": 1.023301124572754
    },
    {
      "bits_per_byte": 1.4184527103886437,
      "learning_rate": 3.149203746880175e-05,
      "loss_nats": 0.983196496963501,
      "step": 7640,
      "unclipped_grad_norm": 0.9981763958930969
    },
    {
      "bits_per_byte": 1.5608753271864544,
      "learning_rate": 3.141044040082763e-05,
      "loss_nats": 1.081916332244873,
      "step": 7650,
      "unclipped_grad_norm": 1.0295648574829102
    },
    {
      "bits_per_byte": 1.4665378556913289,
      "learning_rate": 3.133112600984091e-05,
      "loss_nats": 1.0165265798568726,
      "step": 7660,
      "unclipped_grad_norm": 1.0587892532348633
    },
    {
      "bits_per_byte": 1.4931012640476933,
      "learning_rate": 3.125409565110571e-05,
      "loss_nats": 1.034938931465149,
      "step": 7670,
      "unclipped_grad_norm": 1.0342050790786743
    },
    {
      "bits_per_byte": 1.4465245785757257,
      "learning_rate": 3.117935064085835e-05,
      "loss_nats": 1.0026544332504272,
      "step": 7680,
      "unclipped_grad_norm": 1.0567405223846436
    },
    {
      "bits_per_byte": 1.4541416901824393,
      "learning_rate": 3.110689225628479e-05,
      "loss_nats": 1.0079342126846313,
      "step": 7690,
      "unclipped_grad_norm": 1.0190099477767944
    },
    {
      "bits_per_byte": 1.3949941048283057,
      "learning_rate": 3.1036721735498926e-05,
      "loss_nats": 0.9669362306594849,
      "step": 7700,
      "unclipped_grad_norm": 1.002891182899475
    },
    {
      "bits_per_byte": 1.4762163513512843,
      "learning_rate": 3.096884027752132e-05,
      "loss_nats": 1.0232352018356323,
      "step": 7710,
      "unclipped_grad_norm": 0.9984713792800903
    },
    {
      "bits_per_byte": 1.480895139368007,
      "learning_rate": 3.090324904225885e-05,
      "loss_nats": 1.0264782905578613,
      "step": 7720,
      "unclipped_grad_norm": 1.0342158079147339
    },
    {
      "bits_per_byte": 1.486845911070511,
      "learning_rate": 3.083994915048472e-05,
      "loss_nats": 1.030603051185608,
      "step": 7730,
      "unclipped_grad_norm": 1.027107834815979
    },
    {
      "bits_per_byte": 1.4387832095037634,
      "learning_rate": 3.077894168381942e-05,
      "loss_nats": 0.9972885251045227,
      "step": 7740,
      "unclipped_grad_norm": 1.0455139875411987
    },
    {
      "bits_per_byte": 1.4381068877292456,
      "learning_rate": 3.0720227684712254e-05,
      "loss_nats": 0.9968197345733643,
      "step": 7750,
      "unclipped_grad_norm": 1.0159157514572144
    },
    {
      "bits_per_byte": 1.4538223183997872,
      "learning_rate": 3.0663808156423456e-05,
      "loss_nats": 1.0077128410339355,
      "step": 7760,
      "unclipped_grad_norm": 0.9867706894874573
    },
    {
      "bits_per_byte": 1.3986834766546148,
      "learning_rate": 3.060968406300711e-05,
      "loss_nats": 0.9694935083389282,
      "step": 7770,
      "unclipped_grad_norm": 0.9943742156028748
    },
    {
      "bits_per_byte": 1.4760859885019304,
      "learning_rate": 3.055785632929463e-05,
      "loss_nats": 1.0231448411941528,
      "step": 7780,
      "unclipped_grad_norm": 1.0422483682632446
    },
    {
      "bits_per_byte": 1.4162891686407895,
      "learning_rate": 3.0508325840878984e-05,
      "loss_nats": 0.9816968441009521,
      "step": 7790,
      "unclipped_grad_norm": 0.9937946796417236
    },
    {
      "bits_per_byte": 1.45477768202533,
      "learning_rate": 3.046109344409957e-05,
      "loss_nats": 1.0083750486373901,
      "step": 7800,
      "unclipped_grad_norm": 1.0428839921951294
    },
    {
      "bits_per_byte": 1.4292601001688625,
      "learning_rate": 3.0416159946027727e-05,
      "loss_nats": 0.9906876087188721,
      "step": 7810,
      "unclipped_grad_norm": 1.028254508972168
    },
    {
      "bits_per_byte": 1.4070698666586379,
      "learning_rate": 3.0373526114452946e-05,
      "loss_nats": 0.975306510925293,
      "step": 7820,
      "unclipped_grad_norm": 1.0253064632415771
    },
    {
      "bits_per_byte": 1.4530960356651936,
      "learning_rate": 3.0333192677869803e-05,
      "loss_nats": 1.0072094202041626,
      "step": 7830,
      "unclipped_grad_norm": 1.0885143280029297
    },
    {
      "bits_per_byte": 1.4330722676078875,
      "learning_rate": 3.029516032546544e-05,
      "loss_nats": 0.9933300018310547,
      "step": 7840,
      "unclipped_grad_norm": 1.0074925422668457
    },
    {
      "bits_per_byte": 1.3858685333908742,
      "learning_rate": 3.02594297071078e-05,
      "loss_nats": 0.9606108665466309,
      "step": 7850,
      "unclipped_grad_norm": 0.9880329966545105
    },
    {
      "bits_per_byte": 1.4517502714221922,
      "learning_rate": 3.0226001433334603e-05,
      "loss_nats": 1.0062766075134277,
      "step": 7860,
      "unclipped_grad_norm": 1.0082415342330933
    },
    {
      "bits_per_byte": 1.4675547891058807,
      "learning_rate": 3.019487607534278e-05,
      "loss_nats": 1.0172314643859863,
      "step": 7870,
      "unclipped_grad_norm": 1.023923397064209
    },
    {
      "bits_per_byte": 1.3867994754799946,
      "learning_rate": 3.016605416497886e-05,
      "loss_nats": 0.9612561464309692,
      "step": 7880,
      "unclipped_grad_norm": 1.0151392221450806
    },
    {
      "bits_per_byte": 1.4568015738606837,
      "learning_rate": 3.0139536194729734e-05,
      "loss_nats": 1.0097779035568237,
      "step": 7890,
      "unclipped_grad_norm": 1.0505098104476929
    },
    {
      "bits_per_byte": 1.446776189193938,
      "learning_rate": 3.0115322617714362e-05,
      "loss_nats": 1.00282883644104,
      "step": 7900,
      "unclipped_grad_norm": 1.0279992818832397
    },
    {
      "bits_per_byte": 1.461830518554563,
      "learning_rate": 3.0093413847675965e-05,
      "loss_nats": 1.0132637023925781,
      "step": 7910,
      "unclipped_grad_norm": 1.0125948190689087
    },
    {
      "bits_per_byte": 1.4593548282953928,
      "learning_rate": 3.0073810258974985e-05,
      "loss_nats": 1.0115476846694946,
      "step": 7920,
      "unclipped_grad_norm": 1.0411789417266846
    },
    {
      "bits_per_byte": 1.4317973602170415,
      "learning_rate": 3.0056512186582658e-05,
      "loss_nats": 0.9924463033676147,
      "step": 7930,
      "unclipped_grad_norm": 1.0330348014831543
    },
    {
      "bits_per_byte": 1.431196452834927,
      "learning_rate": 3.0041519926075316e-05,
      "loss_nats": 0.9920297861099243,
      "step": 7940,
      "unclipped_grad_norm": 1.0142537355422974
    },
    {
      "bits_per_byte": 1.49434246283897,
      "learning_rate": 3.0028833733629313e-05,
      "loss_nats": 1.035799264907837,
      "step": 7950,
      "unclipped_grad_norm": 1.0716009140014648
    },
    {
      "bits_per_byte": 1.4186909923514142,
      "learning_rate": 3.0018453826016686e-05,
      "loss_nats": 0.9833616614341736,
      "step": 7960,
      "unclipped_grad_norm": 1.0115852355957031
    },
    {
      "bits_per_byte": 1.4420433126335188,
      "learning_rate": 3.001038038060143e-05,
      "loss_nats": 0.9995482563972473,
      "step": 7970,
      "unclipped_grad_norm": 1.0335451364517212
    },
    {
      "bits_per_byte": 1.4872165336831205,
      "learning_rate": 3.0004613535336433e-05,
      "loss_nats": 1.0308599472045898,
      "step": 7980,
      "unclipped_grad_norm": 1.0748283863067627
    },
    {
      "bits_per_byte": 1.5234063269562288,
      "learning_rate": 3.000115338876117e-05,
      "loss_nats": 1.055944800376892,
      "step": 7990,
      "unclipped_grad_norm": 1.076164722442627
    },
    {
      "bits_per_byte": 1.4309937712808851,
      "learning_rate": 3e-05,
      "loss_nats": 0.9918892979621887,
      "step": 8000,
      "unclipped_grad_norm": 1.0322442054748535
    }
  ]
}