{
  "architecture": "block",
  "batch_size": 32,
  "canonical_sha256_without_self": "6db370af4c7fc867e59305adb946f528981bb02fea76beb10dfc1fa42360d383",
  "depth": 16,
  "diagnostics": [
    {
      "activation_grad_rms_by_block": [
        0.00026696090935729444,
        0.00023596938990522176,
        0.00020828393462579697,
        0.00018593396816868335,
        0.00016940363275352865,
        0.00015857137623243034,
        0.00015041495498735458,
        0.00014485203428193927,
        0.00013822126493323594,
        0.00013424422650132328,
        0.00013012201816309243,
        0.00012678767961915582,
        0.00012444572348613292,
        0.00012238229101058096,
        0.00012059078289894387,
        0.00011903457198059186
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00022428705051424913,
        "first_to_last_ratio": 1.84426351740015,
        "imbalance_abs_log_ratio": 0.6120800202390253,
        "last_quartile_mean": 0.0001216133423440624,
        "mean": 0.00015851367243158165,
        "normalized": [
          1.6841506808980231,
          1.488637454961949,
          1.3139808789408836,
          1.1729837894515815,
          1.0687004480742655,
          1.0003640304332333,
          0.9489083981211609,
          0.9138141338846396,
          0.8719832353445448,
          0.8468936744826624,
          0.8208882941580717,
          0.7998532724291052,
          0.785078798422557,
          0.772061419896786,
          0.7607595045215659,
          0.7509419859789702
        ],
        "population_cv": 0.27150931413441437
      },
      "activation_output_rms_by_block": [
        0.0048858714289963245,
        0.006843581795692444,
        0.004983869381248951,
        0.007334618363529444,
        0.005787155125290155,
        0.007929109036922455,
        0.005876626819372177,
        0.008892860263586044,
        0.006918553728610277,
        0.009880779311060905,
        0.007473294623196125,
        0.010543949902057648,
        0.008116732351481915,
        0.011967957951128483,
        0.008228716440498829,
        0.012017112225294113
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.006011985242366791,
        "first_to_last_ratio": 0.5962715478148781,
        "imbalance_abs_log_ratio": 0.5170590985397028,
        "last_quartile_mean": 0.010082629742100835,
        "mean": 0.007980049296747893,
        "normalized": [
          0.612260807835794,
          0.8575864059488212,
          0.6245411771177937,
          0.9191194303171183,
          0.725202929216063,
          0.9936165482278163,
          0.736414851693595,
          1.114386632575083,
          0.866981326973731,
          1.2381852471873345,
          0.9364973003665171,
          1.321288817896729,
          1.017128096538166,
          1.499734839483483,
          1.031161103710509,
          1.5058944849114457
        ],
        "population_cv": 0.2698221312597705
      },
      "bits_per_byte": 8.029409055280222,
      "branch_output_rms_by_sublayer": [
        0.0029694242402911186,
        0.0038970725145190954,
        0.003062367672100663,
        0.0038208949845284224,
        0.003184171626344323,
        0.0038178148679435253,
        0.003822639584541321,
        0.0037665588315576315,
        0.004263176582753658,
        0.0038892142474651337,
        0.004122398793697357,
        0.003805827582255006,
        0.004464023746550083,
        0.0037649995647370815,
        0.005492920521646738,
        0.0037115272134542465,
        0.005817505065351725,
        0.003727257251739502,
        0.006086217239499092,
        0.00368275074288249,
        0.0065266890451312065,
        0.0036771725863218307,
        0.0064827753230929375,
        0.003755199955776334,
        0.007209060713648796,
        0.0037714343052357435,
        0.007561341859400272,
        0.0037067290395498276,
        0.007512317039072514,
        0.0035454558674246073,
        0.007687372621148825,
        0.0036056456156075
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.006887647168374071,
        0.005982369825085796,
        0.005174924477158116,
        0.004946754810206586,
        0.004967935165757302,
        0.004123664448940147,
        0.00409438714185196,
        0.004247982317789705,
        0.0044010147653324285,
        0.004095379628913903,
        0.00386485833459634,
        0.0038097011691932423,
        0.0037333099925226194,
        0.004058250822051008,
        0.003851730608686184,
        0.00381805442027398
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.005747924070206143,
        "first_to_last_ratio": 1.487043658003427,
        "imbalance_abs_log_ratio": 0.39679002683413417,
        "last_quartile_mean": 0.003865336460883448,
        "mean": 0.004503622818545837,
        "normalized": [
          1.5293570189783356,
          1.3283461040410636,
          1.1490581440008412,
          1.0983945613375836,
          1.10309752080024,
          0.9156327283801325,
          0.909131893770355,
          0.9432366983079914,
          0.9772165526849029,
          0.9093522690331003,
          0.858166522889289,
          0.8459192349556811,
          0.8289570736583262,
          0.9011080602352411,
          0.8552515971863425,
          0.8477740197405745
        ],
        "population_cv": 0.191198476656264
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.02822229079902172,
        0.014168336987495422,
        0.01429277565330267,
        0.014353902079164982,
        0.014476437121629715,
        0.009725152514874935,
        0.00981744285672903,
        0.009922475554049015,
        0.010004729963839054,
        0.0075368694961071014,
        0.007599710486829281,
        0.007675632368773222,
        0.007730280980467796,
        0.006251458544284105,
        0.0062932223081588745,
        0.006393968593329191,
        0.006436701864004135,
        0.005461040884256363,
        0.005505258683115244,
        0.005599391181021929,
        0.005641042720526457,
        0.004922671243548393,
        0.004954405128955841,
        0.005046301055699587,
        0.005088083446025848,
        0.00459245452657342,
        0.0046256110072135925,
        0.0047518787905573845,
        0.004772109445184469,
        0.004340849816799164,
        0.004358765669167042,
        0.004422077443450689
      ],
      "loss_nats": 5.5655622482299805,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 2.1972246170043945,
        "mean_weights": [
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519
        ],
        "sources": 9
      },
      "step": 0,
      "stream_state_rms_by_sublayer": [
        0.0029694242402911186,
        0.0048858714289963245,
        0.005668045952916145,
        0.006843581795692444,
        0.003184171626344323,
        0.004983869381248951,
        0.006258165929466486,
        0.007334618363529444,
        0.004263176582753658,
        0.005787155125290155,
        0.006935684010386467,
        0.007929109036922455,
        0.004464023746550083,
        0.005876626819372177,
        0.008052977733314037,
        0.008892860263586044,
        0.005817505065351725,
        0.006918553728610277,
        0.009139667265117168,
        0.009880779311060905,
        0.0065266890451312065,
        0.007473294623196125,
        0.00979399960488081,
        0.010543949902057648,
        0.007209060713648796,
        0.008116732351481915,
        0.0113961361348629,
        0.011967957951128483,
        0.007512317039072514,
        0.008228716440498829,
        0.011409307830035686,
        0.012017112225294113
      ]
    },
    {
      "activation_grad_rms_by_block": [
        8.836213237373158e-05,
        8.102934953058138e-05,
        6.889242649776861e-05,
        6.090942770242691e-05,
        5.060216426500119e-05,
        4.3459844164317474e-05,
        3.709086377057247e-05,
        3.2571624615229666e-05,
        2.8896074582007714e-05,
        2.6415440515847877e-05,
        2.491714622010477e-05,
        2.3614533347426914e-05,
        2.2646127035841346e-05,
        2.2026059014024213e-05,
        2.118862721545156e-05,
        2.081490310956724e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 7.479833402612712e-05,
        "first_to_last_ratio": 3.4518703579034327,
        "imbalance_abs_log_ratio": 1.2389162168680456,
        "last_quartile_mean": 2.166892909372109e-05,
        "mean": 4.083979649749381e-05,
        "normalized": [
          2.1636281262849595,
          1.984078190388482,
          1.6868944609456193,
          1.4914233890995197,
          1.2390405585910906,
          1.0641542782169457,
          0.9082039322318514,
          0.7975462026905171,
          0.7075469777079071,
          0.6468063697983631,
          0.6101192551641105,
          0.578223580249134,
          0.5545112605355675,
          0.5393283244047475,
          0.518823036171393,
          0.5096720575197914
        ],
        "population_cv": 0.5343944992100478
      },
      "activation_output_rms_by_block": [
        0.009077239781618118,
        0.016407418996095657,
        0.010880433954298496,
        0.01978805474936962,
        0.012077964842319489,
        0.02216547541320324,
        0.013358108699321747,
        0.02644582837820053,
        0.017588114365935326,
        0.03464006260037422,
        0.023338599130511284,
        0.0432722382247448,
        0.023765092715620995,
        0.04745635390281677,
        0.02874108962714672,
        0.05685042589902878
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.014038286870345473,
        "first_to_last_ratio": 0.3580899608898232,
        "imbalance_abs_log_ratio": 1.0269710367470868,
        "last_quartile_mean": 0.039203240536153316,
        "mean": 0.025365781330037862,
        "normalized": [
          0.35785374254838964,
          0.6468327855789793,
          0.4289414078254319,
          0.7801082289523973,
          0.4761518947581916,
          0.8738337339112493,
          0.5266192484086122,
          1.042578899270242,
          0.6933795626909267,
          1.3656217464649458,
          0.9200820123318648,
          1.7059296404760187,
          0.9368957496877358,
          1.8708808250514843,
          1.133065418060348,
          2.2412251039831825
        ],
        "population_cv": 0.5292067698975893
      },
      "bits_per_byte": 5.216988460540972,
      "branch_output_rms_by_sublayer": [
        0.004302801564335823,
        0.007946929894387722,
        0.004027083050459623,
        0.008643880486488342,
        0.004623814020305872,
        0.0097366226837039,
        0.005764479748904705,
        0.009610678069293499,
        0.0056845624931156635,
        0.010435172356665134,
        0.006739616859704256,
        0.010610987432301044,
        0.006770467385649681,
        0.010895426385104656,
        0.009306530468165874,
        0.011636263690888882,
        0.010160733014345169,
        0.012227990664541721,
        0.012430583126842976,
        0.012838989496231079,
        0.015689926221966743,
        0.013102410361170769,
        0.015432302840054035,
        0.013170000165700912,
        0.015551909804344177,
        0.012761367484927177,
        0.018755123019218445,
        0.013125414028763771,
        0.019000543281435966,
        0.01451444998383522,
        0.022471774369478226,
        0.013329307548701763
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00035471167919676877,
        0.00030778414793465906,
        0.0002905596100869004,
        0.0002669019974562734,
        0.0002519467909729747,
        0.0002484952213280207,
        0.00022573236044592232,
        0.00021694664140763627,
        0.00020872399951088957,
        0.00018641204799172754,
        0.0001823569017298164,
        0.0001572320679108608,
        0.00014731276753985554,
        0.00014544881518036755,
        0.00014011786536140868,
        0.0001387938590477502
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.0003049893586686504,
        "first_to_last_ratio": 2.1340115402633257,
        "imbalance_abs_log_ratio": 0.7580035606734332,
        "last_quartile_mean": 0.0001429183267823455,
        "mean": 0.00021684229831886448,
        "normalized": [
          1.6358048311919693,
          1.4193916515405378,
          1.339958173933553,
          1.230857630294052,
          1.1618895064582344,
          1.1459720878009276,
          1.0409978228232257,
          1.0004811934275772,
          0.9625612766931799,
          0.8596664462466195,
          0.840965545668873,
          0.7250986967480513,
          0.6793543911033149,
          0.6707585019528177,
          0.6461740465200508,
          0.6400681975970168
        ],
        "population_cv": 0.294333589312772
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.6924017667770386,
          "mean_weights": [
            0.4816230535507202,
            0.5183769464492798
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6925060749053955,
          "mean_weights": [
            0.5170555114746094,
            0.4829444885253906
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6922789812088013,
          "mean_weights": [
            0.5104873776435852,
            0.4895126223564148
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6927056312561035,
          "mean_weights": [
            0.5133966207504272,
            0.48660340905189514
          ],
          "sources": 2
        },
        {
          "entropy_mean": 1.097900390625,
          "mean_weights": [
            0.3491937518119812,
            0.32761698961257935,
            0.32318928837776184
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0982853174209595,
          "mean_weights": [
            0.3437661826610565,
            0.3295633792877197,
            0.32667046785354614
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.097551703453064,
          "mean_weights": [
            0.3531244695186615,
            0.3277324438095093,
            0.3191430866718292
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.098252296447754,
          "mean_weights": [
            0.3423447012901306,
            0.3336608111858368,
            0.323994517326355
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.3856292963027954,
          "mean_weights": [
            0.2598890960216522,
            0.25528842210769653,
            0.2465861737728119,
            0.23823627829551697
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862417936325073,
          "mean_weights": [
            0.2525826096534729,
            0.2512264847755432,
            0.24808365106582642,
            0.24810725450515747
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3854787349700928,
          "mean_weights": [
            0.2609134912490845,
            0.2559056282043457,
            0.24643474817276,
            0.23674613237380981
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.386183738708496,
          "mean_weights": [
            0.25091099739074707,
            0.2541857361793518,
            0.24937650561332703,
            0.24552680552005768
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.6086668968200684,
          "mean_weights": [
            0.20840778946876526,
            0.2072581648826599,
            0.20017966628074646,
            0.19401825964450836,
            0.1901361048221588
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6093640327453613,
          "mean_weights": [
            0.20082800090312958,
            0.20345230400562286,
            0.19957157969474792,
            0.1978151500225067,
            0.19833296537399292
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6085686683654785,
          "mean_weights": [
            0.2064795196056366,
            0.20873931050300598,
            0.20255593955516815,
            0.1941128820180893,
            0.18811233341693878
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6093556880950928,
          "mean_weights": [
            0.2007957398891449,
            0.20384852588176727,
            0.2003384232521057,
            0.19736894965171814,
            0.19764836132526398
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.7904106378555298,
          "mean_weights": [
            0.17261356115341187,
            0.1785532385110855,
            0.16990220546722412,
            0.16298523545265198,
            0.1574374884366989,
            0.1585082709789276
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791741132736206,
          "mean_weights": [
            0.16599833965301514,
            0.16723501682281494,
            0.16713106632232666,
            0.1660757064819336,
            0.16713276505470276,
            0.1664271056652069
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7893474102020264,
          "mean_weights": [
            0.17279052734375,
            0.18296070396900177,
            0.1714734584093094,
            0.16615013778209686,
            0.1573297083377838,
            0.14929547905921936
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7916802167892456,
          "mean_weights": [
            0.1656593233346939,
            0.16598697006702423,
            0.1664597988128662,
            0.16510574519634247,
            0.16595518589019775,
            0.17083296179771423
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.9424470663070679,
          "mean_weights": [
            0.14923666417598724,
            0.16057875752449036,
            0.15159368515014648,
            0.14453238248825073,
            0.13740569353103638,
            0.1286274939775467,
            0.12802532315254211
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9457793235778809,
          "mean_weights": [
            0.1414237767457962,
            0.14120322465896606,
            0.14217954874038696,
            0.14177344739437103,
            0.14084486663341522,
            0.145671084523201,
            0.14690405130386353
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.940324306488037,
          "mean_weights": [
            0.15125462412834167,
            0.16445259749889374,
            0.15269356966018677,
            0.1458570957183838,
            0.1393839716911316,
            0.12596309185028076,
            0.12039506435394287
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9458446502685547,
          "mean_weights": [
            0.1414341926574707,
            0.14160503447055817,
            0.14392146468162537,
            0.1416645050048828,
            0.14250624179840088,
            0.1432597041130066,
            0.1456088423728943
          ],
          "sources": 7
        },
        {
          "entropy_mean": 2.073972225189209,
          "mean_weights": [
            0.13393926620483398,
            0.1460031270980835,
            0.13552871346473694,
            0.12954440712928772,
            0.12087934464216232,
            0.11472145467996597,
            0.10819212347269058,
            0.11119155585765839
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0792579650878906,
          "mean_weights": [
            0.12344250082969666,
            0.1223982647061348,
            0.12281741201877594,
            0.12404577434062958,
            0.1245848536491394,
            0.1274431347846985,
            0.1261020004749298,
            0.12916606664657593
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.070988178253174,
          "mean_weights": [
            0.13450230658054352,
            0.1483522206544876,
            0.13837148249149323,
            0.13302284479141235,
            0.12319637835025787,
            0.11490452289581299,
            0.10592599213123322,
            0.10172426700592041
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0791521072387695,
          "mean_weights": [
            0.12388300150632858,
            0.12212474644184113,
            0.12333755195140839,
            0.12223256379365921,
            0.12343418598175049,
            0.12649230659008026,
            0.12949144840240479,
            0.12900421023368835
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.18794584274292,
          "mean_weights": [
            0.12259161472320557,
            0.13473162055015564,
            0.1250339150428772,
            0.11940491199493408,
            0.11259052157402039,
            0.10314525663852692,
            0.09605535864830017,
            0.09219000488519669,
            0.09425680339336395
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.197138786315918,
          "mean_weights": [
            0.11026664823293686,
            0.11047966778278351,
            0.11075460910797119,
            0.10965213924646378,
            0.10944131761789322,
            0.111859992146492,
            0.11317223310470581,
            0.1125045120716095,
            0.11186887323856354
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.189896583557129,
          "mean_weights": [
            0.12157853692770004,
            0.13215823471546173,
            0.1242145225405693,
            0.11826580762863159,
            0.11163709312677383,
            0.1045699343085289,
            0.09787097573280334,
            0.09515326470136642,
            0.09455163776874542
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.028616169467568398,
        0.013977974653244019,
        0.015653911978006363,
        0.015701081603765488,
        0.01704590953886509,
        0.011780167929828167,
        0.012960037216544151,
        0.013451050966978073,
        0.014845823869109154,
        0.011557077057659626,
        0.012834185734391212,
        0.013193685561418533,
        0.014761172235012054,
        0.012218841351568699,
        0.013451457023620605,
        0.01399444043636322,
        0.015514986589550972,
        0.013508559204638004,
        0.014893320389091969,
        0.015537911094725132,
        0.017302624881267548,
        0.015652775764465332,
        0.017217284068465233,
        0.017610477283596992,
        0.019684365019202232,
        0.017625300213694572,
        0.019488012418150902,
        0.019592516124248505,
        0.022200310602784157,
        0.019619757309556007,
        0.022100279107689857,
        0.021992051973938942
      ],
      "loss_nats": 3.616140842437744,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 2.1926791667938232,
        "mean_weights": [
          0.09771968424320221,
          0.09518586099147797,
          0.10263510048389435,
          0.10697779804468155,
          0.11385782063007355,
          0.11843808740377426,
          0.1225852519273758,
          0.12233239412307739,
          0.12026800215244293
        ],
        "sources": 9
      },
      "step": 100,
      "stream_state_rms_by_sublayer": [
        0.004302801564335823,
        0.009077239781618118,
        0.010217261500656605,
        0.016407418996095657,
        0.004623814020305872,
        0.010880433954298496,
        0.012616241350769997,
        0.01978805474936962,
        0.0056845624931156635,
        0.012077964842319489,
        0.014198417775332928,
        0.02216547541320324,
        0.006770467385649681,
        0.013358108699321747,
        0.017934804782271385,
        0.02644582837820053,
        0.010160733014345169,
        0.017588114365935326,
        0.02522732876241207,
        0.03464006260037422,
        0.015689926221966743,
        0.023338599130511284,
        0.03383738547563553,
        0.0432722382247448,
        0.015551909804344177,
        0.023765092715620995,
        0.03784027695655823,
        0.04745635390281677,
        0.019000543281435966,
        0.02874108962714672,
        0.04672664403915405,
        0.05685042589902878
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.00015875036478973925,
        0.00016262078133877367,
        0.00014673685654997826,
        0.00013248690811451524,
        0.00010898245091084391,
        9.717086504679173e-05,
        8.911665645428002e-05,
        7.969579746713862e-05,
        7.858985918574035e-05,
        6.784927973058075e-05,
        6.598868640139699e-05,
        5.512806092156097e-05,
        4.8474703362444416e-05,
        4.3001487938454375e-05,
        3.610895510064438e-05,
        3.5184959415346384e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.0001501487276982516,
        "first_to_last_ratio": 3.689835475493601,
        "imbalance_abs_log_ratio": 1.3055818704713245,
        "last_quartile_mean": 4.069252645422239e-05,
        "mean": 8.786791704551433e-05,
        "normalized": [
          1.806693160912291,
          1.8507412808538344,
          1.6699708094135275,
          1.5077961623454543,
          1.2402985591930278,
          1.1058742293442396,
          1.0142115512777987,
          0.9069954102344013,
          0.8944090383413996,
          0.7721735305895072,
          0.7509986422827776,
          0.6273969245567232,
          0.5516769372982315,
          0.489387815079083,
          0.4109458413807676,
          0.4004301068969357
        ],
        "population_cv": 0.4737558240032668
      },
      "activation_output_rms_by_block": [
        0.02468283288180828,
        0.04329179599881172,
        0.023283371701836586,
        0.03508218377828598,
        0.025414658710360527,
        0.03976430371403694,
        0.02767801284790039,
        0.03781749680638313,
        0.03550800308585167,
        0.04360951855778694,
        0.040389809757471085,
        0.05188606306910515,
        0.03616104647517204,
        0.06982707977294922,
        0.04418032616376877,
        0.09354322403669357
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.03158504609018564,
        "first_to_last_ratio": 0.5184002104527677,
        "imbalance_abs_log_ratio": 0.6570077279781673,
        "last_quartile_mean": 0.0609279191121459,
        "mean": 0.042007482959888875,
        "normalized": [
          0.5875818102545408,
          1.0305734347413573,
          0.5542672414833535,
          0.8351412964157943,
          0.6050031308618963,
          0.9466004842995747,
          0.6588829155588524,
          0.9002561958423794,
          0.8452780453367492,
          1.038136910022145,
          0.9614908323842579,
          1.2351623904400295,
          0.8608239277202269,
          1.662253361850413,
          1.0517251463496313,
          2.2268228764387987
        ],
        "population_cv": 0.4130647858768022
      },
      "bits_per_byte": 3.137961051112634,
      "branch_output_rms_by_sublayer": [
        0.013650949113070965,
        0.01843169331550598,
        0.019044609740376472,
        0.014011893421411514,
        0.018698271363973618,
        0.012105143629014492,
        0.021805183961987495,
        0.010238726623356342,
        0.020464032888412476,
        0.011298555880784988,
        0.021483227610588074,
        0.011180058121681213,
        0.022897671908140182,
        0.01260110642760992,
        0.023534495383501053,
        0.012234007939696312,
        0.028167055919766426,
        0.01504397764801979,
        0.03861295431852341,
        0.015801267698407173,
        0.035523753613233566,
        0.0176701582968235,
        0.020830485969781876,
        0.024693600833415985,
        0.022914253175258636,
        0.028566768392920494,
        0.02199499122798443,
        0.03558572381734848,
        0.013380669057369232,
        0.04152822494506836,
        0.013844908215105534,
        0.04695267602801323
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0008395536207241178,
        0.0006692968987216388,
        0.0005333404576068953,
        0.0004419910350210091,
        0.0004333787114417237,
        0.0004609433225170186,
        0.0004705681902150354,
        0.0005107414594492524,
        0.0005339389183678795,
        0.0004946080122888242,
        0.000448918054550765,
        0.00035668617352665593,
        0.00032572588316331664,
        0.000267052972482006,
        0.0002153120435976381,
        0.0002101978579413297
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.0006210455030184152,
        "first_to_last_ratio": 2.439565392966498,
        "imbalance_abs_log_ratio": 0.8918199058035992,
        "last_quartile_mean": 0.00025457218929607263,
        "mean": 0.0004507658507259441,
        "normalized": [
          1.8625049332642285,
          1.484799475478804,
          1.183187361571351,
          0.980533539329114,
          0.9614275587730993,
          1.022578178392804,
          1.04393043407614,
          1.1330526893878927,
          1.1845150148530281,
          1.0972614972768537,
          0.9959007627303548,
          0.7912892535053935,
          0.7226055004804499,
          0.5924427772244185,
          0.47765828589473863,
          0.46631273776132937
        ],
        "population_cv": 0.34661834917809303
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.6546831130981445,
          "mean_weights": [
            0.3866303563117981,
            0.6133696436882019
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6832369565963745,
          "mean_weights": [
            0.48925086855888367,
            0.510749101638794
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6867785453796387,
          "mean_weights": [
            0.5347544550895691,
            0.4652456045150757
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6725893020629883,
          "mean_weights": [
            0.4844614565372467,
            0.5155385732650757
          ],
          "sources": 2
        },
        {
          "entropy_mean": 1.0954453945159912,
          "mean_weights": [
            0.332688570022583,
            0.3308390974998474,
            0.3364723324775696
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0907697677612305,
          "mean_weights": [
            0.3209903836250305,
            0.33586689829826355,
            0.34314271807670593
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0933208465576172,
          "mean_weights": [
            0.33219069242477417,
            0.312181293964386,
            0.35562801361083984
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0861705541610718,
          "mean_weights": [
            0.319164514541626,
            0.32808834314346313,
            0.3527471721172333
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.3827753067016602,
          "mean_weights": [
            0.24703660607337952,
            0.23925676941871643,
            0.2577625513076782,
            0.25594407320022583
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3791232109069824,
          "mean_weights": [
            0.24712958931922913,
            0.2454403042793274,
            0.24896711111068726,
            0.25846296548843384
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3816200494766235,
          "mean_weights": [
            0.2438572347164154,
            0.2418915331363678,
            0.2618292272090912,
            0.252422034740448
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3765439987182617,
          "mean_weights": [
            0.24315571784973145,
            0.25071609020233154,
            0.23800383508205414,
            0.26812440156936646
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.6069204807281494,
          "mean_weights": [
            0.1977769434452057,
            0.19419988989830017,
            0.19346478581428528,
            0.20617546141147614,
            0.20838290452957153
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6016991138458252,
          "mean_weights": [
            0.19928976893424988,
            0.19587931036949158,
            0.19414740800857544,
            0.18995173275470734,
            0.22073179483413696
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6063685417175293,
          "mean_weights": [
            0.19720253348350525,
            0.19060829281806946,
            0.18910765647888184,
            0.20065464079380035,
            0.2224268764257431
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6036955118179321,
          "mean_weights": [
            0.19818633794784546,
            0.20014071464538574,
            0.19487890601158142,
            0.18811310827732086,
            0.2186809629201889
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.7846078872680664,
          "mean_weights": [
            0.15616348385810852,
            0.1693512499332428,
            0.1684132218360901,
            0.14385344088077545,
            0.16338428854942322,
            0.19883427023887634
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7881443500518799,
          "mean_weights": [
            0.16932255029678345,
            0.1670200079679489,
            0.16782812774181366,
            0.15219610929489136,
            0.1653819978237152,
            0.1782512068748474
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7859947681427002,
          "mean_weights": [
            0.15620459616184235,
            0.18192027509212494,
            0.16651901602745056,
            0.14659695327281952,
            0.16306468844413757,
            0.18569447100162506
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7886135578155518,
          "mean_weights": [
            0.17137612402439117,
            0.16813044250011444,
            0.16593416035175323,
            0.15192663669586182,
            0.16282179951667786,
            0.17981085181236267
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.937737226486206,
          "mean_weights": [
            0.12751945853233337,
            0.15645764768123627,
            0.14123380184173584,
            0.1275305449962616,
            0.13446006178855896,
            0.15007954835891724,
            0.16271892189979553
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.943496823310852,
          "mean_weights": [
            0.14288359880447388,
            0.1535882204771042,
            0.14250940084457397,
            0.1380811333656311,
            0.1316976547241211,
            0.13936954736709595,
            0.15187042951583862
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9273982048034668,
          "mean_weights": [
            0.12311284244060516,
            0.16783711314201355,
            0.141703262925148,
            0.11705678701400757,
            0.12322613596916199,
            0.14240297675132751,
            0.184660866856575
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.937564730644226,
          "mean_weights": [
            0.13963916897773743,
            0.16193091869354248,
            0.14834126830101013,
            0.12877905368804932,
            0.1304946392774582,
            0.13670659065246582,
            0.15410837531089783
          ],
          "sources": 7
        },
        {
          "entropy_mean": 2.0582637786865234,
          "mean_weights": [
            0.10422320663928986,
            0.14683584868907928,
            0.12450192868709564,
            0.10290674865245819,
            0.1043052226305008,
            0.1136581152677536,
            0.13712583482265472,
            0.1664431095123291
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.073244333267212,
          "mean_weights": [
            0.11922207474708557,
            0.13852310180664062,
            0.1310172826051712,
            0.1193084791302681,
            0.1166592389345169,
            0.11741948127746582,
            0.12622521817684174,
            0.13162514567375183
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.05503249168396,
          "mean_weights": [
            0.10400840640068054,
            0.1620255410671234,
            0.13802370429039001,
            0.10328710824251175,
            0.10247690975666046,
            0.11085900664329529,
            0.13948379456996918,
            0.13983553647994995
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0654377937316895,
          "mean_weights": [
            0.12162524461746216,
            0.14755912125110626,
            0.1359378546476364,
            0.11907999217510223,
            0.10864339768886566,
            0.11126887798309326,
            0.12606029212474823,
            0.12982521951198578
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.158114194869995,
          "mean_weights": [
            0.08706305176019669,
            0.1523345708847046,
            0.1240597814321518,
            0.09008736908435822,
            0.08798909187316895,
            0.09299354255199432,
            0.11455917358398438,
            0.09811963140964508,
            0.15279380977153778
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1800241470336914,
          "mean_weights": [
            0.10917439311742783,
            0.13041174411773682,
            0.1256810426712036,
            0.10147396475076675,
            0.10038554668426514,
            0.10393467545509338,
            0.11099927127361298,
            0.11076852679252625,
            0.10717083513736725
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1717891693115234,
          "mean_weights": [
            0.09730195999145508,
            0.15644855797290802,
            0.1323733627796173,
            0.10013202577829361,
            0.09525948762893677,
            0.09674295783042908,
            0.12173393368721008,
            0.10994285345077515,
            0.09006483852863312
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.04337252303957939,
        0.018801197409629822,
        0.025790294632315636,
        0.029389970004558563,
        0.031753405928611755,
        0.022941799834370613,
        0.02458134852349758,
        0.02529502473771572,
        0.026913003996014595,
        0.020798955112695694,
        0.022398613393306732,
        0.022611256688833237,
        0.02451939694583416,
        0.01929222047328949,
        0.020593678578734398,
        0.020593887194991112,
        0.021762654185295105,
        0.01771371066570282,
        0.019139636307954788,
        0.0189701896160841,
        0.02020873688161373,
        0.016272971406579018,
        0.017277248203754425,
        0.01622883230447769,
        0.018368225544691086,
        0.014602466486394405,
        0.017397841438651085,
        0.017420755699276924,
        0.02048002928495407,
        0.016635844483971596,
        0.021542873233556747,
        0.021214226260781288
      ],
      "loss_nats": 2.1750688552856445,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 2.173111915588379,
        "mean_weights": [
          0.1217261254787445,
          0.09291160106658936,
          0.09640607982873917,
          0.10597392916679382,
          0.09402626007795334,
          0.09312883019447327,
          0.11353092640638351,
          0.147413432598114,
          0.13488279283046722
        ],
        "sources": 9
      },
      "step": 500,
      "stream_state_rms_by_sublayer": [
        0.013650949113070965,
        0.02468283288180828,
        0.0356261320412159,
        0.04329179599881172,
        0.018698271363973618,
        0.023283371701836586,
        0.03009101189672947,
        0.03508218377828598,
        0.020464032888412476,
        0.025414658710360527,
        0.034230802208185196,
        0.03976430371403694,
        0.022897671908140182,
        0.02767801284790039,
        0.031892068684101105,
        0.03781749680638313,
        0.028167055919766426,
        0.03550800308585167,
        0.035372670739889145,
        0.04360951855778694,
        0.035523753613233566,
        0.040389809757471085,
        0.03893519565463066,
        0.05188606306910515,
        0.022914253175258636,
        0.03616104647517204,
        0.04390736669301987,
        0.06982707977294922,
        0.013380669057369232,
        0.04418032616376877,
        0.051131755113601685,
        0.09354322403669357
      ]
    },
    {
      "activation_grad_rms_by_block": [
        7.532809104304761e-05,
        7.798708247719333e-05,
        7.679135887883604e-05,
        7.304617611225694e-05,
        7.868888496886939e-05,
        7.320931035792455e-05,
        8.902617992134765e-05,
        7.941434887470677e-05,
        8.246234938269481e-05,
        7.884583465056494e-05,
        0.0001634695363463834,
        0.00012072100798832253,
        0.000141287746373564,
        8.843849354889244e-05,
        5.323962977854535e-05,
        4.3788353650597855e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 7.578817712783348e-05,
        "first_to_last_ratio": 0.9277698246768501,
        "imbalance_abs_log_ratio": 0.07497161071478954,
        "last_quartile_mean": 8.168855583789991e-05,
        "mean": 8.723402402210922e-05,
        "normalized": [
          0.8635173246617158,
          0.8939984524550617,
          0.8802913741474712,
          0.8373587820933674,
          0.9020435071174282,
          0.8392288579897432,
          1.0205442305262016,
          0.9103598024387758,
          0.9453003034893238,
          0.9038426867776004,
          1.8739194732659872,
          1.383875263598136,
          1.6196403634636298,
          1.0138073365327953,
          0.6103080807673381,
          0.5019641606754243
        ],
        "population_cv": 0.3374467083690701
      },
      "activation_output_rms_by_block": [
        0.09266375005245209,
        0.20113667845726013,
        0.14358533918857574,
        0.17041823267936707,
        0.05199930816888809,
        0.07249397784471512,
        0.03721517324447632,
        0.07351206243038177,
        0.05156216770410538,
        0.08937478810548782,
        0.062482453882694244,
        0.09533937275409698,
        0.05472978577017784,
        0.08922836184501648,
        0.08201298862695694,
        0.20069743692874908
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.15195100009441376,
        "first_to_last_ratio": 1.4245342605399758,
        "imbalance_abs_log_ratio": 0.353844925589721,
        "last_quartile_mean": 0.10666714329272509,
        "mean": 0.09802824235521257,
        "normalized": [
          0.9452760533712127,
          2.0518237767482,
          1.4647344044819621,
          1.7384605557023458,
          0.5304523157771692,
          0.739521347144304,
          0.37963725912400215,
          0.7499069723601224,
          0.5259929839123919,
          0.9117248861979151,
          0.6373923716420872,
          0.9725704599356962,
          0.5583063049509795,
          0.9102311711526044,
          0.8366261258645934,
          2.0473430116344136
        ],
        "population_cv": 0.5182803671925184
      },
      "bits_per_byte": 2.0687232408845437,
      "branch_output_rms_by_sublayer": [
        0.04482239484786987,
        0.07626598328351974,
        0.10980217158794403,
        0.05728650838136673,
        0.14111557602882385,
        0.014705370180308819,
        0.04088159278035164,
        0.014925715513527393,
        0.04827577620744705,
        0.015019712969660759,
        0.04055904969573021,
        0.02146836742758751,
        0.028491884469985962,
        0.0222648773342371,
        0.0389801524579525,
        0.027360456064343452,
        0.036527831107378006,
        0.03173159062862396,
        0.03612164035439491,
        0.0401885062456131,
        0.04509544372558594,
        0.040587812662124634,
        0.02791302651166916,
        0.05573032796382904,
        0.02739257924258709,
        0.044091735035181046,
        0.017023129388689995,
        0.05625532940030098,
        0.02216578833758831,
        0.07780560851097107,
        0.0160520039498806,
        0.13296744227409363
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0003765937729119843,
        0.0003149609092301355,
        0.00027693435492303714,
        0.0001635333627570268,
        0.000180917699642147,
        0.00017837059454152616,
        0.000200147742296221,
        0.00024611271555124715,
        0.0002957900398296376,
        0.0002986151161329657,
        0.0009593337211615513,
        0.0007552262449127961,
        0.0007605792150238727,
        0.0005098371211048817,
        0.00029447700210242914,
        0.0002817503499702034
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00028300559995554593,
        "first_to_last_ratio": 0.6130161476493403,
        "imbalance_abs_log_ratio": 0.48936400138742103,
        "last_quartile_mean": 0.00046166092205034674,
        "mean": 0.00038082374763072894,
        "normalized": [
          0.9888925657996352,
          0.827051651031862,
          0.727198229222749,
          0.4294200762805349,
          0.47506937465878935,
          0.46838096534485474,
          0.5255652872002538,
          0.6462640974530133,
          0.7767111207477916,
          0.7841294509357175,
          2.5191016241239836,
          1.9831385243472572,
          1.997194817171378,
          1.3387744968027897,
          0.7732632324914067,
          0.7398444863879827
        ],
        "population_cv": 0.6091669692594756
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.573702335357666,
          "mean_weights": [
            0.7102901935577393,
            0.28970980644226074
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6441590189933777,
          "mean_weights": [
            0.564466118812561,
            0.43553388118743896
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.3999374806880951,
          "mean_weights": [
            0.8575316667556763,
            0.14246836304664612
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.676960825920105,
          "mean_weights": [
            0.46661365032196045,
            0.5333863496780396
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.9920796155929565,
          "mean_weights": [
            0.5238763093948364,
            0.2048875242471695,
            0.2712361812591553
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.092700719833374,
          "mean_weights": [
            0.2899855673313141,
            0.34371256828308105,
            0.36630189418792725
          ],
          "sources": 3
        },
        {
          "entropy_mean": 0.9979273080825806,
          "mean_weights": [
            0.49421679973602295,
            0.18891796469688416,
            0.3168652057647705
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.091429591178894,
          "mean_weights": [
            0.3115330636501312,
            0.3673863708972931,
            0.3210805654525757
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.2927134037017822,
          "mean_weights": [
            0.35746461153030396,
            0.11402665078639984,
            0.22846516966819763,
            0.3000435531139374
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.378767490386963,
          "mean_weights": [
            0.2549898624420166,
            0.23654252290725708,
            0.21668663620948792,
            0.2917810082435608
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.2672696113586426,
          "mean_weights": [
            0.3839718699455261,
            0.12064628303050995,
            0.1715085655450821,
            0.32387328147888184
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.375225305557251,
          "mean_weights": [
            0.23853997886180878,
            0.24968335032463074,
            0.2089165896177292,
            0.3028600811958313
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.5798927545547485,
          "mean_weights": [
            0.22726407647132874,
            0.12221817672252655,
            0.18549029529094696,
            0.23680329322814941,
            0.22822415828704834
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.594498634338379,
          "mean_weights": [
            0.21032124757766724,
            0.19536492228507996,
            0.1600019335746765,
            0.1794220358133316,
            0.2548898458480835
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5424928665161133,
          "mean_weights": [
            0.23003876209259033,
            0.11985675245523453,
            0.1379462629556656,
            0.19240085780620575,
            0.319757342338562
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.595255732536316,
          "mean_weights": [
            0.2046268880367279,
            0.19130583107471466,
            0.17092418670654297,
            0.17953911423683167,
            0.253603994846344
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.7457678318023682,
          "mean_weights": [
            0.17887650430202484,
            0.11389653384685516,
            0.11488981544971466,
            0.1474636197090149,
            0.20503896474838257,
            0.23983454704284668
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7712774276733398,
          "mean_weights": [
            0.16810789704322815,
            0.16247668862342834,
            0.1383969932794571,
            0.13383737206459045,
            0.1661752164363861,
            0.23100584745407104
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.675686001777649,
          "mean_weights": [
            0.16882894933223724,
            0.11588791012763977,
            0.08043210208415985,
            0.1250714659690857,
            0.19613179564476013,
            0.3136477470397949
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7837879657745361,
          "mean_weights": [
            0.17464125156402588,
            0.18035069108009338,
            0.1619444340467453,
            0.15330034494400024,
            0.15676669776439667,
            0.17299658060073853
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.6526988744735718,
          "mean_weights": [
            0.09642067551612854,
            0.07173497974872589,
            0.05561239272356033,
            0.09171071648597717,
            0.11456114053726196,
            0.12951162457466125,
            0.44044846296310425
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.8849880695343018,
          "mean_weights": [
            0.13606244325637817,
            0.13500544428825378,
            0.12342768162488937,
            0.11247340589761734,
            0.11233137547969818,
            0.11334770917892456,
            0.2673519253730774
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.644601583480835,
          "mean_weights": [
            0.10291393101215363,
            0.08116044104099274,
            0.05550362914800644,
            0.08702410012483597,
            0.11089185625314713,
            0.11528079956769943,
            0.4472252428531647
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9121408462524414,
          "mean_weights": [
            0.14125728607177734,
            0.1494411677122116,
            0.12786200642585754,
            0.11922307312488556,
            0.1195257231593132,
            0.11599892377853394,
            0.2266918122768402
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.7500674724578857,
          "mean_weights": [
            0.07664570212364197,
            0.0604618638753891,
            0.041497331112623215,
            0.06405743956565857,
            0.0716431736946106,
            0.07514288276433945,
            0.270142525434494,
            0.3404091000556946
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.952926754951477,
          "mean_weights": [
            0.10562807321548462,
            0.10351885855197906,
            0.07600137591362,
            0.08402621746063232,
            0.08850346505641937,
            0.09127135574817657,
            0.15880608558654785,
            0.2922445833683014
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.7527103424072266,
          "mean_weights": [
            0.08054207265377045,
            0.06648321449756622,
            0.04810335487127304,
            0.06408451497554779,
            0.07014857977628708,
            0.07217346131801605,
            0.20459653437137604,
            0.39386826753616333
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.9524379968643188,
          "mean_weights": [
            0.10777557641267776,
            0.09555133432149887,
            0.06806209683418274,
            0.08129536360502243,
            0.0918373167514801,
            0.10075562447309494,
            0.2000727653503418,
            0.25464993715286255
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.000408887863159,
          "mean_weights": [
            0.07838036119937897,
            0.07984089851379395,
            0.04941036179661751,
            0.06521518528461456,
            0.06765615195035934,
            0.06713232398033142,
            0.18892306089401245,
            0.23950761556625366,
            0.16393405199050903
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.096604347229004,
          "mean_weights": [
            0.09755367040634155,
            0.08867941796779633,
            0.060870569199323654,
            0.06980546563863754,
            0.08883253484964371,
            0.09541568160057068,
            0.17317476868629456,
            0.19020704925060272,
            0.13546085357666016
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.027402877807617,
          "mean_weights": [
            0.08629745244979858,
            0.08307671546936035,
            0.05121873319149017,
            0.0663638487458229,
            0.07480055093765259,
            0.08018200099468231,
            0.15043827891349792,
            0.25634968280792236,
            0.1512727439403534
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.055510263890028,
        0.04334520921111107,
        0.05630175396800041,
        0.058260008692741394,
        0.11489111185073853,
        0.08273965120315552,
        0.11550594866275787,
        0.09142354130744934,
        0.12066955119371414,
        0.0711931437253952,
        0.08880246430635452,
        0.06823744624853134,
        0.09300445020198822,
        0.06510575115680695,
        0.07136023044586182,
        0.05894701927900314,
        0.07342072576284409,
        0.05002941936254501,
        0.059311795979738235,
        0.04801420122385025,
        0.06730704009532928,
        0.02817549556493759,
        0.046817973256111145,
        0.03746112436056137,
        0.05246857553720474,
        0.03106791339814663,
        0.03552677109837532,
        0.03383694216609001,
        0.03949129208922386,
        0.03412646800279617,
        0.039247769862413406,
        0.04047730192542076
      ],
      "loss_nats": 1.433929681777954,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 1.913235068321228,
        "mean_weights": [
          0.09498223662376404,
          0.06303347647190094,
          0.06038350611925125,
          0.06479769200086594,
          0.05589054524898529,
          0.06028280779719353,
          0.07675591856241226,
          0.2476816177368164,
          0.27619218826293945
        ],
        "sources": 9
      },
      "step": 2000,
      "stream_state_rms_by_sublayer": [
        0.04482239484786987,
        0.09266375005245209,
        0.17245197296142578,
        0.20113667845726013,
        0.14111557602882385,
        0.14358533918857574,
        0.16786204278469086,
        0.17041823267936707,
        0.04827577620744705,
        0.05199930816888809,
        0.06626670062541962,
        0.07249397784471512,
        0.028491884469985962,
        0.03721517324447632,
        0.05938867852091789,
        0.07351206243038177,
        0.036527831107378006,
        0.05156216770410538,
        0.066385917365551,
        0.08937478810548782,
        0.04509544372558594,
        0.062482453882694244,
        0.07032289355993271,
        0.09533937275409698,
        0.02739257924258709,
        0.05472978577017784,
        0.058526892215013504,
        0.08922836184501648,
        0.02216578833758831,
        0.08201298862695694,
        0.0856289193034172,
        0.20069743692874908
      ]
    },
    {
      "activation_grad_rms_by_block": [
        5.492530908668414e-05,
        5.6656335800653324e-05,
        6.325786671368405e-05,
        5.9555983170866966e-05,
        6.814395601395518e-05,
        6.255660991882905e-05,
        7.046976679703221e-05,
        6.272071914281696e-05,
        7.296914554899558e-05,
        6.464503530878574e-05,
        0.00014864247350487858,
        0.00010719253623392433,
        0.0001445752423023805,
        9.124213829636574e-05,
        6.016199040459469e-05,
        4.548503056867048e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 5.859887369297212e-05,
        "first_to_last_ratio": 0.6864419649392262,
        "imbalance_abs_log_ratio": 0.37623359494524666,
        "last_quartile_mean": 8.536610039300285e-05,
        "mean": 7.707500867581984e-05,
        "normalized": [
          0.7126215102706234,
          0.7350804985173837,
          0.8207312305308824,
          0.772701608395031,
          0.8841251812318441,
          0.8116328625007921,
          0.9143011205283218,
          0.8137620769739057,
          0.9467289955931933,
          0.8387288748897194,
          1.9285430654970663,
          1.3907560709435642,
          1.8757732861305145,
          1.18380964029642,
          0.7805641729816484,
          0.5901398047190899
        ],
        "population_cv": 0.3863401678553132
      },
      "activation_output_rms_by_block": [
        0.14486487209796906,
        0.30995267629623413,
        0.2574450671672821,
        0.28713259100914,
        0.05978552997112274,
        0.08792455494403839,
        0.04697949066758156,
        0.09852420538663864,
        0.07244286686182022,
        0.13140366971492767,
        0.08212646842002869,
        0.12985831499099731,
        0.06840891391038895,
        0.11060241609811783,
        0.0924442708492279,
        0.23933660984039307
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.24984880164265633,
        "first_to_last_ratio": 1.9565592145673032,
        "imbalance_abs_log_ratio": 0.6711874277699096,
        "last_quartile_mean": 0.12769805267453194,
        "mean": 0.13870203238911927,
        "normalized": [
          1.0444322235420485,
          2.234665714390418,
          1.8561016210998063,
          2.0701397525568246,
          0.43103571693454673,
          0.6339096365752733,
          0.3387080193301259,
          0.7103299330916477,
          0.5222913147990983,
          0.9473809968860692,
          0.5921071739571081,
          0.9362394534110972,
          0.4932077254533107,
          0.7974102051210767,
          0.6664954309384717,
          1.725545081913077
        ],
        "population_cv": 0.5979328649415788
      },
      "bits_per_byte": 1.9415087059372296,
      "branch_output_rms_by_sublayer": [
        0.06868160516023636,
        0.12181960046291351,
        0.16538235545158386,
        0.0929274708032608,
        0.2537953853607178,
        0.01320737786591053,
        0.04932497814297676,
        0.013867277652025223,
        0.055594079196453094,
        0.0172379482537508,
        0.04591496288776398,
        0.035032741725444794,
        0.03727728873491287,
        0.025992874056100845,
        0.0513935461640358,
        0.04214895889163017,
        0.05297093465924263,
        0.04459983855485916,
        0.05040198564529419,
        0.06986947357654572,
        0.054249413311481476,
        0.05820845440030098,
        0.03775457665324211,
        0.08249213546514511,
        0.03362919017672539,
        0.05606101080775261,
        0.02498369663953781,
        0.07096737623214722,
        0.02770768664777279,
        0.08668208867311478,
        0.022235019132494926,
        0.16932900249958038
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0003274625120219605,
        0.0002950920627246982,
        0.0002675536202763891,
        0.00011636112532418193,
        0.00013949143839672265,
        0.0001705676155207084,
        0.00017763939173179324,
        0.00023071980568273914,
        0.00033685005198124334,
        0.0003711494248343089,
        0.0010020275600187679,
        0.0008489747430264115,
        0.0009447094037588545,
        0.0006663788253529681,
        0.0003910019531118643,
        0.00034755869426738895
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00025161733008680743,
        "first_to_last_ratio": 0.4283488185904071,
        "imbalance_abs_log_ratio": 0.8478174186232621,
        "last_quartile_mean": 0.000587412219122769,
        "mean": 0.00041459613925193755,
        "normalized": [
          0.789834928547107,
          0.7117578645501561,
          0.6453355324512677,
          0.2806613817826046,
          0.3364513684290676,
          0.4114066663246461,
          0.4284636901161473,
          0.5564928947457894,
          0.8124775416120065,
          0.895207141832,
          2.4168762444984226,
          2.047715023488232,
          2.278625665601732,
          1.607296263190791,
          0.9430911581023291,
          0.8383066347277007
        ],
        "population_cv": 0.6738437321539833
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.4194117784500122,
          "mean_weights": [
            0.8254866600036621,
            0.1745133101940155
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.5809611082077026,
          "mean_weights": [
            0.6678193211555481,
            0.3321806788444519
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.31704896688461304,
          "mean_weights": [
            0.8999109268188477,
            0.10008908808231354
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6732906103134155,
          "mean_weights": [
            0.4609490633010864,
            0.5390509366989136
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.9927794337272644,
          "mean_weights": [
            0.4832823872566223,
            0.17606672644615173,
            0.34065088629722595
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0832945108413696,
          "mean_weights": [
            0.2762538492679596,
            0.3156396150588989,
            0.40810656547546387
          ],
          "sources": 3
        },
        {
          "entropy_mean": 0.9905548095703125,
          "mean_weights": [
            0.4605642259120941,
            0.16346418857574463,
            0.37597161531448364
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.09372079372406,
          "mean_weights": [
            0.32242417335510254,
            0.34579548239707947,
            0.3317803740501404
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.2746227979660034,
          "mean_weights": [
            0.37224799394607544,
            0.09596331417560577,
            0.2427763193845749,
            0.2890123426914215
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3707606792449951,
          "mean_weights": [
            0.27432072162628174,
            0.2153838574886322,
            0.20746882259845734,
            0.30282658338546753
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.198214054107666,
          "mean_weights": [
            0.44253695011138916,
            0.10026887059211731,
            0.13153521716594696,
            0.325658917427063
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.372360110282898,
          "mean_weights": [
            0.24982473254203796,
            0.2343549281358719,
            0.2051662802696228,
            0.31065404415130615
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.5764262676239014,
          "mean_weights": [
            0.2228686511516571,
            0.11472160369157791,
            0.20744341611862183,
            0.22938773036003113,
            0.22557863593101501
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5863620042800903,
          "mean_weights": [
            0.2179623544216156,
            0.17305074632167816,
            0.15634828805923462,
            0.18247520923614502,
            0.2701634168624878
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.4903430938720703,
          "mean_weights": [
            0.2547823488712311,
            0.09995707869529724,
            0.10266982018947601,
            0.1966245472431183,
            0.3459661602973938
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5923073291778564,
          "mean_weights": [
            0.22257764637470245,
            0.17832019925117493,
            0.1619993895292282,
            0.1890055239200592,
            0.2480972558259964
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.691253662109375,
          "mean_weights": [
            0.18423792719841003,
            0.09657466411590576,
            0.08509969711303711,
            0.15257084369659424,
            0.18859976530075073,
            0.29291707277297974
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7595478296279907,
          "mean_weights": [
            0.1724337935447693,
            0.14752474427223206,
            0.13264453411102295,
            0.13973915576934814,
            0.15476800501346588,
            0.2528897523880005
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.5512144565582275,
          "mean_weights": [
            0.16704371571540833,
            0.08849328011274338,
            0.04574323818087578,
            0.12566779553890228,
            0.1718127727508545,
            0.40123921632766724
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7799327373504639,
          "mean_weights": [
            0.18437626957893372,
            0.17012780904769897,
            0.17119532823562622,
            0.15378496050834656,
            0.1488233208656311,
            0.17169234156608582
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.5764116048812866,
          "mean_weights": [
            0.09582588821649551,
            0.05901939421892166,
            0.040849827229976654,
            0.09081389009952545,
            0.10021352022886276,
            0.13485172390937805,
            0.4784257411956787
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.8431057929992676,
          "mean_weights": [
            0.13672879338264465,
            0.12035658955574036,
            0.11983893811702728,
            0.10482897609472275,
            0.10095320641994476,
            0.10664592683315277,
            0.3106476068496704
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.5526533126831055,
          "mean_weights": [
            0.10306400060653687,
            0.06465981900691986,
            0.04082629829645157,
            0.08607284724712372,
            0.09878605604171753,
            0.11178258061408997,
            0.4948084056377411
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.8949611186981201,
          "mean_weights": [
            0.1451701521873474,
            0.13929708302021027,
            0.1330569088459015,
            0.11840799450874329,
            0.10893036425113678,
            0.10698798298835754,
            0.24814949929714203
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.676889419555664,
          "mean_weights": [
            0.07736113667488098,
            0.04877929389476776,
            0.032506294548511505,
            0.06437233090400696,
            0.06250663101673126,
            0.06869478523731232,
            0.2701185345649719,
            0.3756609857082367
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.8798964023590088,
          "mean_weights": [
            0.10042576491832733,
            0.08721707761287689,
            0.06748545169830322,
            0.0845440924167633,
            0.07678259164094925,
            0.08073122054338455,
            0.1554347574710846,
            0.34737905859947205
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.663856029510498,
          "mean_weights": [
            0.07920834422111511,
            0.05299394577741623,
            0.03748705983161926,
            0.06209969520568848,
            0.060945168137550354,
            0.06926221400499344,
            0.19488413631916046,
            0.44311946630477905
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.9090958833694458,
          "mean_weights": [
            0.1065661758184433,
            0.0803120881319046,
            0.06028372794389725,
            0.08741366118192673,
            0.08287163078784943,
            0.09473372995853424,
            0.1887454390525818,
            0.29907354712486267
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.8632590770721436,
          "mean_weights": [
            0.0691547840833664,
            0.05765335261821747,
            0.03405343368649483,
            0.05519368499517441,
            0.05223262310028076,
            0.057386673986911774,
            0.1615200936794281,
            0.25146448612213135,
            0.2613408863544464
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.0701708793640137,
          "mean_weights": [
            0.09429951012134552,
            0.07856893539428711,
            0.04940951615571976,
            0.076503224670887,
            0.08159374445676804,
            0.0895766168832779,
            0.16711056232452393,
            0.2026878148317337,
            0.16025006771087646
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.9422186613082886,
          "mean_weights": [
            0.07884211093187332,
            0.06442099809646606,
            0.037228696048259735,
            0.06615173816680908,
            0.06501315534114838,
            0.0695233941078186,
            0.13319489359855652,
            0.2805612087249756,
            0.20506379008293152
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.05865398049354553,
        0.05170911177992821,
        0.06799201667308807,
        0.06424286961555481,
        0.17415167391300201,
        0.13562268018722534,
        0.1871119737625122,
        0.1505064219236374,
        0.18585707247257233,
        0.1044568344950676,
        0.12573961913585663,
        0.08286763727664948,
        0.13412274420261383,
        0.0987003743648529,
        0.09964247792959213,
        0.07276491075754166,
        0.1041010394692421,
        0.06199917569756508,
        0.0827898308634758,
        0.06073001027107239,
        0.10040723532438278,
        0.036688607186079025,
        0.0660671666264534,
        0.05293381214141846,
        0.07866564393043518,
        0.04279600828886032,
        0.04821427911520004,
        0.04622510448098183,
        0.053889211267232895,
        0.04172775521874428,
        0.05102568119764328,
        0.053032174706459045
      ],
      "loss_nats": 1.3457512855529785,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 1.7949464321136475,
        "mean_weights": [
          0.0834353119134903,
          0.05303390324115753,
          0.048570070415735245,
          0.05988207459449768,
          0.04784438759088516,
          0.05165069177746773,
          0.06509792059659958,
          0.2334408313035965,
          0.3570447862148285
        ],
        "sources": 9
      },
      "step": 4000,
      "stream_state_rms_by_sublayer": [
        0.06868160516023636,
        0.14486487209796906,
        0.2633455693721771,
        0.30995267629623413,
        0.2537953853607178,
        0.2574450671672821,
        0.28461524844169617,
        0.28713259100914,
        0.055594079196453094,
        0.05978552997112274,
        0.07737205922603607,
        0.08792455494403839,
        0.03727728873491287,
        0.04697949066758156,
        0.07717122882604599,
        0.09852420538663864,
        0.05297093465924263,
        0.07244286686182022,
        0.09201814234256744,
        0.13140366971492767,
        0.054249413311481476,
        0.08212646842002869,
        0.09215818345546722,
        0.12985831499099731,
        0.03362919017672539,
        0.06840891391038895,
        0.07437368482351303,
        0.11060241609811783,
        0.02770768664777279,
        0.0924442708492279,
        0.09714528173208237,
        0.23933660984039307
      ]
    },
    {
      "activation_grad_rms_by_block": [
        6.899879372213036e-05,
        7.050983549561352e-05,
        8.400688238907605e-05,
        8.067955786827952e-05,
        9.6201452834066e-05,
        8.90634983079508e-05,
        0.00010480848868610337,
        9.241609222954139e-05,
        0.00012084992340533063,
        0.00010282631410518661,
        0.00023664520995225757,
        0.0001637248496990651,
        0.00023947370937094092,
        0.0001411983830621466,
        0.00010364947956986725,
        6.421437865355983e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 7.604876736877486e-05,
        "first_to_last_ratio": 0.5545581271583456,
        "imbalance_abs_log_ratio": 0.5895836496274593,
        "last_quartile_mean": 0.00013713398766412865,
        "mean": 0.00011620417808444472,
        "normalized": [
          0.5937720558721171,
          0.6067753901617422,
          0.7229248016196876,
          0.6942913688494942,
          0.8278656965686476,
          0.7664397251123708,
          0.9019339098972828,
          0.7952906147865294,
          1.0399791590755874,
          0.8848762221825061,
          2.0364604255476015,
          1.4089411619958057,
          2.060801197672231,
          1.2150886946555293,
          0.891960007621636,
          0.5525995683812308
        ],
        "population_cv": 0.4517546138899352
      },
      "activation_output_rms_by_block": [
        0.15090593695640564,
        0.3367030918598175,
        0.29793402552604675,
        0.3255155086517334,
        0.05378299206495285,
        0.08782409131526947,
        0.044081781059503555,
        0.09709836542606354,
        0.0669482871890068,
        0.12247904390096664,
        0.07360157370567322,
        0.12167256325483322,
        0.06349270045757294,
        0.10058031976222992,
        0.07551935315132141,
        0.19356463849544525
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.2777646407485008,
        "first_to_last_ratio": 2.565024996839381,
        "imbalance_abs_log_ratio": 0.941968223931731,
        "last_quartile_mean": 0.10828925296664238,
        "mean": 0.13823151704855263,
        "normalized": [
          1.0916897982346618,
          2.4357910485895444,
          2.155326309711265,
          2.3548573842056504,
          0.38907908422984344,
          0.6353405734845695,
          0.3188981934128684,
          0.7024329002477675,
          0.4843199916954669,
          0.8860428252259355,
          0.5324514645948746,
          0.8802085505008006,
          0.45932144718683565,
          0.7276221943430017,
          0.5463251417894509,
          1.400293092547464
        ],
        "population_cv": 0.6866316887805146
      },
      "bits_per_byte": 1.8676321823979032,
      "branch_output_rms_by_sublayer": [
        0.07169820368289948,
        0.12625248730182648,
        0.19025787711143494,
        0.09543261677026749,
        0.2934682369232178,
        0.013759235851466656,
        0.04595690965652466,
        0.014787186868488789,
        0.048878561705350876,
        0.017967192456126213,
        0.039782870560884476,
        0.03611106425523758,
        0.03420666605234146,
        0.025706099346280098,
        0.046541497111320496,
        0.04672740399837494,
        0.04840917885303497,
        0.04226908087730408,
        0.0452793650329113,
        0.07270561158657074,
        0.04624715819954872,
        0.05458534136414528,
        0.0338163822889328,
        0.0825076773762703,
        0.03031807579100132,
        0.0527048334479332,
        0.022609181702136993,
        0.06718777120113373,
        0.02396680787205696,
        0.07009968906641006,
        0.018549330532550812,
        0.1417386531829834
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00046247293295990515,
        0.0003824975682884986,
        0.00041690386457328876,
        0.00021732383526524047,
        0.00022655054548791778,
        0.0002917516444309179,
        0.000280629308661355,
        0.0003847878626551635,
        0.0005123546145807317,
        0.0005899763687075028,
        0.0015400377759773427,
        0.0012824907104132966,
        0.001537981149104749,
        0.001057571910144345,
        0.0006563117026664665,
        0.0005021677227557722
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00036979955027173326,
        "first_to_last_ratio": 0.394029142562531,
        "imbalance_abs_log_ratio": 0.9313304065247964,
        "last_quartile_mean": 0.0009385081211678332,
        "mean": 0.0006463630947920308,
        "normalized": [
          0.7155002144865761,
          0.5917688855852271,
          0.6449994870258314,
          0.3362256246006202,
          0.3505004343739815,
          0.45137423033842894,
          0.434166664097133,
          0.595312241301416,
          0.792673063652533,
          0.9127630792369563,
          2.3826202151483513,
          1.9841645055956383,
          2.3794383706260316,
          1.6361885736756394,
          1.0153916706485797,
          0.7769127396070564
        ],
        "population_cv": 0.675897465620864
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.43482720851898193,
          "mean_weights": [
            0.8150075674057007,
            0.18499243259429932
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.5611247420310974,
          "mean_weights": [
            0.6986296772956848,
            0.3013703525066376
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.31192272901535034,
          "mean_weights": [
            0.9031083583831787,
            0.09689167141914368
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6764674186706543,
          "mean_weights": [
            0.4619978070259094,
            0.5380021929740906
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.9888706207275391,
          "mean_weights": [
            0.48646074533462524,
            0.1758481115102768,
            0.33769115805625916
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0781972408294678,
          "mean_weights": [
            0.276906281709671,
            0.2993447184562683,
            0.42374899983406067
          ],
          "sources": 3
        },
        {
          "entropy_mean": 0.9839773178100586,
          "mean_weights": [
            0.46118396520614624,
            0.15739500522613525,
            0.3814210295677185
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0957449674606323,
          "mean_weights": [
            0.3292894959449768,
            0.32970887422561646,
            0.3410016596317291
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.2506439685821533,
          "mean_weights": [
            0.3648650050163269,
            0.0835169106721878,
            0.212466761469841,
            0.3391513228416443
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3655824661254883,
          "mean_weights": [
            0.2861315608024597,
            0.20811524987220764,
            0.20031729340553284,
            0.3054359257221222
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.1652674674987793,
          "mean_weights": [
            0.43905186653137207,
            0.08880466222763062,
            0.10791443288326263,
            0.3642290234565735
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.363007664680481,
          "mean_weights": [
            0.2524264454841614,
            0.21885012090206146,
            0.19597390294075012,
            0.33274954557418823
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.5649902820587158,
          "mean_weights": [
            0.22229184210300446,
            0.10384112596511841,
            0.1934116780757904,
            0.2539443373680115,
            0.22651101648807526
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5797945261001587,
          "mean_weights": [
            0.21991753578186035,
            0.1605391800403595,
            0.15625253319740295,
            0.18265283107757568,
            0.2806379497051239
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.4139833450317383,
          "mean_weights": [
            0.24990326166152954,
            0.08050692081451416,
            0.06835947930812836,
            0.20686665177345276,
            0.394363671541214
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5895863771438599,
          "mean_weights": [
            0.2244223803281784,
            0.16505929827690125,
            0.1643591672182083,
            0.19251328706741333,
            0.2536458969116211
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6177464723587036,
          "mean_weights": [
            0.1738494634628296,
            0.07661910355091095,
            0.06195354461669922,
            0.14467598497867584,
            0.19003543257713318,
            0.35286644101142883
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7444759607315063,
          "mean_weights": [
            0.172578826546669,
            0.1303727626800537,
            0.13036483526229858,
            0.13626831769943237,
            0.1541597843170166,
            0.27625542879104614
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.470452904701233,
          "mean_weights": [
            0.15901926159858704,
            0.07292591035366058,
            0.03418249264359474,
            0.11740352213382721,
            0.16174395382404327,
            0.45472484827041626
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.778969645500183,
          "mean_weights": [
            0.18775978684425354,
            0.15501554310321808,
            0.17432880401611328,
            0.15116485953330994,
            0.14906319975852966,
            0.1826678216457367
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.531482219696045,
          "mean_weights": [
            0.09255115687847137,
            0.04949861764907837,
            0.03270748257637024,
            0.08605319261550903,
            0.09672325849533081,
            0.1471216380596161,
            0.4953446388244629
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.8260389566421509,
          "mean_weights": [
            0.13673919439315796,
            0.10493747889995575,
            0.11646746844053268,
            0.09983028471469879,
            0.10169579833745956,
            0.1137751042842865,
            0.32655465602874756
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.4906136989593506,
          "mean_weights": [
            0.10088150203227997,
            0.05257141590118408,
            0.03239753842353821,
            0.08061188459396362,
            0.09571854770183563,
            0.11411523073911667,
            0.523703932762146
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.8863525390625,
          "mean_weights": [
            0.14810651540756226,
            0.12629631161689758,
            0.13384857773780823,
            0.11124427616596222,
            0.1082591712474823,
            0.11263860762119293,
            0.2596065402030945
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.5897703170776367,
          "mean_weights": [
            0.07248121500015259,
            0.03783262521028519,
            0.02471475675702095,
            0.053565122187137604,
            0.058412034064531326,
            0.06658737361431122,
            0.26026394963264465,
            0.42614293098449707
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.814478874206543,
          "mean_weights": [
            0.09743618965148926,
            0.07339455187320709,
            0.06428516656160355,
            0.07672879844903946,
            0.07112542539834976,
            0.07841956615447998,
            0.14423531293869019,
            0.3943749666213989
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.5432119369506836,
          "mean_weights": [
            0.07432855665683746,
            0.04178062081336975,
            0.029931148514151573,
            0.0517733134329319,
            0.053720228374004364,
            0.06251072138547897,
            0.17899465560913086,
            0.5069607496261597
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.8694422245025635,
          "mean_weights": [
            0.10448876023292542,
            0.07235300540924072,
            0.05752243846654892,
            0.08492527902126312,
            0.07930260896682739,
            0.08949960768222809,
            0.1757010817527771,
            0.33620721101760864
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.746185302734375,
          "mean_weights": [
            0.06162667274475098,
            0.04293470457196236,
            0.02593991532921791,
            0.04546971619129181,
            0.044149380177259445,
            0.052234895527362823,
            0.14164786040782928,
            0.2587880492210388,
            0.3272087574005127
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.061832904815674,
          "mean_weights": [
            0.09089995175600052,
            0.06807379424571991,
            0.04443599283695221,
            0.08077649772167206,
            0.08381937444210052,
            0.09205536544322968,
            0.15836367011070251,
            0.20683875679969788,
            0.17473658919334412
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.849208116531372,
          "mean_weights": [
            0.07359056174755096,
            0.050991274416446686,
            0.029770921915769577,
            0.0592363178730011,
            0.055861424654722214,
            0.06031768023967743,
            0.11290363222360611,
            0.2822529673576355,
            0.275075227022171
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.06120532751083374,
        0.05362993851304054,
        0.06824863702058792,
        0.06727377325296402,
        0.1878596842288971,
        0.1534913182258606,
        0.21103329956531525,
        0.1696806252002716,
        0.20562554895877838,
        0.10344218462705612,
        0.13388557732105255,
        0.08117230981588364,
        0.14070458710193634,
        0.10246162861585617,
        0.10507974773645401,
        0.06554406136274338,
        0.11050253361463547,
        0.05469200015068054,
        0.08388946205377579,
        0.05699421092867851,
        0.10618287324905396,
        0.03451571613550186,
        0.06762637197971344,
        0.050157804042100906,
        0.08278355002403259,
        0.03850048407912254,
        0.04651740565896034,
        0.04368532821536064,
        0.05266447737812996,
        0.036846842616796494,
        0.04717044159770012,
        0.048093993216753006
      ],
      "loss_nats": 1.294543981552124,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 1.636631727218628,
        "mean_weights": [
          0.07672396302223206,
          0.03940216451883316,
          0.03699969872832298,
          0.05207572132349014,
          0.04068697616457939,
          0.04208337888121605,
          0.051315970718860626,
          0.2188674658536911,
          0.4418446719646454
        ],
        "sources": 9
      },
      "step": 8000,
      "stream_state_rms_by_sublayer": [
        0.07169820368289948,
        0.15090593695640564,
        0.29125523567199707,
        0.3367030918598175,
        0.2934682369232178,
        0.29793402552604675,
        0.3226769268512726,
        0.3255155086517334,
        0.048878561705350876,
        0.05378299206495285,
        0.0762643963098526,
        0.08782409131526947,
        0.03420666605234146,
        0.044081781059503555,
        0.07245275378227234,
        0.09709836542606354,
        0.04840917885303497,
        0.0669482871890068,
        0.08269722014665604,
        0.12247904390096664,
        0.04624715819954872,
        0.07360157370567322,
        0.08215674012899399,
        0.12167256325483322,
        0.03031807579100132,
        0.06349270045757294,
        0.06805067509412766,
        0.10058031976222992,
        0.02396680787205696,
        0.07551935315132141,
        0.07906875014305115,
        0.19356463849544525
      ]
    }
  ],
  "environment": {
    "autocast": "cuda-bfloat16-forward-fp32-cross-entropy",
    "compile": false,
    "compute_capability": [
      12,
      0
    ],
    "cublas_workspace_config": ":4096:8",
    "cuda": "12.8",
    "deterministic_algorithms": true,
    "gpu": "NVIDIA GeForce RTX 5090",
    "python": "3.10.14",
    "torch": "2.11.0+cu128"
  },
  "evaluations": [
    {
      "bits_per_byte": 8.030247040746552,
      "cross_entropy_nats": 5.566143095493317,
      "step": 0
    },
    {
      "bits_per_byte": 5.230785295754151,
      "cross_entropy_nats": 3.6257040798664093,
      "step": 100
    },
    {
      "bits_per_byte": 3.16099769723908,
      "cross_entropy_nats": 2.191036641597748,
      "step": 500
    },
    {
      "bits_per_byte": 1.983385019569791,
      "cross_entropy_nats": 1.3747777342796326,
      "step": 2000
    },
    {
      "bits_per_byte": 1.818564586202164,
      "cross_entropy_nats": 1.2605329155921936,
      "step": 4000
    },
    {
      "bits_per_byte": 1.7311772990915377,
      "cross_entropy_nats": 1.1999606639146805,
      "step": 8000
    }
  ],
  "gradient_gate": null,
  "hashes": {
    "final_mixer_parameters": "7517e8197003ce4ba5267d82e82d6a41d49e7ce8744b4095b5a5160f1d4d01c2",
    "final_model_state": "9867cf3c0d2372bb731808e8fd98ce5f38db0a40bd3cc08683bc5f66c9472428",
    "final_optimizer_state": "fa42da28c86335ac8bab241bdce833e1a77d2508025d738b471764f2dd16cf93",
    "final_public_parameters": "8c73d93d872090c7e7d7cdebca989ac8c083cf6d1cb5fb16509c6e6993d3c9a0",
    "initial_mixer_parameters": "64fe0b90f5ea84ed88b6d93838558443c700c2b89ade2455ed526bd8db5d506f",
    "initial_public_parameter_elements": 9541824,
    "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe",
    "initial_public_parameter_tensors": 115,
    "initial_public_parameters": "9fd4f04212ab5cea822f469902d8e80ecc368da329f3c20abacfe6b7a50ed523"
  },
  "manifest": {
    "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716",
    "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371",
    "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
    "input_gate_tensor_hashes": {
      "0": "add6249175ac3bb3fa911069fbe8f25d17d9ab05df34b66ee89996983522ce83",
      "1": "619b5ee4f70e975312ead9adf5df4f9d8584ce2839369c749442e429923405cf",
      "7999": "fe22994f7750c2f8323fdb5ac51c9be1e208733abaf6a337f5b1ae1df1cd2730"
    },
    "path": "experiments/k3/attnres_gradient/manifest.json",
    "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338"
  },
  "model": {
    "attnres_aggregation_groups": 8,
    "context": 256,
    "d_ff": 768,
    "d_head": 32,
    "d_model": 192,
    "heads": 6,
    "layers": 16,
    "parameters": {
      "core": 9541824,
      "embedding": 98304,
      "mixer": 12672,
      "total": 9554496
    },
    "sublayers": 32,
    "sublayers_per_attnres_group": 4,
    "transformer_blocks_per_attnres_group": 2,
    "vocabulary": 256
  },
  "optimizer": {
    "betas": [
      0.9,
      0.95
    ],
    "epsilon": 1e-08,
    "grad_clip": 1.0,
    "min_lr": 3e-05,
    "name": "AdamW",
    "peak_lr": 0.0003,
    "warmup_steps": 400,
    "weight_decay_ndim_ge_2": 0.1
  },
  "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
  "run_kind": "formal",
  "schema_version": 1,
  "seed": 2026073002,
  "steps": 8000,
  "target_bytes_seen": 65536000,
  "timing": {
    "mean_ms": 54.937352295237,
    "measured_steps": 7980,
    "median_ms": 54.89947448950261,
    "p95_ms": 55.28610254696105,
    "peak_allocated_bytes": 6536921088,
    "peak_reserved_bytes": 6889144320,
    "warmup_steps_excluded": 20
  },
  "training_history": [
    {
      "bits_per_byte": 8.033595800932828,
      "learning_rate": 7.499999999999999e-07,
      "loss_nats": 5.568464279174805,
      "step": 1,
      "unclipped_grad_norm": 13.626673698425293
    },
    {
      "bits_per_byte": 7.5882983213188435,
      "learning_rate": 7.499999999999999e-06,
      "loss_nats": 5.259807586669922,
      "step": 10,
      "unclipped_grad_norm": 10.73694896697998
    },
    {
      "bits_per_byte": 6.986202883052029,
      "learning_rate": 1.4999999999999999e-05,
      "loss_nats": 4.842466831207275,
      "step": 20,
      "unclipped_grad_norm": 4.4299492835998535
    },
    {
      "bits_per_byte": 6.739012218966915,
      "learning_rate": 2.2499999999999998e-05,
      "loss_nats": 4.6711273193359375,
      "step": 30,
      "unclipped_grad_norm": 3.205341339111328
    },
    {
      "bits_per_byte": 6.623832685961678,
      "learning_rate": 2.9999999999999997e-05,
      "loss_nats": 4.5912909507751465,
      "step": 40,
      "unclipped_grad_norm": 3.10516619682312
    },
    {
      "bits_per_byte": 6.3865585415253125,
      "learning_rate": 3.75e-05,
      "loss_nats": 4.426825046539307,
      "step": 50,
      "unclipped_grad_norm": 3.1714048385620117
    },
    {
      "bits_per_byte": 6.103358837933146,
      "learning_rate": 4.4999999999999996e-05,
      "loss_nats": 4.230525970458984,
      "step": 60,
      "unclipped_grad_norm": 2.932384490966797
    },
    {
      "bits_per_byte": 5.8558104499342365,
      "learning_rate": 5.2499999999999995e-05,
      "loss_nats": 4.058938503265381,
      "step": 70,
      "unclipped_grad_norm": 2.6460657119750977
    },
    {
      "bits_per_byte": 5.640669096802708,
      "learning_rate": 5.9999999999999995e-05,
      "loss_nats": 3.90981388092041,
      "step": 80,
      "unclipped_grad_norm": 2.5319507122039795
    },
    {
      "bits_per_byte": 5.451491620506636,
      "learning_rate": 6.749999999999999e-05,
      "loss_nats": 3.778686046600342,
      "step": 90,
      "unclipped_grad_norm": 2.4583022594451904
    },
    {
      "bits_per_byte": 5.270375658987107,
      "learning_rate": 7.5e-05,
      "loss_nats": 3.6531460285186768,
      "step": 100,
      "unclipped_grad_norm": 2.433866262435913
    },
    {
      "bits_per_byte": 5.10413206918359,
      "learning_rate": 8.249999999999999e-05,
      "loss_nats": 3.537914752960205,
      "step": 110,
      "unclipped_grad_norm": 2.285804271697998
    },
    {
      "bits_per_byte": 4.823540998439711,
      "learning_rate": 8.999999999999999e-05,
      "loss_nats": 3.343423843383789,
      "step": 120,
      "unclipped_grad_norm": 2.0681240558624268
    },
    {
      "bits_per_byte": 4.663901886394628,
      "learning_rate": 9.75e-05,
      "loss_nats": 3.2327704429626465,
      "step": 130,
      "unclipped_grad_norm": 1.8398888111114502
    },
    {
      "bits_per_byte": 4.463271741412299,
      "learning_rate": 0.00010499999999999999,
      "loss_nats": 3.0937042236328125,
      "step": 140,
      "unclipped_grad_norm": 1.681030035018921
    },
    {
      "bits_per_byte": 4.327779681478958,
      "learning_rate": 0.0001125,
      "loss_nats": 2.999788284301758,
      "step": 150,
      "unclipped_grad_norm": 1.4562726020812988
    },
    {
      "bits_per_byte": 4.208912840439223,
      "learning_rate": 0.00011999999999999999,
      "loss_nats": 2.917396068572998,
      "step": 160,
      "unclipped_grad_norm": 1.1446112394332886
    },
    {
      "bits_per_byte": 4.0496410833363825,
      "learning_rate": 0.00012749999999999998,
      "loss_nats": 2.806997299194336,
      "step": 170,
      "unclipped_grad_norm": 1.193884253501892
    },
    {
      "bits_per_byte": 3.9053046435997705,
      "learning_rate": 0.00013499999999999997,
      "loss_nats": 2.7069509029388428,
      "step": 180,
      "unclipped_grad_norm": 1.1651051044464111
    },
    {
      "bits_per_byte": 3.9164302012840855,
      "learning_rate": 0.0001425,
      "loss_nats": 2.714662551879883,
      "step": 190,
      "unclipped_grad_norm": 1.1523211002349854
    },
    {
      "bits_per_byte": 3.7885273917680182,
      "learning_rate": 0.00015,
      "loss_nats": 2.626007080078125,
      "step": 200,
      "unclipped_grad_norm": 0.7202922701835632
    },
    {
      "bits_per_byte": 3.72972377171561,
      "learning_rate": 0.0001575,
      "loss_nats": 2.58524751663208,
      "step": 210,
      "unclipped_grad_norm": 0.606606662273407
    },
    {
      "bits_per_byte": 3.7010143638417796,
      "learning_rate": 0.00016499999999999997,
      "loss_nats": 2.565347671508789,
      "step": 220,
      "unclipped_grad_norm": 0.6799013018608093
    },
    {
      "bits_per_byte": 3.715506997864687,
      "learning_rate": 0.0001725,
      "loss_nats": 2.5753931999206543,
      "step": 230,
      "unclipped_grad_norm": 0.6519911289215088
    },
    {
      "bits_per_byte": 3.7068112110717615,
      "learning_rate": 0.00017999999999999998,
      "loss_nats": 2.5693657398223877,
      "step": 240,
      "unclipped_grad_norm": 0.912699818611145
    },
    {
      "bits_per_byte": 3.5753026132977044,
      "learning_rate": 0.0001875,
      "loss_nats": 2.478210926055908,
      "step": 250,
      "unclipped_grad_norm": 0.6829943060874939
    },
    {
      "bits_per_byte": 3.5693214006659977,
      "learning_rate": 0.000195,
      "loss_nats": 2.474065065383911,
      "step": 260,
      "unclipped_grad_norm": 0.640023410320282
    },
    {
      "bits_per_byte": 3.539509240034835,
      "learning_rate": 0.00020249999999999996,
      "loss_nats": 2.4534008502960205,
      "step": 270,
      "unclipped_grad_norm": 0.6751542687416077
    },
    {
      "bits_per_byte": 3.5394008909647914,
      "learning_rate": 0.00020999999999999998,
      "loss_nats": 2.4533257484436035,
      "step": 280,
      "unclipped_grad_norm": 0.7841543555259705
    },
    {
      "bits_per_byte": 3.4452169999768865,
      "learning_rate": 0.00021749999999999997,
      "loss_nats": 2.388042449951172,
      "step": 290,
      "unclipped_grad_norm": 0.8627198934555054
    },
    {
      "bits_per_byte": 3.5355663657811545,
      "learning_rate": 0.000225,
      "loss_nats": 2.4506678581237793,
      "step": 300,
      "unclipped_grad_norm": 1.1167978048324585
    },
    {
      "bits_per_byte": 3.4429038333227817,
      "learning_rate": 0.00023249999999999996,
      "loss_nats": 2.386439085006714,
      "step": 310,
      "unclipped_grad_norm": 0.8460172414779663
    },
    {
      "bits_per_byte": 3.4221950583670893,
      "learning_rate": 0.00023999999999999998,
      "loss_nats": 2.372084856033325,
      "step": 320,
      "unclipped_grad_norm": 0.821152925491333
    },
    {
      "bits_per_byte": 3.429671832130699,
      "learning_rate": 0.0002475,
      "loss_nats": 2.377267360687256,
      "step": 330,
      "unclipped_grad_norm": 0.9970561265945435
    },
    {
      "bits_per_byte": 3.425132178078524,
      "learning_rate": 0.00025499999999999996,
      "loss_nats": 2.3741207122802734,
      "step": 340,
      "unclipped_grad_norm": 1.0991977453231812
    },
    {
      "bits_per_byte": 3.393254505810499,
      "learning_rate": 0.0002625,
      "loss_nats": 2.352024793624878,
      "step": 350,
      "unclipped_grad_norm": 0.9725445508956909
    },
    {
      "bits_per_byte": 3.382284076477262,
      "learning_rate": 0.00026999999999999995,
      "loss_nats": 2.3444206714630127,
      "step": 360,
      "unclipped_grad_norm": 1.0039488077163696
    },
    {
      "bits_per_byte": 3.4119693139120884,
      "learning_rate": 0.00027749999999999997,
      "loss_nats": 2.364996910095215,
      "step": 370,
      "unclipped_grad_norm": 1.7546980381011963
    },
    {
      "bits_per_byte": 3.446912748914394,
      "learning_rate": 0.000285,
      "loss_nats": 2.3892178535461426,
      "step": 380,
      "unclipped_grad_norm": 0.9759950041770935
    },
    {
      "bits_per_byte": 3.3737516732026567,
      "learning_rate": 0.0002925,
      "loss_nats": 2.3385064601898193,
      "step": 390,
      "unclipped_grad_norm": 0.9621731638908386
    },
    {
      "bits_per_byte": 3.3279967208365115,
      "learning_rate": 0.0003,
      "loss_nats": 2.3067915439605713,
      "step": 400,
      "unclipped_grad_norm": 0.9455732107162476
    },
    {
      "bits_per_byte": 3.307004174506895,
      "learning_rate": 0.0002999988466112388,
      "loss_nats": 2.292240619659424,
      "step": 410,
      "unclipped_grad_norm": 1.4261541366577148
    },
    {
      "bits_per_byte": 3.3082469211420302,
      "learning_rate": 0.00029999538646466355,
      "loss_nats": 2.2931020259857178,
      "step": 420,
      "unclipped_grad_norm": 0.9587261080741882
    },
    {
      "bits_per_byte": 3.315426508884949,
      "learning_rate": 0.0002999896196193985,
      "loss_nats": 2.2980785369873047,
      "step": 430,
      "unclipped_grad_norm": 2.1751644611358643
    },
    {
      "bits_per_byte": 3.299565580891777,
      "learning_rate": 0.00029998154617398326,
      "loss_nats": 2.2870845794677734,
      "step": 440,
      "unclipped_grad_norm": 1.11757493019104
    },
    {
      "bits_per_byte": 3.305290883339,
      "learning_rate": 0.0002999711662663707,
      "loss_nats": 2.291053056716919,
      "step": 450,
      "unclipped_grad_norm": 1.8777321577072144
    },
    {
      "bits_per_byte": 3.227380334741036,
      "learning_rate": 0.0002999584800739247,
      "loss_nats": 2.2370495796203613,
      "step": 460,
      "unclipped_grad_norm": 1.1160534620285034
    },
    {
      "bits_per_byte": 3.2510836716135514,
      "learning_rate": 0.0002999434878134173,
      "loss_nats": 2.253479480743408,
      "step": 470,
      "unclipped_grad_norm": 1.6702216863632202
    },
    {
      "bits_per_byte": 3.198324209843066,
      "learning_rate": 0.000299926189741025,
      "loss_nats": 2.216909408569336,
      "step": 480,
      "unclipped_grad_norm": 1.0277327299118042
    },
    {
      "bits_per_byte": 3.1569424323230604,
      "learning_rate": 0.000299906586152324,
      "loss_nats": 2.188225746154785,
      "step": 490,
      "unclipped_grad_norm": 1.0115255117416382
    },
    {
      "bits_per_byte": 3.1652801512368893,
      "learning_rate": 0.0002998846773822856,
      "loss_nats": 2.194005012512207,
      "step": 500,
      "unclipped_grad_norm": 1.3175867795944214
    },
    {
      "bits_per_byte": 3.134954794375552,
      "learning_rate": 0.0002998604638052702,
      "loss_nats": 2.172985076904297,
      "step": 510,
      "unclipped_grad_norm": 0.9790136814117432
    },
    {
      "bits_per_byte": 3.0134504274021845,
      "learning_rate": 0.0002998339458350211,
      "loss_nats": 2.0887646675109863,
      "step": 520,
      "unclipped_grad_norm": 1.0963760614395142
    },
    {
      "bits_per_byte": 3.0906245462891544,
      "learning_rate": 0.0002998051239246572,
      "loss_nats": 2.1422576904296875,
      "step": 530,
      "unclipped_grad_norm": 1.0304492712020874
    },
    {
      "bits_per_byte": 3.024995622754605,
      "learning_rate": 0.00029977399856666537,
      "loss_nats": 2.0967671871185303,
      "step": 540,
      "unclipped_grad_norm": 1.323768973350525
    },
    {
      "bits_per_byte": 2.9805828229957934,
      "learning_rate": 0.0002997405702928922,
      "loss_nats": 2.0659825801849365,
      "step": 550,
      "unclipped_grad_norm": 1.1477609872817993
    },
    {
      "bits_per_byte": 2.9872017472968992,
      "learning_rate": 0.0002997048396745345,
      "loss_nats": 2.070570468902588,
      "step": 560,
      "unclipped_grad_norm": 1.223684310913086
    },
    {
      "bits_per_byte": 2.9579880862907393,
      "learning_rate": 0.0002996668073221302,
      "loss_nats": 2.050321102142334,
      "step": 570,
      "unclipped_grad_norm": 1.308420181274414
    },
    {
      "bits_per_byte": 2.8546603937044113,
      "learning_rate": 0.000299626473885547,
      "loss_nats": 1.978699803352356,
      "step": 580,
      "unclipped_grad_norm": 1.0794485807418823
    },
    {
      "bits_per_byte": 2.911534540396789,
      "learning_rate": 0.00029958384005397226,
      "loss_nats": 2.0181219577789307,
      "step": 590,
      "unclipped_grad_norm": 1.4487859010696411
    },
    {
      "bits_per_byte": 2.825803768681442,
      "learning_rate": 0.0002995389065559004,
      "loss_nats": 1.9586979150772095,
      "step": 600,
      "unclipped_grad_norm": 1.2767362594604492
    },
    {
      "bits_per_byte": 2.883143472409706,
      "learning_rate": 0.000299491674159121,
      "loss_nats": 1.9984427690505981,
      "step": 610,
      "unclipped_grad_norm": 1.1309611797332764
    },
    {
      "bits_per_byte": 2.7721532647968363,
      "learning_rate": 0.00029944214367070535,
      "loss_nats": 1.9215102195739746,
      "step": 620,
      "unclipped_grad_norm": 1.533046007156372
    },
    {
      "bits_per_byte": 2.7733690101557857,
      "learning_rate": 0.00029939031593699284,
      "loss_nats": 1.922352910041809,
      "step": 630,
      "unclipped_grad_norm": 0.9842169880867004
    },
    {
      "bits_per_byte": 2.7997499449195846,
      "learning_rate": 0.00029933619184357654,
      "loss_nats": 1.940638780593872,
      "step": 640,
      "unclipped_grad_norm": 1.2905892133712769
    },
    {
      "bits_per_byte": 2.715926288819655,
      "learning_rate": 0.00029927977231528774,
      "loss_nats": 1.8825366497039795,
      "step": 650,
      "unclipped_grad_norm": 1.1535414457321167
    },
    {
      "bits_per_byte": 2.6764600540649592,
      "learning_rate": 0.00029922105831618057,
      "loss_nats": 1.8551807403564453,
      "step": 660,
      "unclipped_grad_norm": 1.1525585651397705
    },
    {
      "bits_per_byte": 2.696502224265908,
      "learning_rate": 0.0002991600508495153,
      "loss_nats": 1.8690729141235352,
      "step": 670,
      "unclipped_grad_norm": 1.264070749282837
    },
    {
      "bits_per_byte": 2.7320018511611033,
      "learning_rate": 0.00029909675095774114,
      "loss_nats": 1.8936793804168701,
      "step": 680,
      "unclipped_grad_norm": 1.6425000429153442
    },
    {
      "bits_per_byte": 2.602245068668735,
      "learning_rate": 0.0002990311597224787,
      "loss_nats": 1.8037388324737549,
      "step": 690,
      "unclipped_grad_norm": 1.3727850914001465
    },
    {
      "bits_per_byte": 2.5536510267021466,
      "learning_rate": 0.00029896327826450103,
      "loss_nats": 1.7700560092926025,
      "step": 700,
      "unclipped_grad_norm": 2.108612298965454
    },
    {
      "bits_per_byte": 2.623769213372168,
      "learning_rate": 0.0002988931077437152,
      "loss_nats": 1.8186582326889038,
      "step": 710,
      "unclipped_grad_norm": 1.441254734992981
    },
    {
      "bits_per_byte": 2.6835147824033516,
      "learning_rate": 0.00029882064935914164,
      "loss_nats": 1.8600707054138184,
      "step": 720,
      "unclipped_grad_norm": 1.4497488737106323
    },
    {
      "bits_per_byte": 2.4942898388956416,
      "learning_rate": 0.0002987459043488943,
      "loss_nats": 1.728909969329834,
      "step": 730,
      "unclipped_grad_norm": 1.7889822721481323
    },
    {
      "bits_per_byte": 2.518159998939494,
      "learning_rate": 0.0002986688739901591,
      "loss_nats": 1.7454555034637451,
      "step": 740,
      "unclipped_grad_norm": 1.1689488887786865
    },
    {
      "bits_per_byte": 2.5159042744907776,
      "learning_rate": 0.0002985895595991723,
      "loss_nats": 1.743891954421997,
      "step": 750,
      "unclipped_grad_norm": 1.466252088546753
    },
    {
      "bits_per_byte": 2.5144026939660944,
      "learning_rate": 0.00029850796253119824,
      "loss_nats": 1.7428511381149292,
      "step": 760,
      "unclipped_grad_norm": 1.454863429069519
    },
    {
      "bits_per_byte": 2.485389736713491,
      "learning_rate": 0.00029842408418050544,
      "loss_nats": 1.722740888595581,
      "step": 770,
      "unclipped_grad_norm": 1.2171542644500732
    },
    {
      "bits_per_byte": 2.4154798210336645,
      "learning_rate": 0.0002983379259803436,
      "loss_nats": 1.6742830276489258,
      "step": 780,
      "unclipped_grad_norm": 1.3978826999664307
    },
    {
      "bits_per_byte": 2.492438617641754,
      "learning_rate": 0.0002982494894029186,
      "loss_nats": 1.7276268005371094,
      "step": 790,
      "unclipped_grad_norm": 1.4353379011154175
    },
    {
      "bits_per_byte": 2.4424274386320564,
      "learning_rate": 0.0002981587759593675,
      "loss_nats": 1.6929616928100586,
      "step": 800,
      "unclipped_grad_norm": 1.3198370933532715
    },
    {
      "bits_per_byte": 2.473405641636069,
      "learning_rate": 0.00029806578719973283,
      "loss_nats": 1.7144341468811035,
      "step": 810,
      "unclipped_grad_norm": 1.3822002410888672
    },
    {
      "bits_per_byte": 2.441873482513627,
      "learning_rate": 0.00029797052471293596,
      "loss_nats": 1.6925777196884155,
      "step": 820,
      "unclipped_grad_norm": 1.3033113479614258
    },
    {
      "bits_per_byte": 2.4253691674235776,
      "learning_rate": 0.0002978729901267499,
      "loss_nats": 1.6811378002166748,
      "step": 830,
      "unclipped_grad_norm": 1.450317621231079
    },
    {
      "bits_per_byte": 2.4107564895103684,
      "learning_rate": 0.0002977731851077716,
      "loss_nats": 1.6710090637207031,
      "step": 840,
      "unclipped_grad_norm": 1.322706699371338
    },
    {
      "bits_per_byte": 2.347782118191886,
      "learning_rate": 0.0002976711113613935,
      "loss_nats": 1.6273585557937622,
      "step": 850,
      "unclipped_grad_norm": 1.2753863334655762
    },
    {
      "bits_per_byte": 2.3967822111971615,
      "learning_rate": 0.0002975667706317743,
      "loss_nats": 1.661322832107544,
      "step": 860,
      "unclipped_grad_norm": 2.4916832447052
    },
    {
      "bits_per_byte": 2.3462526764777634,
      "learning_rate": 0.00029746016470180915,
      "loss_nats": 1.626298427581787,
      "step": 870,
      "unclipped_grad_norm": 1.3712787628173828
    },
    {
      "bits_per_byte": 2.4080783756711326,
      "learning_rate": 0.00029735129539309926,
      "loss_nats": 1.6691527366638184,
      "step": 880,
      "unclipped_grad_norm": 1.4296884536743164
    },
    {
      "bits_per_byte": 2.363901364126654,
      "learning_rate": 0.0002972401645659207,
      "loss_nats": 1.6385315656661987,
      "step": 890,
      "unclipped_grad_norm": 1.3781569004058838
    },
    {
      "bits_per_byte": 2.2957977822288393,
      "learning_rate": 0.00029712677411919283,
      "loss_nats": 1.5913257598876953,
      "step": 900,
      "unclipped_grad_norm": 1.3345599174499512
    },
    {
      "bits_per_byte": 2.3445476404771095,
      "learning_rate": 0.0002970111259904454,
      "loss_nats": 1.6251165866851807,
      "step": 910,
      "unclipped_grad_norm": 1.6203302145004272
    },
    {
      "bits_per_byte": 2.297230053745225,
      "learning_rate": 0.0002968932221557859,
      "loss_nats": 1.5923185348510742,
      "step": 920,
      "unclipped_grad_norm": 1.685173511505127
    },
    {
      "bits_per_byte": 2.297947909329926,
      "learning_rate": 0.0002967730646298655,
      "loss_nats": 1.5928161144256592,
      "step": 930,
      "unclipped_grad_norm": 1.4890943765640259
    },
    {
      "bits_per_byte": 2.2609902135207154,
      "learning_rate": 0.0002966506554658449,
      "loss_nats": 1.5671989917755127,
      "step": 940,
      "unclipped_grad_norm": 1.3654035329818726
    },
    {
      "bits_per_byte": 2.2430326450308753,
      "learning_rate": 0.00029652599675535876,
      "loss_nats": 1.5547517538070679,
      "step": 950,
      "unclipped_grad_norm": 1.1998120546340942
    },
    {
      "bits_per_byte": 2.2479709548677502,
      "learning_rate": 0.0002963990906284808,
      "loss_nats": 1.558174729347229,
      "step": 960,
      "unclipped_grad_norm": 1.40827214717865
    },
    {
      "bits_per_byte": 2.2631277858882894,
      "learning_rate": 0.0002962699392536863,
      "loss_nats": 1.5686806440353394,
      "step": 970,
      "unclipped_grad_norm": 1.4645981788635254
    },
    {
      "bits_per_byte": 2.2528444272452495,
      "learning_rate": 0.0002961385448378161,
      "loss_nats": 1.5615527629852295,
      "step": 980,
      "unclipped_grad_norm": 1.594014048576355
    },
    {
      "bits_per_byte": 2.177915369882643,
      "learning_rate": 0.00029600490962603817,
      "loss_nats": 1.5096158981323242,
      "step": 990,
      "unclipped_grad_norm": 1.2862826585769653
    },
    {
      "bits_per_byte": 2.2127330855671676,
      "learning_rate": 0.00029586903590180956,
      "loss_nats": 1.5337496995925903,
      "step": 1000,
      "unclipped_grad_norm": 1.4609711170196533
    },
    {
      "bits_per_byte": 2.208716086790965,
      "learning_rate": 0.00029573092598683735,
      "loss_nats": 1.5309653282165527,
      "step": 1010,
      "unclipped_grad_norm": 1.2298572063446045
    },
    {
      "bits_per_byte": 2.233356385145381,
      "learning_rate": 0.00029559058224103895,
      "loss_nats": 1.5480446815490723,
      "step": 1020,
      "unclipped_grad_norm": 1.469051718711853
    },
    {
      "bits_per_byte": 2.1593615375422615,
      "learning_rate": 0.0002954480070625017,
      "loss_nats": 1.4967553615570068,
      "step": 1030,
      "unclipped_grad_norm": 1.3269134759902954
    },
    {
      "bits_per_byte": 2.236164345886018,
      "learning_rate": 0.00029530320288744205,
      "loss_nats": 1.5499910116195679,
      "step": 1040,
      "unclipped_grad_norm": 1.3589463233947754
    },
    {
      "bits_per_byte": 2.2039167389359875,
      "learning_rate": 0.0002951561721901638,
      "loss_nats": 1.5276386737823486,
      "step": 1050,
      "unclipped_grad_norm": 1.4478572607040405
    },
    {
      "bits_per_byte": 2.1642855728191144,
      "learning_rate": 0.00029500691748301594,
      "loss_nats": 1.5001684427261353,
      "step": 1060,
      "unclipped_grad_norm": 1.4022979736328125
    },
    {
      "bits_per_byte": 2.2672321518511303,
      "learning_rate": 0.00029485544131634963,
      "loss_nats": 1.5715255737304688,
      "step": 1070,
      "unclipped_grad_norm": 1.4223567247390747
    },
    {
      "bits_per_byte": 2.187999916581285,
      "learning_rate": 0.0002947017462784745,
      "loss_nats": 1.5166059732437134,
      "step": 1080,
      "unclipped_grad_norm": 1.188625454902649
    },
    {
      "bits_per_byte": 2.1147346193831256,
      "learning_rate": 0.00029454583499561476,
      "loss_nats": 1.4658223390579224,
      "step": 1090,
      "unclipped_grad_norm": 1.1601507663726807
    },
    {
      "bits_per_byte": 2.1969277079702247,
      "learning_rate": 0.00029438771013186413,
      "loss_nats": 1.522794246673584,
      "step": 1100,
      "unclipped_grad_norm": 1.1728529930114746
    },
    {
      "bits_per_byte": 2.2198211785398274,
      "learning_rate": 0.0002942273743891401,
      "loss_nats": 1.5386627912521362,
      "step": 1110,
      "unclipped_grad_norm": 1.3392254114151
    },
    {
      "bits_per_byte": 2.126032503683605,
      "learning_rate": 0.00029406483050713824,
      "loss_nats": 1.4736534357070923,
      "step": 1120,
      "unclipped_grad_norm": 1.3545775413513184
    },
    {
      "bits_per_byte": 2.1170070261489755,
      "learning_rate": 0.00029390008126328504,
      "loss_nats": 1.4673974514007568,
      "step": 1130,
      "unclipped_grad_norm": 1.2578480243682861
    },
    {
      "bits_per_byte": 2.127665650936198,
      "learning_rate": 0.00029373312947269045,
      "loss_nats": 1.4747854471206665,
      "step": 1140,
      "unclipped_grad_norm": 1.6450976133346558
    },
    {
      "bits_per_byte": 2.1180404698980104,
      "learning_rate": 0.00029356397798809995,
      "loss_nats": 1.4681137800216675,
      "step": 1150,
      "unclipped_grad_norm": 1.3040683269500732
    },
    {
      "bits_per_byte": 2.0764574726764953,
      "learning_rate": 0.0002933926296998457,
      "loss_nats": 1.4392906427383423,
      "step": 1160,
      "unclipped_grad_norm": 1.3104010820388794
    },
    {
      "bits_per_byte": 2.1839928927988326,
      "learning_rate": 0.0002932190875357972,
      "loss_nats": 1.5138285160064697,
      "step": 1170,
      "unclipped_grad_norm": 1.3359254598617554
    },
    {
      "bits_per_byte": 2.091760488950268,
      "learning_rate": 0.00029304335446131126,
      "loss_nats": 1.4498978853225708,
      "step": 1180,
      "unclipped_grad_norm": 1.3903101682662964
    },
    {
      "bits_per_byte": 2.1040587963308166,
      "learning_rate": 0.00029286543347918116,
      "loss_nats": 1.4584224224090576,
      "step": 1190,
      "unclipped_grad_norm": 1.4786474704742432
    },
    {
      "bits_per_byte": 2.081396986391926,
      "learning_rate": 0.0002926853276295856,
      "loss_nats": 1.4427144527435303,
      "step": 1200,
      "unclipped_grad_norm": 1.1696821451187134
    },
    {
      "bits_per_byte": 2.2277741722636764,
      "learning_rate": 0.0002925030399900368,
      "loss_nats": 1.544175386428833,
      "step": 1210,
      "unclipped_grad_norm": 1.6089588403701782
    },
    {
      "bits_per_byte": 2.071474963298349,
      "learning_rate": 0.00029231857367532726,
      "loss_nats": 1.4358370304107666,
      "step": 1220,
      "unclipped_grad_norm": 1.219665765762329
    },
    {
      "bits_per_byte": 2.020156544159443,
      "learning_rate": 0.0002921319318374776,
      "loss_nats": 1.4002658128738403,
      "step": 1230,
      "unclipped_grad_norm": 1.1330406665802002
    },
    {
      "bits_per_byte": 2.0202031514578267,
      "learning_rate": 0.00029194311766568174,
      "loss_nats": 1.4002981185913086,
      "step": 1240,
      "unclipped_grad_norm": 1.4843499660491943
    },
    {
      "bits_per_byte": 2.1912978558942307,
      "learning_rate": 0.000291752134386253,
      "loss_nats": 1.5188919305801392,
      "step": 1250,
      "unclipped_grad_norm": 1.2303693294525146
    },
    {
      "bits_per_byte": 2.124901717754182,
      "learning_rate": 0.00029155898526256856,
      "loss_nats": 1.472869634628296,
      "step": 1260,
      "unclipped_grad_norm": 1.2569689750671387
    },
    {
      "bits_per_byte": 2.0056606424661694,
      "learning_rate": 0.0002913636735950141,
      "loss_nats": 1.3902180194854736,
      "step": 1270,
      "unclipped_grad_norm": 1.3274110555648804
    },
    {
      "bits_per_byte": 2.135876962601643,
      "learning_rate": 0.0002911662027209272,
      "loss_nats": 1.4804770946502686,
      "step": 1280,
      "unclipped_grad_norm": 1.203521966934204
    },
    {
      "bits_per_byte": 2.0353620462701763,
      "learning_rate": 0.00029096657601454013,
      "loss_nats": 1.4108054637908936,
      "step": 1290,
      "unclipped_grad_norm": 1.1280517578125
    },
    {
      "bits_per_byte": 2.0816712987200523,
      "learning_rate": 0.00029076479688692266,
      "loss_nats": 1.4429045915603638,
      "step": 1300,
      "unclipped_grad_norm": 1.2584275007247925
    },
    {
      "bits_per_byte": 2.144767433755345,
      "learning_rate": 0.00029056086878592323,
      "loss_nats": 1.4866394996643066,
      "step": 1310,
      "unclipped_grad_norm": 1.2562772035598755
    },
    {
      "bits_per_byte": 2.0818640912716693,
      "learning_rate": 0.00029035479519611045,
      "loss_nats": 1.4430382251739502,
      "step": 1320,
      "unclipped_grad_norm": 1.1640011072158813
    },
    {
      "bits_per_byte": 1.9798470784675668,
      "learning_rate": 0.0002901465796387134,
      "loss_nats": 1.3723254203796387,
      "step": 1330,
      "unclipped_grad_norm": 1.20253324508667
    },
    {
      "bits_per_byte": 2.0020558861040847,
      "learning_rate": 0.00028993622567156154,
      "loss_nats": 1.3877193927764893,
      "step": 1340,
      "unclipped_grad_norm": 1.1452504396438599
    },
    {
      "bits_per_byte": 2.176502532405805,
      "learning_rate": 0.0002897237368890237,
      "loss_nats": 1.5086365938186646,
      "step": 1350,
      "unclipped_grad_norm": 1.2560323476791382
    },
    {
      "bits_per_byte": 2.009120589436227,
      "learning_rate": 0.0002895091169219469,
      "loss_nats": 1.3926162719726562,
      "step": 1360,
      "unclipped_grad_norm": 1.1015381813049316
    },
    {
      "bits_per_byte": 2.0924102394052277,
      "learning_rate": 0.00028929236943759425,
      "loss_nats": 1.4503482580184937,
      "step": 1370,
      "unclipped_grad_norm": 1.1648008823394775
    },
    {
      "bits_per_byte": 2.0633229856474515,
      "learning_rate": 0.00028907349813958225,
      "loss_nats": 1.4301865100860596,
      "step": 1380,
      "unclipped_grad_norm": 1.124997854232788
    },
    {
      "bits_per_byte": 2.0840274610368725,
      "learning_rate": 0.0002888525067678175,
      "loss_nats": 1.4445377588272095,
      "step": 1390,
      "unclipped_grad_norm": 1.2330824136734009
    },
    {
      "bits_per_byte": 2.084845582507027,
      "learning_rate": 0.00028862939909843273,
      "loss_nats": 1.4451048374176025,
      "step": 1400,
      "unclipped_grad_norm": 1.2093770503997803
    },
    {
      "bits_per_byte": 2.0117663705371007,
      "learning_rate": 0.0002884041789437225,
      "loss_nats": 1.3944501876831055,
      "step": 1410,
      "unclipped_grad_norm": 1.1717233657836914
    },
    {
      "bits_per_byte": 2.0551273244032355,
      "learning_rate": 0.0002881768501520778,
      "loss_nats": 1.4245057106018066,
      "step": 1420,
      "unclipped_grad_norm": 1.118058204650879
    },
    {
      "bits_per_byte": 2.04396186674392,
      "learning_rate": 0.0002879474166079205,
      "loss_nats": 1.4167664051055908,
      "step": 1430,
      "unclipped_grad_norm": 1.282050371170044
    },
    {
      "bits_per_byte": 2.01288047414939,
      "learning_rate": 0.00028771588223163694,
      "loss_nats": 1.3952224254608154,
      "step": 1440,
      "unclipped_grad_norm": 1.2009295225143433
    },
    {
      "bits_per_byte": 2.0606357567277205,
      "learning_rate": 0.00028748225097951054,
      "loss_nats": 1.4283238649368286,
      "step": 1450,
      "unclipped_grad_norm": 1.1897740364074707
    },
    {
      "bits_per_byte": 2.0059322030718816,
      "learning_rate": 0.000287246526843655,
      "loss_nats": 1.3904062509536743,
      "step": 1460,
      "unclipped_grad_norm": 1.0764319896697998
    },
    {
      "bits_per_byte": 2.00337052148728,
      "learning_rate": 0.00028700871385194533,
      "loss_nats": 1.3886306285858154,
      "step": 1470,
      "unclipped_grad_norm": 1.102418065071106
    },
    {
      "bits_per_byte": 1.9153828214446602,
      "learning_rate": 0.00028676881606794945,
      "loss_nats": 1.3276422023773193,
      "step": 1480,
      "unclipped_grad_norm": 1.2223329544067383
    },
    {
      "bits_per_byte": 2.0269326606034372,
      "learning_rate": 0.00028652683759085854,
      "loss_nats": 1.4049626588821411,
      "step": 1490,
      "unclipped_grad_norm": 1.1502348184585571
    },
    {
      "bits_per_byte": 1.9137665284918515,
      "learning_rate": 0.0002862827825554171,
      "loss_nats": 1.326521873474121,
      "step": 1500,
      "unclipped_grad_norm": 1.104032039642334
    },
    {
      "bits_per_byte": 1.9998878728073082,
      "learning_rate": 0.00028603665513185245,
      "loss_nats": 1.386216640472412,
      "step": 1510,
      "unclipped_grad_norm": 1.2259501218795776
    },
    {
      "bits_per_byte": 1.9900984483375461,
      "learning_rate": 0.000285788459525803,
      "loss_nats": 1.379431128501892,
      "step": 1520,
      "unclipped_grad_norm": 1.192402958869934
    },
    {
      "bits_per_byte": 1.9577958069392392,
      "learning_rate": 0.00028553819997824707,
      "loss_nats": 1.3570406436920166,
      "step": 1530,
      "unclipped_grad_norm": 1.1483442783355713
    },
    {
      "bits_per_byte": 2.0275844748502077,
      "learning_rate": 0.00028528588076542964,
      "loss_nats": 1.4054144620895386,
      "step": 1540,
      "unclipped_grad_norm": 1.132032871246338
    },
    {
      "bits_per_byte": 2.0123937632474482,
      "learning_rate": 0.00028503150619878996,
      "loss_nats": 1.3948850631713867,
      "step": 1550,
      "unclipped_grad_norm": 1.1288913488388062
    },
    {
      "bits_per_byte": 2.071048790289511,
      "learning_rate": 0.0002847750806248876,
      "loss_nats": 1.4355416297912598,
      "step": 1560,
      "unclipped_grad_norm": 1.146075963973999
    },
    {
      "bits_per_byte": 2.0218080935556784,
      "learning_rate": 0.000284516608425328,
      "loss_nats": 1.4014105796813965,
      "step": 1570,
      "unclipped_grad_norm": 1.1695427894592285
    },
    {
      "bits_per_byte": 2.0099278759993773,
      "learning_rate": 0.000284256094016688,
      "loss_nats": 1.3931758403778076,
      "step": 1580,
      "unclipped_grad_norm": 1.1285966634750366
    },
    {
      "bits_per_byte": 1.999800161655368,
      "learning_rate": 0.0002839935418504399,
      "loss_nats": 1.3861558437347412,
      "step": 1590,
      "unclipped_grad_norm": 1.136389136314392
    },
    {
      "bits_per_byte": 2.019440580383901,
      "learning_rate": 0.000283728956412876,
      "loss_nats": 1.3997695446014404,
      "step": 1600,
      "unclipped_grad_norm": 1.4346710443496704
    },
    {
      "bits_per_byte": 2.0397723693689023,
      "learning_rate": 0.00028346234222503136,
      "loss_nats": 1.4138624668121338,
      "step": 1610,
      "unclipped_grad_norm": 1.3147592544555664
    },
    {
      "bits_per_byte": 2.0494484572717453,
      "learning_rate": 0.00028319370384260675,
      "loss_nats": 1.4205694198608398,
      "step": 1620,
      "unclipped_grad_norm": 1.2143282890319824
    },
    {
      "bits_per_byte": 1.921803965697274,
      "learning_rate": 0.00028292304585589105,
      "loss_nats": 1.3320930004119873,
      "step": 1630,
      "unclipped_grad_norm": 1.3924999237060547
    },
    {
      "bits_per_byte": 2.0487386848716347,
      "learning_rate": 0.0002826503728896824,
      "loss_nats": 1.4200774431228638,
      "step": 1640,
      "unclipped_grad_norm": 1.1033058166503906
    },
    {
      "bits_per_byte": 1.9803865880433233,
      "learning_rate": 0.0002823756896032095,
      "loss_nats": 1.3726993799209595,
      "step": 1650,
      "unclipped_grad_norm": 1.1198365688323975
    },
    {
      "bits_per_byte": 1.9604881953384965,
      "learning_rate": 0.0002820990006900518,
      "loss_nats": 1.358906865119934,
      "step": 1660,
      "unclipped_grad_norm": 1.0561511516571045
    },
    {
      "bits_per_byte": 2.1180337625746266,
      "learning_rate": 0.00028182031087805947,
      "loss_nats": 1.468109130859375,
      "step": 1670,
      "unclipped_grad_norm": 1.1442371606826782
    },
    {
      "bits_per_byte": 2.035602821981384,
      "learning_rate": 0.0002815396249292725,
      "loss_nats": 1.4109723567962646,
      "step": 1680,
      "unclipped_grad_norm": 1.0968410968780518
    },
    {
      "bits_per_byte": 1.9074798746722124,
      "learning_rate": 0.0002812569476398391,
      "loss_nats": 1.3221642971038818,
      "step": 1690,
      "unclipped_grad_norm": 1.049187183380127
    },
    {
      "bits_per_byte": 1.9460256582794904,
      "learning_rate": 0.00028097228383993446,
      "loss_nats": 1.3488821983337402,
      "step": 1700,
      "unclipped_grad_norm": 1.1202492713928223
    },
    {
      "bits_per_byte": 2.030805537918217,
      "learning_rate": 0.0002806856383936774,
      "loss_nats": 1.4076471328735352,
      "step": 1710,
      "unclipped_grad_norm": 1.272226095199585
    },
    {
      "bits_per_byte": 2.0120939974869945,
      "learning_rate": 0.00028039701619904774,
      "loss_nats": 1.3946772813796997,
      "step": 1720,
      "unclipped_grad_norm": 1.1422768831253052
    },
    {
      "bits_per_byte": 1.9049754633103482,
      "learning_rate": 0.0002801064221878024,
      "loss_nats": 1.3204283714294434,
      "step": 1730,
      "unclipped_grad_norm": 1.1759874820709229
    },
    {
      "bits_per_byte": 1.9628171843964806,
      "learning_rate": 0.00027981386132539134,
      "loss_nats": 1.3605211973190308,
      "step": 1740,
      "unclipped_grad_norm": 1.0964117050170898
    },
    {
      "bits_per_byte": 2.0712307479341234,
      "learning_rate": 0.0002795193386108724,
      "loss_nats": 1.4356677532196045,
      "step": 1750,
      "unclipped_grad_norm": 1.1146889925003052
    },
    {
      "bits_per_byte": 1.9217773083863903,
      "learning_rate": 0.00027922285907682625,
      "loss_nats": 1.332074522972107,
      "step": 1760,
      "unclipped_grad_norm": 1.044190764427185
    },
    {
      "bits_per_byte": 1.9951038314082574,
      "learning_rate": 0.00027892442778927005,
      "loss_nats": 1.382900595664978,
      "step": 1770,
      "unclipped_grad_norm": 1.0358567237854004
    },
    {
      "bits_per_byte": 1.9346029145744843,
      "learning_rate": 0.00027862404984757116,
      "loss_nats": 1.3409645557403564,
      "step": 1780,
      "unclipped_grad_norm": 1.1725130081176758
    },
    {
      "bits_per_byte": 1.9131714685198662,
      "learning_rate": 0.0002783217303843597,
      "loss_nats": 1.3261094093322754,
      "step": 1790,
      "unclipped_grad_norm": 1.2142103910446167
    },
    {
      "bits_per_byte": 1.9953318804033016,
      "learning_rate": 0.00027801747456544134,
      "loss_nats": 1.3830586671829224,
      "step": 1800,
      "unclipped_grad_norm": 1.2420480251312256
    },
    {
      "bits_per_byte": 1.9723242133535257,
      "learning_rate": 0.00027771128758970843,
      "loss_nats": 1.3671109676361084,
      "step": 1810,
      "unclipped_grad_norm": 1.1430673599243164
    },
    {
      "bits_per_byte": 2.0342909383206025,
      "learning_rate": 0.00027740317468905165,
      "loss_nats": 1.4100630283355713,
      "step": 1820,
      "unclipped_grad_norm": 1.1687225103378296
    },
    {
      "bits_per_byte": 1.8908906001526369,
      "learning_rate": 0.0002770931411282704,
      "loss_nats": 1.310665488243103,
      "step": 1830,
      "unclipped_grad_norm": 1.146242380142212
    },
    {
      "bits_per_byte": 1.978836336428446,
      "learning_rate": 0.0002767811922049827,
      "loss_nats": 1.3716248273849487,
      "step": 1840,
      "unclipped_grad_norm": 1.0820521116256714
    },
    {
      "bits_per_byte": 1.8676517884201014,
      "learning_rate": 0.00027646733324953503,
      "loss_nats": 1.2945575714111328,
      "step": 1850,
      "unclipped_grad_norm": 1.0944465398788452
    },
    {
      "bits_per_byte": 2.0069171477133727,
      "learning_rate": 0.00027615156962491097,
      "loss_nats": 1.3910889625549316,
      "step": 1860,
      "unclipped_grad_norm": 1.2028480768203735
    },
    {
      "bits_per_byte": 2.0278259384920188,
      "learning_rate": 0.0002758339067266396,
      "loss_nats": 1.4055818319320679,
      "step": 1870,
      "unclipped_grad_norm": 1.1793227195739746
    },
    {
      "bits_per_byte": 1.8922862393643884,
      "learning_rate": 0.0002755143499827035,
      "loss_nats": 1.3116328716278076,
      "step": 1880,
      "unclipped_grad_norm": 1.123720645904541
    },
    {
      "bits_per_byte": 1.9751340659033223,
      "learning_rate": 0.0002751929048534456,
      "loss_nats": 1.369058609008789,
      "step": 1890,
      "unclipped_grad_norm": 1.006156086921692
    },
    {
      "bits_per_byte": 1.8741764662285347,
      "learning_rate": 0.0002748695768314763,
      "loss_nats": 1.2990801334381104,
      "step": 1900,
      "unclipped_grad_norm": 1.0260387659072876
    },
    {
      "bits_per_byte": 1.8761183223394267,
      "learning_rate": 0.00027454437144157947,
      "loss_nats": 1.3004261255264282,
      "step": 1910,
      "unclipped_grad_norm": 1.11003839969635
    },
    {
      "bits_per_byte": 1.9401261374069434,
      "learning_rate": 0.00027421729424061787,
      "loss_nats": 1.3447929620742798,
      "step": 1920,
      "unclipped_grad_norm": 1.1032140254974365
    },
    {
      "bits_per_byte": 1.9031926911515042,
      "learning_rate": 0.0002738883508174385,
      "loss_nats": 1.31919264793396,
      "step": 1930,
      "unclipped_grad_norm": 1.1424028873443604
    },
    {
      "bits_per_byte": 1.8781860697507502,
      "learning_rate": 0.0002735575467927768,
      "loss_nats": 1.3018593788146973,
      "step": 1940,
      "unclipped_grad_norm": 1.0988123416900635
    },
    {
      "bits_per_byte": 1.9390512458390508,
      "learning_rate": 0.0002732248878191609,
      "loss_nats": 1.3440479040145874,
      "step": 1950,
      "unclipped_grad_norm": 1.28093421459198
    },
    {
      "bits_per_byte": 1.9095305958011006,
      "learning_rate": 0.0002728903795808148,
      "loss_nats": 1.3235857486724854,
      "step": 1960,
      "unclipped_grad_norm": 1.052490234375
    },
    {
      "bits_per_byte": 1.8582639434401043,
      "learning_rate": 0.0002725540277935614,
      "loss_nats": 1.2880504131317139,
      "step": 1970,
      "unclipped_grad_norm": 1.0047657489776611
    },
    {
      "bits_per_byte": 1.8448327863383223,
      "learning_rate": 0.00027221583820472456,
      "loss_nats": 1.278740644454956,
      "step": 1980,
      "unclipped_grad_norm": 1.101292610168457
    },
    {
      "bits_per_byte": 1.9462698736437154,
      "learning_rate": 0.0002718758165930314,
      "loss_nats": 1.3490514755249023,
      "step": 1990,
      "unclipped_grad_norm": 1.17987859249115
    },
    {
      "bits_per_byte": 1.896080176642422,
      "learning_rate": 0.0002715339687685131,
      "loss_nats": 1.3142626285552979,
      "step": 2000,
      "unclipped_grad_norm": 1.1739213466644287
    },
    {
      "bits_per_byte": 1.9557370026257606,
      "learning_rate": 0.00027119030057240576,
      "loss_nats": 1.3556135892868042,
      "step": 2010,
      "unclipped_grad_norm": 1.0457618236541748
    },
    {
      "bits_per_byte": 1.9291909645171335,
      "learning_rate": 0.0002708448178770506,
      "loss_nats": 1.3372132778167725,
      "step": 2020,
      "unclipped_grad_norm": 1.1008185148239136
    },
    {
      "bits_per_byte": 1.9305355248815788,
      "learning_rate": 0.00027049752658579376,
      "loss_nats": 1.3381452560424805,
      "step": 2030,
      "unclipped_grad_norm": 1.1080964803695679
    },
    {
      "bits_per_byte": 1.9343679862734056,
      "learning_rate": 0.00027014843263288514,
      "loss_nats": 1.34080171585083,
      "step": 2040,
      "unclipped_grad_norm": 1.104992389678955
    },
    {
      "bits_per_byte": 1.8835350741578853,
      "learning_rate": 0.00026979754198337724,
      "loss_nats": 1.3055670261383057,
      "step": 2050,
      "unclipped_grad_norm": 1.1497511863708496
    },
    {
      "bits_per_byte": 1.7856179557677012,
      "learning_rate": 0.00026944486063302305,
      "loss_nats": 1.2376960515975952,
      "step": 2060,
      "unclipped_grad_norm": 1.0708798170089722
    },
    {
      "bits_per_byte": 1.8782170266279055,
      "learning_rate": 0.0002690903946081738,
      "loss_nats": 1.3018808364868164,
      "step": 2070,
      "unclipped_grad_norm": 1.0489474534988403
    },
    {
      "bits_per_byte": 1.9968320850667778,
      "learning_rate": 0.00026873414996567596,
      "loss_nats": 1.3840985298156738,
      "step": 2080,
      "unclipped_grad_norm": 1.0985883474349976
    },
    {
      "bits_per_byte": 1.8512350124993413,
      "learning_rate": 0.00026837613279276736,
      "loss_nats": 1.2831783294677734,
      "step": 2090,
      "unclipped_grad_norm": 0.9845348000526428
    },
    {
      "bits_per_byte": 1.9251153196569866,
      "learning_rate": 0.0002680163492069739,
      "loss_nats": 1.334388256072998,
      "step": 2100,
      "unclipped_grad_norm": 1.009676456451416
    },
    {
      "bits_per_byte": 1.8359577936231979,
      "learning_rate": 0.00026765480535600414,
      "loss_nats": 1.2725889682769775,
      "step": 2110,
      "unclipped_grad_norm": 1.0760974884033203
    },
    {
      "bits_per_byte": 1.8868116876221235,
      "learning_rate": 0.0002672915074176452,
      "loss_nats": 1.3078382015228271,
      "step": 2120,
      "unclipped_grad_norm": 0.9828676581382751
    },
    {
      "bits_per_byte": 1.910013867050025,
      "learning_rate": 0.00026692646159965624,
      "loss_nats": 1.323920726776123,
      "step": 2130,
      "unclipped_grad_norm": 1.0712695121765137
    },
    {
      "bits_per_byte": 1.8022548694815659,
      "learning_rate": 0.0002665596741396633,
      "loss_nats": 1.2492278814315796,
      "step": 2140,
      "unclipped_grad_norm": 1.0451561212539673
    },
    {
      "bits_per_byte": 1.8551622363317934,
      "learning_rate": 0.000266191151305052,
      "loss_nats": 1.2859004735946655,
      "step": 2150,
      "unclipped_grad_norm": 1.0957838296890259
    },
    {
      "bits_per_byte": 1.9088151479735114,
      "learning_rate": 0.00026582089939286075,
      "loss_nats": 1.323089838027954,
      "step": 2160,
      "unclipped_grad_norm": 1.0826293230056763
    },
    {
      "bits_per_byte": 1.871177432762789,
      "learning_rate": 0.0002654489247296732,
      "loss_nats": 1.2970013618469238,
      "step": 2170,
      "unclipped_grad_norm": 1.0311518907546997
    },
    {
      "bits_per_byte": 1.8055650195627224,
      "learning_rate": 0.00026507523367151007,
      "loss_nats": 1.2515223026275635,
      "step": 2180,
      "unclipped_grad_norm": 1.040400505065918
    },
    {
      "bits_per_byte": 1.9270800494604432,
      "learning_rate": 0.00026469983260372037,
      "loss_nats": 1.3357501029968262,
      "step": 2190,
      "unclipped_grad_norm": 1.10823392868042
    },
    {
      "bits_per_byte": 1.980437322925328,
      "learning_rate": 0.0002643227279408727,
      "loss_nats": 1.372734546661377,
      "step": 2200,
      "unclipped_grad_norm": 1.11663019657135
    },
    {
      "bits_per_byte": 1.9995734885215308,
      "learning_rate": 0.0002639439261266453,
      "loss_nats": 1.3859987258911133,
      "step": 2210,
      "unclipped_grad_norm": 1.0498319864273071
    },
    {
      "bits_per_byte": 1.8802302555389057,
      "learning_rate": 0.000263563433633716,
      "loss_nats": 1.3032763004302979,
      "step": 2220,
      "unclipped_grad_norm": 1.0160832405090332
    },
    {
      "bits_per_byte": 1.8937652901618087,
      "learning_rate": 0.00026318125696365156,
      "loss_nats": 1.3126580715179443,
      "step": 2230,
      "unclipped_grad_norm": 1.0416548252105713
    },
    {
      "bits_per_byte": 1.8469913405893015,
      "learning_rate": 0.000262797402646797,
      "loss_nats": 1.280236840248108,
      "step": 2240,
      "unclipped_grad_norm": 1.0407441854476929
    },
    {
      "bits_per_byte": 1.8529231942002105,
      "learning_rate": 0.00026241187724216334,
      "loss_nats": 1.284348487854004,
      "step": 2250,
      "unclipped_grad_norm": 0.9867598414421082
    },
    {
      "bits_per_byte": 1.8825788506190881,
      "learning_rate": 0.0002620246873373162,
      "loss_nats": 1.3049042224884033,
      "step": 2260,
      "unclipped_grad_norm": 0.9744513034820557
    },
    {
      "bits_per_byte": 1.9268210435882438,
      "learning_rate": 0.00026163583954826264,
      "loss_nats": 1.3355705738067627,
      "step": 2270,
      "unclipped_grad_norm": 1.0475958585739136
    },
    {
      "bits_per_byte": 1.8280555347813536,
      "learning_rate": 0.0002612453405193386,
      "loss_nats": 1.2671115398406982,
      "step": 2280,
      "unclipped_grad_norm": 1.0294238328933716
    },
    {
      "bits_per_byte": 1.8388550133596329,
      "learning_rate": 0.000260853196923095,
      "loss_nats": 1.27459716796875,
      "step": 2290,
      "unclipped_grad_norm": 1.0671623945236206
    },
    {
      "bits_per_byte": 1.8867002428643644,
      "learning_rate": 0.0002604594154601839,
      "loss_nats": 1.3077609539031982,
      "step": 2300,
      "unclipped_grad_norm": 0.9974833726882935
    },
    {
      "bits_per_byte": 1.8601865374940996,
      "learning_rate": 0.0002600640028592441,
      "loss_nats": 1.289383053779602,
      "step": 2310,
      "unclipped_grad_norm": 1.0144871473312378
    },
    {
      "bits_per_byte": 1.874428248829398,
      "learning_rate": 0.0002596669658767859,
      "loss_nats": 1.2992546558380127,
      "step": 2320,
      "unclipped_grad_norm": 1.0221670866012573
    },
    {
      "bits_per_byte": 1.8178058847378828,
      "learning_rate": 0.000259268311297076,
      "loss_nats": 1.2600070238113403,
      "step": 2330,
      "unclipped_grad_norm": 1.0877128839492798
    },
    {
      "bits_per_byte": 1.8362632348111303,
      "learning_rate": 0.00025886804593202114,
      "loss_nats": 1.2728006839752197,
      "step": 2340,
      "unclipped_grad_norm": 1.050228238105774
    },
    {
      "bits_per_byte": 1.780467247357011,
      "learning_rate": 0.00025846617662105193,
      "loss_nats": 1.2341258525848389,
      "step": 2350,
      "unclipped_grad_norm": 0.9972993731498718
    },
    {
      "bits_per_byte": 1.8791837411084054,
      "learning_rate": 0.00025806271023100637,
      "loss_nats": 1.3025509119033813,
      "step": 2360,
      "unclipped_grad_norm": 1.0690455436706543
    },
    {
      "bits_per_byte": 1.8768631792003134,
      "learning_rate": 0.00025765765365601165,
      "loss_nats": 1.3009424209594727,
      "step": 2370,
      "unclipped_grad_norm": 1.0405195951461792
    },
    {
      "bits_per_byte": 1.8106722163627444,
      "learning_rate": 0.00025725101381736707,
      "loss_nats": 1.2550623416900635,
      "step": 2380,
      "unclipped_grad_norm": 1.000722050666809
    },
    {
      "bits_per_byte": 1.7959209204329396,
      "learning_rate": 0.00025684279766342567,
      "loss_nats": 1.2448375225067139,
      "step": 2390,
      "unclipped_grad_norm": 1.0424110889434814
    },
    {
      "bits_per_byte": 1.9177011475782908,
      "learning_rate": 0.00025643301216947504,
      "loss_nats": 1.3292491436004639,
      "step": 2400,
      "unclipped_grad_norm": 1.0584142208099365
    },
    {
      "bits_per_byte": 1.8792028311826512,
      "learning_rate": 0.00025602166433761866,
      "loss_nats": 1.3025641441345215,
      "step": 2410,
      "unclipped_grad_norm": 1.109039068222046
    },
    {
      "bits_per_byte": 1.9615544877738458,
      "learning_rate": 0.0002556087611966561,
      "loss_nats": 1.359645962715149,
      "step": 2420,
      "unclipped_grad_norm": 1.0733245611190796
    },
    {
      "bits_per_byte": 1.8745142401548294,
      "learning_rate": 0.00025519430980196277,
      "loss_nats": 1.299314260482788,
      "step": 2430,
      "unclipped_grad_norm": 1.0562880039215088
    },
    {
      "bits_per_byte": 1.7973332419618249,
      "learning_rate": 0.0002547783172353695,
      "loss_nats": 1.2458164691925049,
      "step": 2440,
      "unclipped_grad_norm": 1.0554269552230835
    },
    {
      "bits_per_byte": 1.8932479663480133,
      "learning_rate": 0.0002543607906050415,
      "loss_nats": 1.3122994899749756,
      "step": 2450,
      "unclipped_grad_norm": 1.1328672170639038
    },
    {
      "bits_per_byte": 1.9065160838967772,
      "learning_rate": 0.0002539417370453568,
      "loss_nats": 1.3214962482452393,
      "step": 2460,
      "unclipped_grad_norm": 1.1163077354431152
    },
    {
      "bits_per_byte": 1.8460499075584784,
      "learning_rate": 0.00025352116371678464,
      "loss_nats": 1.279584288597107,
      "step": 2470,
      "unclipped_grad_norm": 1.0860918760299683
    },
    {
      "bits_per_byte": 1.8485799443353212,
      "learning_rate": 0.0002530990778057627,
      "loss_nats": 1.2813379764556885,
      "step": 2480,
      "unclipped_grad_norm": 0.99190354347229
    },
    {
      "bits_per_byte": 1.7724624868552525,
      "learning_rate": 0.00025267548652457475,
      "loss_nats": 1.2285773754119873,
      "step": 2490,
      "unclipped_grad_norm": 1.025232195854187
    },
    {
      "bits_per_byte": 1.8512375922391042,
      "learning_rate": 0.00025225039711122696,
      "loss_nats": 1.2831801176071167,
      "step": 2500,
      "unclipped_grad_norm": 1.0759034156799316
    },
    {
      "bits_per_byte": 1.8193396260182773,
      "learning_rate": 0.00025182381682932464,
      "loss_nats": 1.2610701322555542,
      "step": 2510,
      "unclipped_grad_norm": 1.0067415237426758
    },
    {
      "bits_per_byte": 1.8146814759196581,
      "learning_rate": 0.0002513957529679477,
      "loss_nats": 1.2578413486480713,
      "step": 2520,
      "unclipped_grad_norm": 0.990001916885376
    },
    {
      "bits_per_byte": 1.8138671380678228,
      "learning_rate": 0.00025096621284152656,
      "loss_nats": 1.2572768926620483,
      "step": 2530,
      "unclipped_grad_norm": 0.9393356442451477
    },
    {
      "bits_per_byte": 1.8368911434694304,
      "learning_rate": 0.0002505352037897168,
      "loss_nats": 1.2732359170913696,
      "step": 2540,
      "unclipped_grad_norm": 1.1141552925109863
    },
    {
      "bits_per_byte": 1.8603834576293374,
      "learning_rate": 0.0002501027331772739,
      "loss_nats": 1.2895195484161377,
      "step": 2550,
      "unclipped_grad_norm": 1.0569089651107788
    },
    {
      "bits_per_byte": 1.816235855118156,
      "learning_rate": 0.00024966880839392746,
      "loss_nats": 1.2589187622070312,
      "step": 2560,
      "unclipped_grad_norm": 0.9373295903205872
    },
    {
      "bits_per_byte": 1.8059492288047498,
      "learning_rate": 0.0002492334368542548,
      "loss_nats": 1.25178861618042,
      "step": 2570,
      "unclipped_grad_norm": 1.143198847770691
    },
    {
      "bits_per_byte": 1.8450266107858446,
      "learning_rate": 0.00024879662599755416,
      "loss_nats": 1.2788749933242798,
      "step": 2580,
      "unclipped_grad_norm": 1.057027816772461
    },
    {
      "bits_per_byte": 1.8154227211448768,
      "learning_rate": 0.00024835838328771803,
      "loss_nats": 1.2583551406860352,
      "step": 2590,
      "unclipped_grad_norm": 1.0370285511016846
    },
    {
      "bits_per_byte": 1.8561616275159571,
      "learning_rate": 0.0002479187162131051,
      "loss_nats": 1.2865931987762451,
      "step": 2600,
      "unclipped_grad_norm": 0.9653204083442688
    },
    {
      "bits_per_byte": 1.7605767658541238,
      "learning_rate": 0.00024747763228641264,
      "loss_nats": 1.2203388214111328,
      "step": 2610,
      "unclipped_grad_norm": 1.0564771890640259
    },
    {
      "bits_per_byte": 1.812449141111459,
      "learning_rate": 0.00024703513904454795,
      "loss_nats": 1.2562940120697021,
      "step": 2620,
      "unclipped_grad_norm": 1.0267692804336548
    },
    {
      "bits_per_byte": 1.8349191903946374,
      "learning_rate": 0.00024659124404849956,
      "loss_nats": 1.2718690633773804,
      "step": 2630,
      "unclipped_grad_norm": 0.9602619409561157
    },
    {
      "bits_per_byte": 1.8363758834474455,
      "learning_rate": 0.00024614595488320833,
      "loss_nats": 1.2728787660598755,
      "step": 2640,
      "unclipped_grad_norm": 1.0029752254486084
    },
    {
      "bits_per_byte": 1.771385015547597,
      "learning_rate": 0.00024569927915743745,
      "loss_nats": 1.2278305292129517,
      "step": 2650,
      "unclipped_grad_norm": 0.9146384000778198
    },
    {
      "bits_per_byte": 1.779632271587072,
      "learning_rate": 0.0002452512245036427,
      "loss_nats": 1.2335470914840698,
      "step": 2660,
      "unclipped_grad_norm": 0.9666176438331604
    },
    {
      "bits_per_byte": 1.8136729696549987,
      "learning_rate": 0.00024480179857784177,
      "loss_nats": 1.2571423053741455,
      "step": 2670,
      "unclipped_grad_norm": 1.0540763139724731
    },
    {
      "bits_per_byte": 1.7953530337197905,
      "learning_rate": 0.00024435100905948387,
      "loss_nats": 1.2444438934326172,
      "step": 2680,
      "unclipped_grad_norm": 1.0946050882339478
    },
    {
      "bits_per_byte": 1.8769845989518226,
      "learning_rate": 0.00024389886365131793,
      "loss_nats": 1.3010265827178955,
      "step": 2690,
      "unclipped_grad_norm": 1.068537712097168
    },
    {
      "bits_per_byte": 1.8642107595416384,
      "learning_rate": 0.00024344537007926163,
      "loss_nats": 1.2921724319458008,
      "step": 2700,
      "unclipped_grad_norm": 1.1353117227554321
    },
    {
      "bits_per_byte": 1.7364708175959382,
      "learning_rate": 0.00024299053609226878,
      "loss_nats": 1.2036298513412476,
      "step": 2710,
      "unclipped_grad_norm": 0.9047647714614868
    },
    {
      "bits_per_byte": 1.8594279220211436,
      "learning_rate": 0.00024253436946219733,
      "loss_nats": 1.2888572216033936,
      "step": 2720,
      "unclipped_grad_norm": 1.0444105863571167
    },
    {
      "bits_per_byte": 1.7814004252205926,
      "learning_rate": 0.0002420768779836764,
      "loss_nats": 1.2347726821899414,
      "step": 2730,
      "unclipped_grad_norm": 0.9630917310714722
    },
    {
      "bits_per_byte": 1.7840532576101513,
      "learning_rate": 0.00024161806947397305,
      "loss_nats": 1.2366114854812622,
      "step": 2740,
      "unclipped_grad_norm": 1.0223243236541748
    },
    {
      "bits_per_byte": 1.8198583256932797,
      "learning_rate": 0.00024115795177285886,
      "loss_nats": 1.2614296674728394,
      "step": 2750,
      "unclipped_grad_norm": 1.0687127113342285
    },
    {
      "bits_per_byte": 1.9021738659277931,
      "learning_rate": 0.0002406965327424758,
      "loss_nats": 1.3184864521026611,
      "step": 2760,
      "unclipped_grad_norm": 1.0979900360107422
    },
    {
      "bits_per_byte": 1.7971648709466301,
      "learning_rate": 0.00024023382026720197,
      "loss_nats": 1.2456997632980347,
      "step": 2770,
      "unclipped_grad_norm": 0.965108335018158
    },
    {
      "bits_per_byte": 1.8086140999798692,
      "learning_rate": 0.00023976982225351698,
      "loss_nats": 1.2536357641220093,
      "step": 2780,
      "unclipped_grad_norm": 1.0402190685272217
    },
    {
      "bits_per_byte": 1.7938860217079309,
      "learning_rate": 0.0002393045466298666,
      "loss_nats": 1.243427038192749,
      "step": 2790,
      "unclipped_grad_norm": 1.0398236513137817
    },
    {
      "bits_per_byte": 1.7550451158717724,
      "learning_rate": 0.00023883800134652763,
      "loss_nats": 1.2165045738220215,
      "step": 2800,
      "unclipped_grad_norm": 1.0063706636428833
    },
    {
      "bits_per_byte": 1.828329675126829,
      "learning_rate": 0.0002383701943754715,
      "loss_nats": 1.2673015594482422,
      "step": 2810,
      "unclipped_grad_norm": 0.9668177366256714
    },
    {
      "bits_per_byte": 1.8254919613875926,
      "learning_rate": 0.00023790113371022884,
      "loss_nats": 1.2653346061706543,
      "step": 2820,
      "unclipped_grad_norm": 0.9527478814125061
    },
    {
      "bits_per_byte": 1.7516809632382448,
      "learning_rate": 0.0002374308273657521,
      "loss_nats": 1.2141727209091187,
      "step": 2830,
      "unclipped_grad_norm": 0.9219958186149597
    },
    {
      "bits_per_byte": 1.8254823303591443,
      "learning_rate": 0.00023695928337827911,
      "loss_nats": 1.2653279304504395,
      "step": 2840,
      "unclipped_grad_norm": 1.0127615928649902
    },
    {
      "bits_per_byte": 1.7530225998976257,
      "learning_rate": 0.00023648650980519564,
      "loss_nats": 1.2151026725769043,
      "step": 2850,
      "unclipped_grad_norm": 1.0182650089263916
    },
    {
      "bits_per_byte": 1.7287747659524784,
      "learning_rate": 0.00023601251472489757,
      "loss_nats": 1.1982953548431396,
      "step": 2860,
      "unclipped_grad_norm": 0.978410542011261
    },
    {
      "bits_per_byte": 1.855343334063152,
      "learning_rate": 0.00023553730623665305,
      "loss_nats": 1.2860260009765625,
      "step": 2870,
      "unclipped_grad_norm": 1.018862009048462
    },
    {
      "bits_per_byte": 1.768852571013642,
      "learning_rate": 0.00023506089246046403,
      "loss_nats": 1.2260751724243164,
      "step": 2880,
      "unclipped_grad_norm": 0.9639438390731812
    },
    {
      "bits_per_byte": 1.8105554401428086,
      "learning_rate": 0.00023458328153692736,
      "loss_nats": 1.2549813985824585,
      "step": 2890,
      "unclipped_grad_norm": 0.9698073267936707
    },
    {
      "bits_per_byte": 1.7403103302764504,
      "learning_rate": 0.0002341044816270961,
      "loss_nats": 1.2062911987304688,
      "step": 2900,
      "unclipped_grad_norm": 0.9920547604560852
    },
    {
      "bits_per_byte": 1.7643693272709504,
      "learning_rate": 0.00023362450091233953,
      "loss_nats": 1.2229676246643066,
      "step": 2910,
      "unclipped_grad_norm": 1.0128531455993652
    },
    {
      "bits_per_byte": 1.8473500963990013,
      "learning_rate": 0.00023314334759420388,
      "loss_nats": 1.2804855108261108,
      "step": 2920,
      "unclipped_grad_norm": 1.0619620084762573
    },
    {
      "bits_per_byte": 1.8003603085996611,
      "learning_rate": 0.0002326610298942717,
      "loss_nats": 1.2479146718978882,
      "step": 2930,
      "unclipped_grad_norm": 1.0764249563217163
    },
    {
      "bits_per_byte": 1.8426398355571707,
      "learning_rate": 0.00023217755605402175,
      "loss_nats": 1.277220606803894,
      "step": 2940,
      "unclipped_grad_norm": 1.0425148010253906
    },
    {
      "bits_per_byte": 1.7862247105599451,
      "learning_rate": 0.00023169293433468802,
      "loss_nats": 1.2381166219711304,
      "step": 2950,
      "unclipped_grad_norm": 1.004768967628479
    },
    {
      "bits_per_byte": 1.7811238771180051,
      "learning_rate": 0.00023120717301711857,
      "loss_nats": 1.2345809936523438,
      "step": 2960,
      "unclipped_grad_norm": 0.9997458457946777
    },
    {
      "bits_per_byte": 1.8373286673332254,
      "learning_rate": 0.00023072028040163408,
      "loss_nats": 1.2735391855239868,
      "step": 2970,
      "unclipped_grad_norm": 1.039349913597107
    },
    {
      "bits_per_byte": 1.7631166056420657,
      "learning_rate": 0.00023023226480788593,
      "loss_nats": 1.2220993041992188,
      "step": 2980,
      "unclipped_grad_norm": 0.9808847904205322
    },
    {
      "bits_per_byte": 1.9032267437163752,
      "learning_rate": 0.00022974313457471416,
      "loss_nats": 1.319216251373291,
      "step": 2990,
      "unclipped_grad_norm": 1.1331971883773804
    },
    {
      "bits_per_byte": 1.81958332543455,
      "learning_rate": 0.0002292528980600049,
      "loss_nats": 1.2612390518188477,
      "step": 3000,
      "unclipped_grad_norm": 0.944084107875824
    },
    {
      "bits_per_byte": 1.7894560925870064,
      "learning_rate": 0.00022876156364054758,
      "loss_nats": 1.2403564453125,
      "step": 3010,
      "unclipped_grad_norm": 1.057902216911316
    },
    {
      "bits_per_byte": 1.7654951257034983,
      "learning_rate": 0.0002282691397118918,
      "loss_nats": 1.223747968673706,
      "step": 3020,
      "unclipped_grad_norm": 0.971477746963501
    },
    {
      "bits_per_byte": 1.791092679492617,
      "learning_rate": 0.00022777563468820376,
      "loss_nats": 1.2414908409118652,
      "step": 3030,
      "unclipped_grad_norm": 1.017361044883728
    },
    {
      "bits_per_byte": 1.760655361925568,
      "learning_rate": 0.00022728105700212282,
      "loss_nats": 1.2203933000564575,
      "step": 3040,
      "unclipped_grad_norm": 0.9738672971725464
    },
    {
      "bits_per_byte": 1.7859287284178103,
      "learning_rate": 0.00022678541510461698,
      "loss_nats": 1.2379114627838135,
      "step": 3050,
      "unclipped_grad_norm": 1.0311107635498047
    },
    {
      "bits_per_byte": 1.7871233199107035,
      "learning_rate": 0.00022628871746483876,
      "loss_nats": 1.2387394905090332,
      "step": 3060,
      "unclipped_grad_norm": 1.097463607788086
    },
    {
      "bits_per_byte": 1.7530494291911602,
      "learning_rate": 0.0002257909725699804,
      "loss_nats": 1.2151212692260742,
      "step": 3070,
      "unclipped_grad_norm": 0.95722895860672
    },
    {
      "bits_per_byte": 1.7753578147825277,
      "learning_rate": 0.00022529218892512886,
      "loss_nats": 1.2305842638015747,
      "step": 3080,
      "unclipped_grad_norm": 1.0311440229415894
    },
    {
      "bits_per_byte": 1.7406597990230035,
      "learning_rate": 0.00022479237505312034,
      "loss_nats": 1.206533432006836,
      "step": 3090,
      "unclipped_grad_norm": 0.9659063220024109
    },
    {
      "bits_per_byte": 1.8326863396384856,
      "learning_rate": 0.00022429153949439496,
      "loss_nats": 1.2703213691711426,
      "step": 3100,
      "unclipped_grad_norm": 1.0265647172927856
    },
    {
      "bits_per_byte": 1.8128345542320425,
      "learning_rate": 0.00022378969080685053,
      "loss_nats": 1.2565611600875854,
      "step": 3110,
      "unclipped_grad_norm": 1.0288156270980835
    },
    {
      "bits_per_byte": 1.6999466900495526,
      "learning_rate": 0.0002232868375656965,
      "loss_nats": 1.1783132553100586,
      "step": 3120,
      "unclipped_grad_norm": 0.9754294753074646
    },
    {
      "bits_per_byte": 1.7980240962703409,
      "learning_rate": 0.00022278298836330733,
      "loss_nats": 1.2462953329086304,
      "step": 3130,
      "unclipped_grad_norm": 1.0256381034851074
    },
    {
      "bits_per_byte": 1.7337763654048706,
      "learning_rate": 0.00022227815180907572,
      "loss_nats": 1.2017621994018555,
      "step": 3140,
      "unclipped_grad_norm": 0.9683999419212341
    },
    {
      "bits_per_byte": 1.7391683654747212,
      "learning_rate": 0.00022177233652926546,
      "loss_nats": 1.2054996490478516,
      "step": 3150,
      "unclipped_grad_norm": 0.9884417057037354
    },
    {
      "bits_per_byte": 1.7859572775378536,
      "learning_rate": 0.00022126555116686416,
      "loss_nats": 1.237931251525879,
      "step": 3160,
      "unclipped_grad_norm": 1.0163674354553223
    },
    {
      "bits_per_byte": 1.8529599984874952,
      "learning_rate": 0.00022075780438143532,
      "loss_nats": 1.2843739986419678,
      "step": 3170,
      "unclipped_grad_norm": 1.0605453252792358
    },
    {
      "bits_per_byte": 1.7071207743476438,
      "learning_rate": 0.00022024910484897065,
      "loss_nats": 1.1832859516143799,
      "step": 3180,
      "unclipped_grad_norm": 1.0476000308990479
    },
    {
      "bits_per_byte": 1.8011364663030052,
      "learning_rate": 0.00021973946126174158,
      "loss_nats": 1.2484526634216309,
      "step": 3190,
      "unclipped_grad_norm": 0.9400752782821655
    },
    {
      "bits_per_byte": 1.7046130953154133,
      "learning_rate": 0.00021922888232815086,
      "loss_nats": 1.18154776096344,
      "step": 3200,
      "unclipped_grad_norm": 0.9428834915161133
    },
    {
      "bits_per_byte": 1.7343896555378473,
      "learning_rate": 0.0002187173767725837,
      "loss_nats": 1.2021872997283936,
      "step": 3210,
      "unclipped_grad_norm": 0.982049822807312
    },
    {
      "bits_per_byte": 1.7352428614687776,
      "learning_rate": 0.0002182049533352588,
      "loss_nats": 1.202778697013855,
      "step": 3220,
      "unclipped_grad_norm": 0.9146644473075867
    },
    {
      "bits_per_byte": 1.7184315573469382,
      "learning_rate": 0.00021769162077207884,
      "loss_nats": 1.1911259889602661,
      "step": 3230,
      "unclipped_grad_norm": 0.9842981696128845
    },
    {
      "bits_per_byte": 1.7760342225483712,
      "learning_rate": 0.00021717738785448102,
      "loss_nats": 1.231053113937378,
      "step": 3240,
      "unclipped_grad_norm": 1.0283819437026978
    },
    {
      "bits_per_byte": 1.780553754630395,
      "learning_rate": 0.00021666226336928708,
      "loss_nats": 1.234185814857483,
      "step": 3250,
      "unclipped_grad_norm": 0.9948496222496033
    },
    {
      "bits_per_byte": 1.8669033199235465,
      "learning_rate": 0.0002161462561185532,
      "loss_nats": 1.2940387725830078,
      "step": 3260,
      "unclipped_grad_norm": 1.0557905435562134
    },
    {
      "bits_per_byte": 1.7253210103578513,
      "learning_rate": 0.00021562937491941957,
      "loss_nats": 1.1959013938903809,
      "step": 3270,
      "unclipped_grad_norm": 1.0562386512756348
    },
    {
      "bits_per_byte": 1.6636741731213793,
      "learning_rate": 0.00021511162860395975,
      "loss_nats": 1.1531710624694824,
      "step": 3280,
      "unclipped_grad_norm": 0.9674620628356934
    },
    {
      "bits_per_byte": 1.7215127985197962,
      "learning_rate": 0.0002145930260190298,
      "loss_nats": 1.193261742591858,
      "step": 3290,
      "unclipped_grad_norm": 1.081762671470642
    },
    {
      "bits_per_byte": 1.784075099406811,
      "learning_rate": 0.00021407357602611698,
      "loss_nats": 1.2366266250610352,
      "step": 3300,
      "unclipped_grad_norm": 0.9871881604194641
    },
    {
      "bits_per_byte": 1.7790666206483843,
      "learning_rate": 0.0002135532875011885,
      "loss_nats": 1.2331550121307373,
      "step": 3310,
      "unclipped_grad_norm": 1.018639326095581
    },
    {
      "bits_per_byte": 1.7964894950766919,
      "learning_rate": 0.0002130321693345397,
      "loss_nats": 1.2452316284179688,
      "step": 3320,
      "unclipped_grad_norm": 1.0376429557800293
    },
    {
      "bits_per_byte": 1.7442502808250664,
      "learning_rate": 0.00021251023043064224,
      "loss_nats": 1.2090221643447876,
      "step": 3330,
      "unclipped_grad_norm": 0.9751812815666199
    },
    {
      "bits_per_byte": 1.740043757167613,
      "learning_rate": 0.00021198747970799188,
      "loss_nats": 1.206106424331665,
      "step": 3340,
      "unclipped_grad_norm": 1.0885974168777466
    },
    {
      "bits_per_byte": 1.7459623681144056,
      "learning_rate": 0.00021146392609895618,
      "loss_nats": 1.2102088928222656,
      "step": 3350,
      "unclipped_grad_norm": 1.028318166732788
    },
    {
      "bits_per_byte": 1.7533509147781228,
      "learning_rate": 0.00021093957854962178,
      "loss_nats": 1.2153302431106567,
      "step": 3360,
      "unclipped_grad_norm": 1.038756251335144
    },
    {
      "bits_per_byte": 1.6755667734285427,
      "learning_rate": 0.00021041444601964157,
      "loss_nats": 1.161414384841919,
      "step": 3370,
      "unclipped_grad_norm": 1.0451469421386719
    },
    {
      "bits_per_byte": 1.794013632834871,
      "learning_rate": 0.00020988853748208164,
      "loss_nats": 1.2435154914855957,
      "step": 3380,
      "unclipped_grad_norm": 1.1703273057937622
    },
    {
      "bits_per_byte": 1.7554886591283476,
      "learning_rate": 0.00020936186192326782,
      "loss_nats": 1.216812014579773,
      "step": 3390,
      "unclipped_grad_norm": 1.036725401878357
    },
    {
      "bits_per_byte": 1.7445490146896152,
      "learning_rate": 0.00020883442834263225,
      "loss_nats": 1.2092292308807373,
      "step": 3400,
      "unclipped_grad_norm": 1.0196164846420288
    },
    {
      "bits_per_byte": 1.794997717563108,
      "learning_rate": 0.0002083062457525596,
      "loss_nats": 1.2441976070404053,
      "step": 3410,
      "unclipped_grad_norm": 1.202675700187683
    },
    {
      "bits_per_byte": 1.7549782146205868,
      "learning_rate": 0.00020777732317823306,
      "loss_nats": 1.2164582014083862,
      "step": 3420,
      "unclipped_grad_norm": 0.9739375710487366
    },
    {
      "bits_per_byte": 1.7093242160704982,
      "learning_rate": 0.00020724766965748005,
      "loss_nats": 1.1848132610321045,
      "step": 3430,
      "unclipped_grad_norm": 1.0012179613113403
    },
    {
      "bits_per_byte": 1.7178566193451037,
      "learning_rate": 0.00020671729424061788,
      "loss_nats": 1.1907274723052979,
      "step": 3440,
      "unclipped_grad_norm": 1.0250310897827148
    },
    {
      "bits_per_byte": 1.7050728049411694,
      "learning_rate": 0.00020618620599029906,
      "loss_nats": 1.1818664073944092,
      "step": 3450,
      "unclipped_grad_norm": 1.0344855785369873
    },
    {
      "bits_per_byte": 1.706558735044624,
      "learning_rate": 0.00020565441398135649,
      "loss_nats": 1.182896375656128,
      "step": 3460,
      "unclipped_grad_norm": 0.9980728626251221
    },
    {
      "bits_per_byte": 1.7653320861504804,
      "learning_rate": 0.00020512192730064827,
      "loss_nats": 1.223634958267212,
      "step": 3470,
      "unclipped_grad_norm": 1.0550833940505981
    },
    {
      "bits_per_byte": 1.8114688400015408,
      "learning_rate": 0.00020458875504690273,
      "loss_nats": 1.2556145191192627,
      "step": 3480,
      "unclipped_grad_norm": 1.0545322895050049
    },
    {
      "bits_per_byte": 1.740959048835505,
      "learning_rate": 0.00020405490633056245,
      "loss_nats": 1.2067408561706543,
      "step": 3490,
      "unclipped_grad_norm": 0.9756637811660767
    },
    {
      "bits_per_byte": 1.704705106033625,
      "learning_rate": 0.00020352039027362917,
      "loss_nats": 1.1816115379333496,
      "step": 3500,
      "unclipped_grad_norm": 0.9728481769561768
    },
    {
      "bits_per_byte": 1.7154590092094253,
      "learning_rate": 0.0002029852160095075,
      "loss_nats": 1.1890655755996704,
      "step": 3510,
      "unclipped_grad_norm": 0.9984562993049622
    },
    {
      "bits_per_byte": 1.7629102264610303,
      "learning_rate": 0.000202449392682849,
      "loss_nats": 1.2219562530517578,
      "step": 3520,
      "unclipped_grad_norm": 1.0547406673431396
    },
    {
      "bits_per_byte": 1.7070648799861134,
      "learning_rate": 0.0002019129294493959,
      "loss_nats": 1.1832472085952759,
      "step": 3530,
      "unclipped_grad_norm": 0.9733150601387024
    },
    {
      "bits_per_byte": 1.7245068444886669,
      "learning_rate": 0.00020137583547582478,
      "loss_nats": 1.1953370571136475,
      "step": 3540,
      "unclipped_grad_norm": 1.0125547647476196
    },
    {
      "bits_per_byte": 1.7698511022845516,
      "learning_rate": 0.00020083811993958973,
      "loss_nats": 1.2267673015594482,
      "step": 3550,
      "unclipped_grad_norm": 1.0299363136291504
    },
    {
      "bits_per_byte": 1.7181577609667646,
      "learning_rate": 0.00020029979202876563,
      "loss_nats": 1.1909362077713013,
      "step": 3560,
      "unclipped_grad_norm": 0.9789594411849976
    },
    {
      "bits_per_byte": 1.7459088815099875,
      "learning_rate": 0.00019976086094189118,
      "loss_nats": 1.2101718187332153,
      "step": 3570,
      "unclipped_grad_norm": 1.0010499954223633
    },
    {
      "bits_per_byte": 1.7705113436812139,
      "learning_rate": 0.00019922133588781175,
      "loss_nats": 1.2272249460220337,
      "step": 3580,
      "unclipped_grad_norm": 1.082968831062317
    },
    {
      "bits_per_byte": 1.7425258107848651,
      "learning_rate": 0.0001986812260855219,
      "loss_nats": 1.207826852798462,
      "step": 3590,
      "unclipped_grad_norm": 1.0055863857269287
    },
    {
      "bits_per_byte": 1.7755588625013865,
      "learning_rate": 0.00019814054076400787,
      "loss_nats": 1.2307236194610596,
      "step": 3600,
      "unclipped_grad_norm": 1.0473418235778809
    },
    {
      "bits_per_byte": 1.7693408297594415,
      "learning_rate": 0.00019759928916209004,
      "loss_nats": 1.226413607597351,
      "step": 3610,
      "unclipped_grad_norm": 1.1090856790542603
    },
    {
      "bits_per_byte": 1.7112244523798812,
      "learning_rate": 0.00019705748052826493,
      "loss_nats": 1.186130404472351,
      "step": 3620,
      "unclipped_grad_norm": 0.9742531776428223
    },
    {
      "bits_per_byte": 1.7137644641504741,
      "learning_rate": 0.00019651512412054723,
      "loss_nats": 1.1878910064697266,
      "step": 3630,
      "unclipped_grad_norm": 1.0429282188415527
    },
    {
      "bits_per_byte": 1.6520834023664803,
      "learning_rate": 0.00019597222920631144,
      "loss_nats": 1.1451369524002075,
      "step": 3640,
      "unclipped_grad_norm": 0.9745917320251465
    },
    {
      "bits_per_byte": 1.7256807980634563,
      "learning_rate": 0.0001954288050621338,
      "loss_nats": 1.196150779724121,
      "step": 3650,
      "unclipped_grad_norm": 1.0385748147964478
    },
    {
      "bits_per_byte": 1.7126971398192194,
      "learning_rate": 0.00019488486097363363,
      "loss_nats": 1.1871511936187744,
      "step": 3660,
      "unclipped_grad_norm": 0.9310101270675659
    },
    {
      "bits_per_byte": 1.7674923602279682,
      "learning_rate": 0.00019434040623531452,
      "loss_nats": 1.2251323461532593,
      "step": 3670,
      "unclipped_grad_norm": 1.0780647993087769
    },
    {
      "bits_per_byte": 1.6993364956042913,
      "learning_rate": 0.0001937954501504058,
      "loss_nats": 1.1778903007507324,
      "step": 3680,
      "unclipped_grad_norm": 0.9666725397109985
    },
    {
      "bits_per_byte": 1.7697671747509305,
      "learning_rate": 0.00019325000203070333,
      "loss_nats": 1.2267091274261475,
      "step": 3690,
      "unclipped_grad_norm": 1.0202223062515259
    },
    {
      "bits_per_byte": 1.6641650116069417,
      "learning_rate": 0.0001927040711964106,
      "loss_nats": 1.1535112857818604,
      "step": 3700,
      "unclipped_grad_norm": 1.0115612745285034
    },
    {
      "bits_per_byte": 1.726182815421325,
      "learning_rate": 0.00019215766697597923,
      "loss_nats": 1.1964987516403198,
      "step": 3710,
      "unclipped_grad_norm": 0.9808560013771057
    },
    {
      "bits_per_byte": 1.7616059100368868,
      "learning_rate": 0.00019161079870594978,
      "loss_nats": 1.2210521697998047,
      "step": 3720,
      "unclipped_grad_norm": 1.0511683225631714
    },
    {
      "bits_per_byte": 1.7329405297216773,
      "learning_rate": 0.0001910634757307922,
      "loss_nats": 1.2011828422546387,
      "step": 3730,
      "unclipped_grad_norm": 1.0690499544143677
    },
    {
      "bits_per_byte": 1.7000292417219667,
      "learning_rate": 0.0001905157074027459,
      "loss_nats": 1.178370475769043,
      "step": 3740,
      "unclipped_grad_norm": 0.9935287237167358
    },
    {
      "bits_per_byte": 1.7278594742845865,
      "learning_rate": 0.00018996750308166037,
      "loss_nats": 1.1976609230041504,
      "step": 3750,
      "unclipped_grad_norm": 1.010900855064392
    },
    {
      "bits_per_byte": 1.743004610484867,
      "learning_rate": 0.00018941887213483482,
      "loss_nats": 1.2081587314605713,
      "step": 3760,
      "unclipped_grad_norm": 1.0119491815567017
    },
    {
      "bits_per_byte": 1.751619221466585,
      "learning_rate": 0.00018886982393685845,
      "loss_nats": 1.21412992477417,
      "step": 3770,
      "unclipped_grad_norm": 1.011305570602417
    },
    {
      "bits_per_byte": 1.8012363882231563,
      "learning_rate": 0.00018832036786945008,
      "loss_nats": 1.2485219240188599,
      "step": 3780,
      "unclipped_grad_norm": 1.0745822191238403
    },
    {
      "bits_per_byte": 1.698407961272283,
      "learning_rate": 0.0001877705133212979,
      "loss_nats": 1.1772466897964478,
      "step": 3790,
      "unclipped_grad_norm": 1.0157029628753662
    },
    {
      "bits_per_byte": 1.7299170747195092,
      "learning_rate": 0.00018722026968789906,
      "loss_nats": 1.199087142944336,
      "step": 3800,
      "unclipped_grad_norm": 1.0643014907836914
    },
    {
      "bits_per_byte": 1.6600432753963636,
      "learning_rate": 0.00018666964637139907,
      "loss_nats": 1.1506543159484863,
      "step": 3810,
      "unclipped_grad_norm": 0.9955329895019531
    },
    {
      "bits_per_byte": 1.661928377232471,
      "learning_rate": 0.00018611865278043115,
      "loss_nats": 1.1519609689712524,
      "step": 3820,
      "unclipped_grad_norm": 1.0222501754760742
    },
    {
      "bits_per_byte": 1.7482129330835965,
      "learning_rate": 0.00018556729832995555,
      "loss_nats": 1.2117688655853271,
      "step": 3830,
      "unclipped_grad_norm": 1.0880961418151855
    },
    {
      "bits_per_byte": 1.6885180989344173,
      "learning_rate": 0.00018501559244109864,
      "loss_nats": 1.17039155960083,
      "step": 3840,
      "unclipped_grad_norm": 0.9980700016021729
    },
    {
      "bits_per_byte": 1.661674014891845,
      "learning_rate": 0.00018446354454099178,
      "loss_nats": 1.1517846584320068,
      "step": 3850,
      "unclipped_grad_norm": 0.961553156375885
    },
    {
      "bits_per_byte": 1.6409110654011305,
      "learning_rate": 0.00018391116406261048,
      "loss_nats": 1.1373928785324097,
      "step": 3860,
      "unclipped_grad_norm": 1.019968032836914
    },
    {
      "bits_per_byte": 1.6069470715954888,
      "learning_rate": 0.00018335846044461305,
      "loss_nats": 1.1138508319854736,
      "step": 3870,
      "unclipped_grad_norm": 0.9610836505889893
    },
    {
      "bits_per_byte": 1.7034871249002144,
      "learning_rate": 0.0001828054431311793,
      "loss_nats": 1.180767297744751,
      "step": 3880,
      "unclipped_grad_norm": 0.9531551003456116
    },
    {
      "bits_per_byte": 1.6934515932570677,
      "learning_rate": 0.00018225212157184934,
      "loss_nats": 1.1738111972808838,
      "step": 3890,
      "unclipped_grad_norm": 1.0675544738769531
    },
    {
      "bits_per_byte": 1.6830007235120874,
      "learning_rate": 0.00018169850522136192,
      "loss_nats": 1.1665672063827515,
      "step": 3900,
      "unclipped_grad_norm": 1.0058032274246216
    },
    {
      "bits_per_byte": 1.7034874688655162,
      "learning_rate": 0.000181144603539493,
      "loss_nats": 1.18076753616333,
      "step": 3910,
      "unclipped_grad_norm": 0.9943428039550781
    },
    {
      "bits_per_byte": 1.743468619676895,
      "learning_rate": 0.00018059042599089412,
      "loss_nats": 1.2084803581237793,
      "step": 3920,
      "unclipped_grad_norm": 1.2279266119003296
    },
    {
      "bits_per_byte": 1.721944302990811,
      "learning_rate": 0.00018003598204493048,
      "loss_nats": 1.1935608386993408,
      "step": 3930,
      "unclipped_grad_norm": 1.008062720298767
    },
    {
      "bits_per_byte": 1.6512047430032224,
      "learning_rate": 0.00017948128117551936,
      "loss_nats": 1.1445279121398926,
      "step": 3940,
      "unclipped_grad_norm": 0.9797955751419067
    },
    {
      "bits_per_byte": 1.750776850442659,
      "learning_rate": 0.00017892633286096816,
      "loss_nats": 1.2135460376739502,
      "step": 3950,
      "unclipped_grad_norm": 1.0508832931518555
    },
    {
      "bits_per_byte": 1.7346926889686676,
      "learning_rate": 0.00017837114658381247,
      "loss_nats": 1.202397346496582,
      "step": 3960,
      "unclipped_grad_norm": 1.0876473188400269
    },
    {
      "bits_per_byte": 1.6890646597988592,
      "learning_rate": 0.00017781573183065386,
      "loss_nats": 1.1707704067230225,
      "step": 3970,
      "unclipped_grad_norm": 1.0190049409866333
    },
    {
      "bits_per_byte": 1.6854375456921626,
      "learning_rate": 0.0001772600980919981,
      "loss_nats": 1.1682562828063965,
      "step": 3980,
      "unclipped_grad_norm": 1.0094109773635864
    },
    {
      "bits_per_byte": 1.710786068602832,
      "learning_rate": 0.0001767042548620927,
      "loss_nats": 1.1858265399932861,
      "step": 3990,
      "unclipped_grad_norm": 1.082020878791809
    },
    {
      "bits_per_byte": 1.6440261871562087,
      "learning_rate": 0.00017614821163876486,
      "loss_nats": 1.139552116394043,
      "step": 4000,
      "unclipped_grad_norm": 0.9749544262886047
    },
    {
      "bits_per_byte": 1.706837174956371,
      "learning_rate": 0.00017559197792325906,
      "loss_nats": 1.1830893754959106,
      "step": 4010,
      "unclipped_grad_norm": 1.1478910446166992
    },
    {
      "bits_per_byte": 1.7296654641012967,
      "learning_rate": 0.0001750355632200748,
      "loss_nats": 1.1989127397537231,
      "step": 4020,
      "unclipped_grad_norm": 1.0259206295013428
    },
    {
      "bits_per_byte": 1.6863480218458304,
      "learning_rate": 0.00017447897703680418,
      "loss_nats": 1.1688873767852783,
      "step": 4030,
      "unclipped_grad_norm": 1.0554102659225464
    },
    {
      "bits_per_byte": 1.6602591136231966,
      "learning_rate": 0.0001739222288839694,
      "loss_nats": 1.1508039236068726,
      "step": 4040,
      "unclipped_grad_norm": 1.00386643409729
    },
    {
      "bits_per_byte": 1.6813582891963477,
      "learning_rate": 0.00017336532827486026,
      "loss_nats": 1.1654287576675415,
      "step": 4050,
      "unclipped_grad_norm": 0.9927709698677063
    },
    {
      "bits_per_byte": 1.6720621109692604,
      "learning_rate": 0.00017280828472537163,
      "loss_nats": 1.1589851379394531,
      "step": 4060,
      "unclipped_grad_norm": 1.0185434818267822
    },
    {
      "bits_per_byte": 1.7963911210003984,
      "learning_rate": 0.0001722511077538408,
      "loss_nats": 1.2451634407043457,
      "step": 4070,
      "unclipped_grad_norm": 1.0195153951644897
    },
    {
      "bits_per_byte": 1.6278679011596953,
      "learning_rate": 0.00017169380688088497,
      "loss_nats": 1.1283520460128784,
      "step": 4080,
      "unclipped_grad_norm": 0.9588671326637268
    },
    {
      "bits_per_byte": 1.6697190193339053,
      "learning_rate": 0.0001711363916292384,
      "loss_nats": 1.1573610305786133,
      "step": 4090,
      "unclipped_grad_norm": 1.0382872819900513
    },
    {
      "bits_per_byte": 1.786499366853373,
      "learning_rate": 0.00017057887152358974,
      "loss_nats": 1.238306999206543,
      "step": 4100,
      "unclipped_grad_norm": 1.0833542346954346
    },
    {
      "bits_per_byte": 1.6685632959201073,
      "learning_rate": 0.0001700212560904194,
      "loss_nats": 1.156559944152832,
      "step": 4110,
      "unclipped_grad_norm": 1.0042530298233032
    },
    {
      "bits_per_byte": 1.6813281922324468,
      "learning_rate": 0.00016946355485783656,
      "loss_nats": 1.1654078960418701,
      "step": 4120,
      "unclipped_grad_norm": 1.0854146480560303
    },
    {
      "bits_per_byte": 1.7532864212840493,
      "learning_rate": 0.0001689057773554166,
      "loss_nats": 1.2152855396270752,
      "step": 4130,
      "unclipped_grad_norm": 1.0470608472824097
    },
    {
      "bits_per_byte": 1.7313932018118645,
      "learning_rate": 0.00016834793311403806,
      "loss_nats": 1.2001103162765503,
      "step": 4140,
      "unclipped_grad_norm": 1.0418859720230103
    },
    {
      "bits_per_byte": 1.6694201134867057,
      "learning_rate": 0.00016779003166571981,
      "loss_nats": 1.157153844833374,
      "step": 4150,
      "unclipped_grad_norm": 1.0297493934631348
    },
    {
      "bits_per_byte": 1.6896791538103921,
      "learning_rate": 0.00016723208254345838,
      "loss_nats": 1.1711963415145874,
      "step": 4160,
      "unclipped_grad_norm": 1.0267698764801025
    },
    {
      "bits_per_byte": 1.7146273011098527,
      "learning_rate": 0.00016667409528106481,
      "loss_nats": 1.1884890794754028,
      "step": 4170,
      "unclipped_grad_norm": 1.1191542148590088
    },
    {
      "bits_per_byte": 1.6875347021367837,
      "learning_rate": 0.00016611607941300184,
      "loss_nats": 1.1697099208831787,
      "step": 4180,
      "unclipped_grad_norm": 1.0307927131652832
    },
    {
      "bits_per_byte": 1.7039927538937512,
      "learning_rate": 0.0001655580444742211,
      "loss_nats": 1.1811177730560303,
      "step": 4190,
      "unclipped_grad_norm": 1.089546799659729
    },
    {
      "bits_per_byte": 1.7112641803722306,
      "learning_rate": 0.00016499999999999997,
      "loss_nats": 1.1861579418182373,
      "step": 4200,
      "unclipped_grad_norm": 1.0752745866775513
    },
    {
      "bits_per_byte": 1.7401058429045744,
      "learning_rate": 0.0001644419555257789,
      "loss_nats": 1.2061494588851929,
      "step": 4210,
      "unclipped_grad_norm": 1.0523611307144165
    },
    {
      "bits_per_byte": 1.693671731050172,
      "learning_rate": 0.0001638839205869981,
      "loss_nats": 1.1739637851715088,
      "step": 4220,
      "unclipped_grad_norm": 0.997297465801239
    },
    {
      "bits_per_byte": 1.7078159282224312,
      "learning_rate": 0.0001633259047189352,
      "loss_nats": 1.1837677955627441,
      "step": 4230,
      "unclipped_grad_norm": 1.0999088287353516
    },
    {
      "bits_per_byte": 1.7184320732948908,
      "learning_rate": 0.0001627679174565416,
      "loss_nats": 1.1911263465881348,
      "step": 4240,
      "unclipped_grad_norm": 0.9742965698242188
    },
    {
      "bits_per_byte": 1.6537339198668108,
      "learning_rate": 0.00016220996833428016,
      "loss_nats": 1.1462810039520264,
      "step": 4250,
      "unclipped_grad_norm": 0.9919567108154297
    },
    {
      "bits_per_byte": 1.6890791063415318,
      "learning_rate": 0.00016165206688596191,
      "loss_nats": 1.1707804203033447,
      "step": 4260,
      "unclipped_grad_norm": 1.0240992307662964
    },
    {
      "bits_per_byte": 1.6052090149258693,
      "learning_rate": 0.00016109422264458335,
      "loss_nats": 1.1126461029052734,
      "step": 4270,
      "unclipped_grad_norm": 0.9614320993423462
    },
    {
      "bits_per_byte": 1.677325124050964,
      "learning_rate": 0.00016053644514216336,
      "loss_nats": 1.1626331806182861,
      "step": 4280,
      "unclipped_grad_norm": 1.0358916521072388
    },
    {
      "bits_per_byte": 1.6570518091672561,
      "learning_rate": 0.00015997874390958058,
      "loss_nats": 1.14858078956604,
      "step": 4290,
      "unclipped_grad_norm": 1.0068124532699585
    },
    {
      "bits_per_byte": 1.6268774530733763,
      "learning_rate": 0.00015942112847641018,
      "loss_nats": 1.1276655197143555,
      "step": 4300,
      "unclipped_grad_norm": 1.0985108613967896
    },
    {
      "bits_per_byte": 1.6144661530912108,
      "learning_rate": 0.00015886360837076156,
      "loss_nats": 1.1190626621246338,
      "step": 4310,
      "unclipped_grad_norm": 0.9936105608940125
    },
    {
      "bits_per_byte": 1.7397008237617926,
      "learning_rate": 0.00015830619311911495,
      "loss_nats": 1.2058687210083008,
      "step": 4320,
      "unclipped_grad_norm": 1.1129765510559082
    },
    {
      "bits_per_byte": 1.6818943591190871,
      "learning_rate": 0.00015774889224615915,
      "loss_nats": 1.1658003330230713,
      "step": 4330,
      "unclipped_grad_norm": 1.0247257947921753
    },
    {
      "bits_per_byte": 1.6456326770979182,
      "learning_rate": 0.00015719171527462834,
      "loss_nats": 1.1406656503677368,
      "step": 4340,
      "unclipped_grad_norm": 1.0460948944091797
    },
    {
      "bits_per_byte": 1.6091236840248087,
      "learning_rate": 0.0001566346717251397,
      "loss_nats": 1.1153595447540283,
      "step": 4350,
      "unclipped_grad_norm": 1.0163395404815674
    },
    {
      "bits_per_byte": 1.686970427059303,
      "learning_rate": 0.00015607777111603056,
      "loss_nats": 1.1693187952041626,
      "step": 4360,
      "unclipped_grad_norm": 1.063033938407898
    },
    {
      "bits_per_byte": 1.65933092325649,
      "learning_rate": 0.00015552102296319576,
      "loss_nats": 1.150160551071167,
      "step": 4370,
      "unclipped_grad_norm": 0.9919126033782959
    },
    {
      "bits_per_byte": 1.6024796502566765,
      "learning_rate": 0.00015496443677992516,
      "loss_nats": 1.1107542514801025,
      "step": 4380,
      "unclipped_grad_norm": 1.0177253484725952
    },
    {
      "bits_per_byte": 1.7322169987094975,
      "learning_rate": 0.0001544080220767409,
      "loss_nats": 1.2006813287734985,
      "step": 4390,
      "unclipped_grad_norm": 0.9770697951316833
    },
    {
      "bits_per_byte": 1.6952770171133256,
      "learning_rate": 0.00015385178836123511,
      "loss_nats": 1.1750764846801758,
      "step": 4400,
      "unclipped_grad_norm": 1.0351864099502563
    },
    {
      "bits_per_byte": 1.6960017520040616,
      "learning_rate": 0.00015329574513790726,
      "loss_nats": 1.1755788326263428,
      "step": 4410,
      "unclipped_grad_norm": 1.1030160188674927
    },
    {
      "bits_per_byte": 1.7127384156554266,
      "learning_rate": 0.00015273990190800186,
      "loss_nats": 1.1871798038482666,
      "step": 4420,
      "unclipped_grad_norm": 1.0353890657424927
    },
    {
      "bits_per_byte": 1.6726172709662455,
      "learning_rate": 0.0001521842681693461,
      "loss_nats": 1.159369945526123,
      "step": 4430,
      "unclipped_grad_norm": 1.0875210762023926
    },
    {
      "bits_per_byte": 1.680560461678995,
      "learning_rate": 0.00015162885341618752,
      "loss_nats": 1.1648757457733154,
      "step": 4440,
      "unclipped_grad_norm": 0.9928231239318848
    },
    {
      "bits_per_byte": 1.6836972532480818,
      "learning_rate": 0.00015107366713903178,
      "loss_nats": 1.1670500040054321,
      "step": 4450,
      "unclipped_grad_norm": 0.9873722791671753
    },
    {
      "bits_per_byte": 1.6829482688035744,
      "learning_rate": 0.0001505187188244806,
      "loss_nats": 1.1665308475494385,
      "step": 4460,
      "unclipped_grad_norm": 1.0991053581237793
    },
    {
      "bits_per_byte": 1.6608082542274016,
      "learning_rate": 0.00014996401795506952,
      "loss_nats": 1.1511845588684082,
      "step": 4470,
      "unclipped_grad_norm": 1.0593136548995972
    },
    {
      "bits_per_byte": 1.698860447626703,
      "learning_rate": 0.00014940957400910585,
      "loss_nats": 1.1775603294372559,
      "step": 4480,
      "unclipped_grad_norm": 1.0394423007965088
    },
    {
      "bits_per_byte": 1.740063707155113,
      "learning_rate": 0.00014885539646050699,
      "loss_nats": 1.206120252609253,
      "step": 4490,
      "unclipped_grad_norm": 1.0491414070129395
    },
    {
      "bits_per_byte": 1.6564844384020596,
      "learning_rate": 0.00014830149477863805,
      "loss_nats": 1.148187518119812,
      "step": 4500,
      "unclipped_grad_norm": 1.0240522623062134
    },
    {
      "bits_per_byte": 1.6424668204608355,
      "learning_rate": 0.00014774787842815066,
      "loss_nats": 1.138471245765686,
      "step": 4510,
      "unclipped_grad_norm": 1.025274634361267
    },
    {
      "bits_per_byte": 1.71951625192593,
      "learning_rate": 0.00014719455686882066,
      "loss_nats": 1.191877841949463,
      "step": 4520,
      "unclipped_grad_norm": 1.063368320465088
    },
    {
      "bits_per_byte": 1.680697359869082,
      "learning_rate": 0.00014664153955538695,
      "loss_nats": 1.1649706363677979,
      "step": 4530,
      "unclipped_grad_norm": 1.0728412866592407
    },
    {
      "bits_per_byte": 1.6916681331676204,
      "learning_rate": 0.00014608883593738946,
      "loss_nats": 1.1725749969482422,
      "step": 4540,
      "unclipped_grad_norm": 1.1416125297546387
    },
    {
      "bits_per_byte": 1.6884355472620032,
      "learning_rate": 0.00014553645545900817,
      "loss_nats": 1.1703343391418457,
      "step": 4550,
      "unclipped_grad_norm": 1.0188941955566406
    },
    {
      "bits_per_byte": 1.6889325771229966,
      "learning_rate": 0.0001449844075589013,
      "loss_nats": 1.1706788539886475,
      "step": 4560,
      "unclipped_grad_norm": 1.0978926420211792
    },
    {
      "bits_per_byte": 1.6867642198609183,
      "learning_rate": 0.0001444327016700444,
      "loss_nats": 1.1691758632659912,
      "step": 4570,
      "unclipped_grad_norm": 1.043805480003357
    },
    {
      "bits_per_byte": 1.6647294586670736,
      "learning_rate": 0.0001438813472195688,
      "loss_nats": 1.153902530670166,
      "step": 4580,
      "unclipped_grad_norm": 1.061089277267456
    },
    {
      "bits_per_byte": 1.6460370083100966,
      "learning_rate": 0.00014333035362860093,
      "loss_nats": 1.1409459114074707,
      "step": 4590,
      "unclipped_grad_norm": 1.0722659826278687
    },
    {
      "bits_per_byte": 1.6445638049228057,
      "learning_rate": 0.00014277973031210088,
      "loss_nats": 1.1399247646331787,
      "step": 4600,
      "unclipped_grad_norm": 1.0390986204147339
    },
    {
      "bits_per_byte": 1.6450002968906956,
      "learning_rate": 0.00014222948667870207,
      "loss_nats": 1.1402273178100586,
      "step": 4610,
      "unclipped_grad_norm": 1.0198664665222168
    },
    {
      "bits_per_byte": 1.7089066421942032,
      "learning_rate": 0.00014167963213054991,
      "loss_nats": 1.1845238208770752,
      "step": 4620,
      "unclipped_grad_norm": 1.0599243640899658
    },
    {
      "bits_per_byte": 1.69811834248823,
      "learning_rate": 0.00014113017606314152,
      "loss_nats": 1.1770459413528442,
      "step": 4630,
      "unclipped_grad_norm": 1.1346435546875
    },
    {
      "bits_per_byte": 1.7044230544862098,
      "learning_rate": 0.00014058112786516518,
      "loss_nats": 1.1814160346984863,
      "step": 4640,
      "unclipped_grad_norm": 1.0656381845474243
    },
    {
      "bits_per_byte": 1.7338580571640303,
      "learning_rate": 0.0001400324969183396,
      "loss_nats": 1.201818823814392,
      "step": 4650,
      "unclipped_grad_norm": 1.0118787288665771
    },
    {
      "bits_per_byte": 1.652913906587497,
      "learning_rate": 0.00013948429259725406,
      "loss_nats": 1.1457126140594482,
      "step": 4660,
      "unclipped_grad_norm": 1.0275835990905762
    },
    {
      "bits_per_byte": 1.66807348933045,
      "learning_rate": 0.00013893652426920776,
      "loss_nats": 1.1562204360961914,
      "step": 4670,
      "unclipped_grad_norm": 1.0584690570831299
    },
    {
      "bits_per_byte": 1.6481818039490068,
      "learning_rate": 0.00013838920129405016,
      "loss_nats": 1.1424325704574585,
      "step": 4680,
      "unclipped_grad_norm": 1.1420236825942993
    },
    {
      "bits_per_byte": 1.658637833173513,
      "learning_rate": 0.0001378423330240207,
      "loss_nats": 1.1496801376342773,
      "step": 4690,
      "unclipped_grad_norm": 1.080548644065857
    },
    {
      "bits_per_byte": 1.6754130209386713,
      "learning_rate": 0.00013729592880358936,
      "loss_nats": 1.1613078117370605,
      "step": 4700,
      "unclipped_grad_norm": 1.063712239265442
    },
    {
      "bits_per_byte": 1.6541807307937524,
      "learning_rate": 0.00013674999796929659,
      "loss_nats": 1.1465907096862793,
      "step": 4710,
      "unclipped_grad_norm": 1.065613865852356
    },
    {
      "bits_per_byte": 1.6715305125954434,
      "learning_rate": 0.00013620454984959417,
      "loss_nats": 1.1586166620254517,
      "step": 4720,
      "unclipped_grad_norm": 1.041472315788269
    },
    {
      "bits_per_byte": 1.7163307892666488,
      "learning_rate": 0.00013565959376468542,
      "loss_nats": 1.1896698474884033,
      "step": 4730,
      "unclipped_grad_norm": 1.0371977090835571
    },
    {
      "bits_per_byte": 1.6468645888260485,
      "learning_rate": 0.00013511513902636634,
      "loss_nats": 1.141519546508789,
      "step": 4740,
      "unclipped_grad_norm": 1.118260383605957
    },
    {
      "bits_per_byte": 1.6331114802018514,
      "learning_rate": 0.0001345711949378662,
      "loss_nats": 1.1319866180419922,
      "step": 4750,
      "unclipped_grad_norm": 1.1252490282058716
    },
    {
      "bits_per_byte": 1.6043413624522664,
      "learning_rate": 0.00013402777079368853,
      "loss_nats": 1.1120446920394897,
      "step": 4760,
      "unclipped_grad_norm": 1.0504546165466309
    },
    {
      "bits_per_byte": 1.6005166402797284,
      "learning_rate": 0.0001334848758794528,
      "loss_nats": 1.10939359664917,
      "step": 4770,
      "unclipped_grad_norm": 0.984404981136322
    },
    {
      "bits_per_byte": 1.6404936635074865,
      "learning_rate": 0.00013294251947173504,
      "loss_nats": 1.13710355758667,
      "step": 4780,
      "unclipped_grad_norm": 1.0672088861465454
    },
    {
      "bits_per_byte": 1.6438352864137509,
      "learning_rate": 0.00013240071083790996,
      "loss_nats": 1.1394197940826416,
      "step": 4790,
      "unclipped_grad_norm": 1.0523202419281006
    },
    {
      "bits_per_byte": 1.6719128300283113,
      "learning_rate": 0.0001318594592359921,
      "loss_nats": 1.158881664276123,
      "step": 4800,
      "unclipped_grad_norm": 1.1081420183181763
    },
    {
      "bits_per_byte": 1.6392536685947656,
      "learning_rate": 0.0001313187739144781,
      "loss_nats": 1.1362440586090088,
      "step": 4810,
      "unclipped_grad_norm": 1.0507419109344482
    },
    {
      "bits_per_byte": 1.648858383697501,
      "learning_rate": 0.0001307786641121882,
      "loss_nats": 1.1429015398025513,
      "step": 4820,
      "unclipped_grad_norm": 1.1054445505142212
    },
    {
      "bits_per_byte": 1.6256828615804833,
      "learning_rate": 0.00013023913905810876,
      "loss_nats": 1.1268374919891357,
      "step": 4830,
      "unclipped_grad_norm": 1.0212544202804565
    },
    {
      "bits_per_byte": 1.5951660880287355,
      "learning_rate": 0.00012970020797123432,
      "loss_nats": 1.1056848764419556,
      "step": 4840,
      "unclipped_grad_norm": 1.0648442506790161
    },
    {
      "bits_per_byte": 1.6564994008926848,
      "learning_rate": 0.00012916188006041027,
      "loss_nats": 1.148197889328003,
      "step": 4850,
      "unclipped_grad_norm": 1.047589898109436
    },
    {
      "bits_per_byte": 1.641553764567405,
      "learning_rate": 0.00012862416452417516,
      "loss_nats": 1.137838363647461,
      "step": 4860,
      "unclipped_grad_norm": 1.0177857875823975
    },
    {
      "bits_per_byte": 1.6551041056462348,
      "learning_rate": 0.00012808707055060405,
      "loss_nats": 1.1472307443618774,
      "step": 4870,
      "unclipped_grad_norm": 1.0283644199371338
    },
    {
      "bits_per_byte": 1.6180980827121316,
      "learning_rate": 0.000127550607317151,
      "loss_nats": 1.1215801239013672,
      "step": 4880,
      "unclipped_grad_norm": 1.0304375886917114
    },
    {
      "bits_per_byte": 1.5903827345602883,
      "learning_rate": 0.00012701478399049248,
      "loss_nats": 1.1023693084716797,
      "step": 4890,
      "unclipped_grad_norm": 1.0921995639801025
    },
    {
      "bits_per_byte": 1.567706994026678,
      "learning_rate": 0.0001264796097263708,
      "loss_nats": 1.0866516828536987,
      "step": 4900,
      "unclipped_grad_norm": 1.0604755878448486
    },
    {
      "bits_per_byte": 1.6439632415059928,
      "learning_rate": 0.0001259450936694375,
      "loss_nats": 1.1395084857940674,
      "step": 4910,
      "unclipped_grad_norm": 1.0503017902374268
    },
    {
      "bits_per_byte": 1.642120275419347,
      "learning_rate": 0.00012541124495309727,
      "loss_nats": 1.1382310390472412,
      "step": 4920,
      "unclipped_grad_norm": 1.1088591814041138
    },
    {
      "bits_per_byte": 1.622944209848144,
      "learning_rate": 0.00012487807269935167,
      "loss_nats": 1.124939203262329,
      "step": 4930,
      "unclipped_grad_norm": 1.099297046661377
    },
    {
      "bits_per_byte": 1.6933755769253864,
      "learning_rate": 0.00012434558601864349,
      "loss_nats": 1.1737585067749023,
      "step": 4940,
      "unclipped_grad_norm": 1.08061945438385
    },
    {
      "bits_per_byte": 1.5670388414280758,
      "learning_rate": 0.0001238137940097009,
      "loss_nats": 1.086188554763794,
      "step": 4950,
      "unclipped_grad_norm": 1.1187851428985596
    },
    {
      "bits_per_byte": 1.6577792957804058,
      "learning_rate": 0.0001232827057593821,
      "loss_nats": 1.1490850448608398,
      "step": 4960,
      "unclipped_grad_norm": 1.095150351524353
    },
    {
      "bits_per_byte": 1.7221861105979241,
      "learning_rate": 0.00012275233034251992,
      "loss_nats": 1.1937284469604492,
      "step": 4970,
      "unclipped_grad_norm": 1.2121388912200928
    },
    {
      "bits_per_byte": 1.5930121773093298,
      "learning_rate": 0.0001222226768217669,
      "loss_nats": 1.1041918992996216,
      "step": 4980,
      "unclipped_grad_norm": 1.1045044660568237
    },
    {
      "bits_per_byte": 1.665033523993799,
      "learning_rate": 0.00012169375424744034,
      "loss_nats": 1.1541132926940918,
      "step": 4990,
      "unclipped_grad_norm": 1.1089035272598267
    },
    {
      "bits_per_byte": 1.6784994215910551,
      "learning_rate": 0.00012116557165736772,
      "loss_nats": 1.1634471416473389,
      "step": 5000,
      "unclipped_grad_norm": 1.0390889644622803
    },
    {
      "bits_per_byte": 1.642436207548982,
      "learning_rate": 0.00012063813807673214,
      "loss_nats": 1.138450026512146,
      "step": 5010,
      "unclipped_grad_norm": 1.1114050149917603
    },
    {
      "bits_per_byte": 1.6556807634745778,
      "learning_rate": 0.00012011146251791832,
      "loss_nats": 1.1476304531097412,
      "step": 5020,
      "unclipped_grad_norm": 1.0220290422439575
    },
    {
      "bits_per_byte": 1.636998288111351,
      "learning_rate": 0.0001195855539803584,
      "loss_nats": 1.1346807479858398,
      "step": 5030,
      "unclipped_grad_norm": 1.1270323991775513
    },
    {
      "bits_per_byte": 1.6052772920382619,
      "learning_rate": 0.00011906042145037817,
      "loss_nats": 1.112693428993225,
      "step": 5040,
      "unclipped_grad_norm": 1.009588360786438
    },
    {
      "bits_per_byte": 1.6583182894082098,
      "learning_rate": 0.0001185360739010438,
      "loss_nats": 1.149458646774292,
      "step": 5050,
      "unclipped_grad_norm": 1.060039758682251
    },
    {
      "bits_per_byte": 1.609498090255737,
      "learning_rate": 0.00011801252029200808,
      "loss_nats": 1.1156190633773804,
      "step": 5060,
      "unclipped_grad_norm": 1.0725975036621094
    },
    {
      "bits_per_byte": 1.6927082842400387,
      "learning_rate": 0.00011748976956935777,
      "loss_nats": 1.1732959747314453,
      "step": 5070,
      "unclipped_grad_norm": 1.120245099067688
    },
    {
      "bits_per_byte": 1.690623166580978,
      "learning_rate": 0.00011696783066546024,
      "loss_nats": 1.1718506813049316,
      "step": 5080,
      "unclipped_grad_norm": 1.1778441667556763
    },
    {
      "bits_per_byte": 1.67411111227164,
      "learning_rate": 0.00011644671249881147,
      "loss_nats": 1.1604053974151611,
      "step": 5090,
      "unclipped_grad_norm": 1.3122999668121338
    },
    {
      "bits_per_byte": 1.5883206625764432,
      "learning_rate": 0.00011592642397388293,
      "loss_nats": 1.1009399890899658,
      "step": 5100,
      "unclipped_grad_norm": 1.0812361240386963
    },
    {
      "bits_per_byte": 1.6343635139001327,
      "learning_rate": 0.00011540697398097015,
      "loss_nats": 1.1328544616699219,
      "step": 5110,
      "unclipped_grad_norm": 1.054042935371399
    },
    {
      "bits_per_byte": 1.6096652573923755,
      "learning_rate": 0.00011488837139604016,
      "loss_nats": 1.1157349348068237,
      "step": 5120,
      "unclipped_grad_norm": 1.183487892150879
    },
    {
      "bits_per_byte": 1.6704048861455463,
      "learning_rate": 0.00011437062508058039,
      "loss_nats": 1.1578364372253418,
      "step": 5130,
      "unclipped_grad_norm": 1.1154927015304565
    },
    {
      "bits_per_byte": 1.6086259662332116,
      "learning_rate": 0.00011385374388144673,
      "loss_nats": 1.1150145530700684,
      "step": 5140,
      "unclipped_grad_norm": 1.1280518770217896
    },
    {
      "bits_per_byte": 1.607841897327928,
      "learning_rate": 0.00011333773663071286,
      "loss_nats": 1.1144710779190063,
      "step": 5150,
      "unclipped_grad_norm": 1.1053173542022705
    },
    {
      "bits_per_byte": 1.6377722100402334,
      "learning_rate": 0.00011282261214551897,
      "loss_nats": 1.1352171897888184,
      "step": 5160,
      "unclipped_grad_norm": 1.0772578716278076
    },
    {
      "bits_per_byte": 1.6387738369988585,
      "learning_rate": 0.00011230837922792112,
      "loss_nats": 1.135911464691162,
      "step": 5170,
      "unclipped_grad_norm": 1.08511221408844
    },
    {
      "bits_per_byte": 1.6731652076918944,
      "learning_rate": 0.0001117950466647412,
      "loss_nats": 1.1597497463226318,
      "step": 5180,
      "unclipped_grad_norm": 1.0842925310134888
    },
    {
      "bits_per_byte": 1.6308702022958073,
      "learning_rate": 0.00011128262322741627,
      "loss_nats": 1.1304330825805664,
      "step": 5190,
      "unclipped_grad_norm": 1.1379977464675903
    },
    {
      "bits_per_byte": 1.5988843529403896,
      "learning_rate": 0.00011077111767184913,
      "loss_nats": 1.1082621812820435,
      "step": 5200,
      "unclipped_grad_norm": 1.1327811479568481
    },
    {
      "bits_per_byte": 1.6538405491103456,
      "learning_rate": 0.00011026053873825837,
      "loss_nats": 1.1463549137115479,
      "step": 5210,
      "unclipped_grad_norm": 1.131410837173462
    },
    {
      "bits_per_byte": 1.6579122383695226,
      "learning_rate": 0.0001097508951510293,
      "loss_nats": 1.1491771936416626,
      "step": 5220,
      "unclipped_grad_norm": 1.111109972000122
    },
    {
      "bits_per_byte": 1.6745703059494434,
      "learning_rate": 0.0001092421956185646,
      "loss_nats": 1.1607236862182617,
      "step": 5230,
      "unclipped_grad_norm": 1.1052273511886597
    },
    {
      "bits_per_byte": 1.6223493218588094,
      "learning_rate": 0.00010873444883313581,
      "loss_nats": 1.124526858329773,
      "step": 5240,
      "unclipped_grad_norm": 1.0814346075057983
    },
    {
      "bits_per_byte": 1.5820224859191963,
      "learning_rate": 0.00010822766347073447,
      "loss_nats": 1.0965744256973267,
      "step": 5250,
      "unclipped_grad_norm": 1.0832712650299072
    },
    {
      "bits_per_byte": 1.5670176875620196,
      "learning_rate": 0.00010772184819092424,
      "loss_nats": 1.0861738920211792,
      "step": 5260,
      "unclipped_grad_norm": 1.0310815572738647
    },
    {
      "bits_per_byte": 1.6681894056371316,
      "learning_rate": 0.0001072170116366926,
      "loss_nats": 1.1563007831573486,
      "step": 5270,
      "unclipped_grad_norm": 1.1380984783172607
    },
    {
      "bits_per_byte": 1.6634579909292448,
      "learning_rate": 0.00010671316243430345,
      "loss_nats": 1.153021216392517,
      "step": 5280,
      "unclipped_grad_norm": 1.1410281658172607
    },
    {
      "bits_per_byte": 1.627283848077365,
      "learning_rate": 0.00010621030919314944,
      "loss_nats": 1.127947211265564,
      "step": 5290,
      "unclipped_grad_norm": 1.0713688135147095
    },
    {
      "bits_per_byte": 1.57530983107337,
      "learning_rate": 0.000105708460505605,
      "loss_nats": 1.0919215679168701,
      "step": 5300,
      "unclipped_grad_norm": 1.103183388710022
    },
    {
      "bits_per_byte": 1.6444668067077193,
      "learning_rate": 0.00010520762494687964,
      "loss_nats": 1.139857530593872,
      "step": 5310,
      "unclipped_grad_norm": 1.1310855150222778
    },
    {
      "bits_per_byte": 1.6034007893346978,
      "learning_rate": 0.00010470781107487111,
      "loss_nats": 1.1113927364349365,
      "step": 5320,
      "unclipped_grad_norm": 1.0775246620178223
    },
    {
      "bits_per_byte": 1.6848084331553064,
      "learning_rate": 0.00010420902743001955,
      "loss_nats": 1.1678202152252197,
      "step": 5330,
      "unclipped_grad_norm": 1.1329472064971924
    },
    {
      "bits_per_byte": 1.5891140185448733,
      "learning_rate": 0.00010371128253516117,
      "loss_nats": 1.1014899015426636,
      "step": 5340,
      "unclipped_grad_norm": 1.0905873775482178
    },
    {
      "bits_per_byte": 1.6324792719772796,
      "learning_rate": 0.000103214584895383,
      "loss_nats": 1.1315484046936035,
      "step": 5350,
      "unclipped_grad_norm": 1.13417649269104
    },
    {
      "bits_per_byte": 1.556942083961222,
      "learning_rate": 0.00010271894299787715,
      "loss_nats": 1.0791900157928467,
      "step": 5360,
      "unclipped_grad_norm": 1.11483633518219
    },
    {
      "bits_per_byte": 1.585282417066301,
      "learning_rate": 0.00010222436531179622,
      "loss_nats": 1.0988340377807617,
      "step": 5370,
      "unclipped_grad_norm": 1.0698448419570923
    },
    {
      "bits_per_byte": 1.6547109533063624,
      "learning_rate": 0.0001017308602881082,
      "loss_nats": 1.1469582319259644,
      "step": 5380,
      "unclipped_grad_norm": 1.1219947338104248
    },
    {
      "bits_per_byte": 1.5562904416971028,
      "learning_rate": 0.00010123843635945242,
      "loss_nats": 1.0787383317947388,
      "step": 5390,
      "unclipped_grad_norm": 1.115169644355774
    },
    {
      "bits_per_byte": 1.6265400231123834,
      "learning_rate": 0.00010074710193999505,
      "loss_nats": 1.1274316310882568,
      "step": 5400,
      "unclipped_grad_norm": 1.079226016998291
    },
    {
      "bits_per_byte": 1.5897746039068374,
      "learning_rate": 0.0001002568654252858,
      "loss_nats": 1.1019477844238281,
      "step": 5410,
      "unclipped_grad_norm": 1.0649707317352295
    },
    {
      "bits_per_byte": 1.5705084194265821,
      "learning_rate": 9.976773519211404e-05,
      "loss_nats": 1.0885934829711914,
      "step": 5420,
      "unclipped_grad_norm": 1.115608811378479
    },
    {
      "bits_per_byte": 1.6107165873370999,
      "learning_rate": 9.92797195983659e-05,
      "loss_nats": 1.1164636611938477,
      "step": 5430,
      "unclipped_grad_norm": 1.1144897937774658
    },
    {
      "bits_per_byte": 1.5986757379848928,
      "learning_rate": 9.879282698288142e-05,
      "loss_nats": 1.1081175804138184,
      "step": 5440,
      "unclipped_grad_norm": 1.0857055187225342
    },
    {
      "bits_per_byte": 1.5965995634236771,
      "learning_rate": 9.830706566531197e-05,
      "loss_nats": 1.1066784858703613,
      "step": 5450,
      "unclipped_grad_norm": 1.0530359745025635
    },
    {
      "bits_per_byte": 1.5899876904112564,
      "learning_rate": 9.782244394597826e-05,
      "loss_nats": 1.1020954847335815,
      "step": 5460,
      "unclipped_grad_norm": 1.1017006635665894
    },
    {
      "bits_per_byte": 1.611714258694755,
      "learning_rate": 9.733897010572829e-05,
      "loss_nats": 1.1171551942825317,
      "step": 5470,
      "unclipped_grad_norm": 1.11431884765625
    },
    {
      "bits_per_byte": 1.5834313677950644,
      "learning_rate": 9.685665240579612e-05,
      "loss_nats": 1.0975509881973267,
      "step": 5480,
      "unclipped_grad_norm": 1.0543001890182495
    },
    {
      "bits_per_byte": 1.6635541292310771,
      "learning_rate": 9.637549908766041e-05,
      "loss_nats": 1.153087854385376,
      "step": 5490,
      "unclipped_grad_norm": 1.154997706413269
    },
    {
      "bits_per_byte": 1.62576197359988,
      "learning_rate": 9.589551837290389e-05,
      "loss_nats": 1.126892328262329,
      "step": 5500,
      "unclipped_grad_norm": 1.1008871793746948
    },
    {
      "bits_per_byte": 1.5806631350467766,
      "learning_rate": 9.541671846307257e-05,
      "loss_nats": 1.0956321954727173,
      "step": 5510,
      "unclipped_grad_norm": 1.0707919597625732
    },
    {
      "bits_per_byte": 1.5578388015028208,
      "learning_rate": 9.493910753953597e-05,
      "loss_nats": 1.0798115730285645,
      "step": 5520,
      "unclipped_grad_norm": 1.0223692655563354
    },
    {
      "bits_per_byte": 1.573020741990386,
      "learning_rate": 9.446269376334689e-05,
      "loss_nats": 1.0903348922729492,
      "step": 5530,
      "unclipped_grad_norm": 1.1420265436172485
    },
    {
      "bits_per_byte": 1.6833484724321321,
      "learning_rate": 9.39874852751024e-05,
      "loss_nats": 1.1668082475662231,
      "step": 5540,
      "unclipped_grad_norm": 1.1607446670532227
    },
    {
      "bits_per_byte": 1.6178791487975832,
      "learning_rate": 9.351349019480433e-05,
      "loss_nats": 1.121428370475769,
      "step": 5550,
      "unclipped_grad_norm": 1.1443098783493042
    },
    {
      "bits_per_byte": 1.6447739677221602,
      "learning_rate": 9.304071662172086e-05,
      "loss_nats": 1.1400704383850098,
      "step": 5560,
      "unclipped_grad_norm": 1.1718472242355347
    },
    {
      "bits_per_byte": 1.59615963180277,
      "learning_rate": 9.256917263424787e-05,
      "loss_nats": 1.1063735485076904,
      "step": 5570,
      "unclipped_grad_norm": 1.093506097793579
    },
    {
      "bits_per_byte": 1.624393163681663,
      "learning_rate": 9.209886628977114e-05,
      "loss_nats": 1.1259435415267944,
      "step": 5580,
      "unclipped_grad_norm": 1.0647783279418945
    },
    {
      "bits_per_byte": 1.6292150412639035,
      "learning_rate": 9.162980562452843e-05,
      "loss_nats": 1.1292858123779297,
      "step": 5590,
      "unclipped_grad_norm": 1.1650937795639038
    },
    {
      "bits_per_byte": 1.6101528282075717,
      "learning_rate": 9.116199865347238e-05,
      "loss_nats": 1.1160728931427002,
      "step": 5600,
      "unclipped_grad_norm": 1.1434698104858398
    },
    {
      "bits_per_byte": 1.5665055232277503,
      "learning_rate": 9.069545337013333e-05,
      "loss_nats": 1.085818886756897,
      "step": 5610,
      "unclipped_grad_norm": 1.1304930448532104
    },
    {
      "bits_per_byte": 1.6202220684502873,
      "learning_rate": 9.0230177746483e-05,
      "loss_nats": 1.1230523586273193,
      "step": 5620,
      "unclipped_grad_norm": 1.1116455793380737
    },
    {
      "bits_per_byte": 1.538340096478599,
      "learning_rate": 8.976617973279797e-05,
      "loss_nats": 1.066296100616455,
      "step": 5630,
      "unclipped_grad_norm": 1.058684229850769
    },
    {
      "bits_per_byte": 1.5357409226761094,
      "learning_rate": 8.930346725752419e-05,
      "loss_nats": 1.0644944906234741,
      "step": 5640,
      "unclipped_grad_norm": 1.033333420753479
    },
    {
      "bits_per_byte": 1.6143326945541414,
      "learning_rate": 8.88420482271411e-05,
      "loss_nats": 1.1189701557159424,
      "step": 5650,
      "unclipped_grad_norm": 1.1008256673812866
    },
    {
      "bits_per_byte": 1.5570710709493691,
      "learning_rate": 8.838193052602691e-05,
      "loss_nats": 1.0792794227600098,
      "step": 5660,
      "unclipped_grad_norm": 1.1397353410720825
    },
    {
      "bits_per_byte": 1.6224767610030988,
      "learning_rate": 8.792312201632352e-05,
      "loss_nats": 1.12461519241333,
      "step": 5670,
      "unclipped_grad_norm": 1.1256861686706543
    },
    {
      "bits_per_byte": 1.6278986860541997,
      "learning_rate": 8.746563053780261e-05,
      "loss_nats": 1.128373384475708,
      "step": 5680,
      "unclipped_grad_norm": 1.2497283220291138
    },
    {
      "bits_per_byte": 1.6402167714395974,
      "learning_rate": 8.700946390773114e-05,
      "loss_nats": 1.1369116306304932,
      "step": 5690,
      "unclipped_grad_norm": 1.099176049232483
    },
    {
      "bits_per_byte": 1.6346018818542285,
      "learning_rate": 8.655462992073833e-05,
      "loss_nats": 1.1330196857452393,
      "step": 5700,
      "unclipped_grad_norm": 1.132989525794983
    },
    {
      "bits_per_byte": 1.6142821316547877,
      "learning_rate": 8.610113634868199e-05,
      "loss_nats": 1.1189351081848145,
      "step": 5710,
      "unclipped_grad_norm": 1.169878363609314
    },
    {
      "bits_per_byte": 1.6389509791292471,
      "learning_rate": 8.564899094051613e-05,
      "loss_nats": 1.1360342502593994,
      "step": 5720,
      "unclipped_grad_norm": 1.1713305711746216
    },
    {
      "bits_per_byte": 1.5731861893005161,
      "learning_rate": 8.519820142215816e-05,
      "loss_nats": 1.090449571609497,
      "step": 5730,
      "unclipped_grad_norm": 1.0991312265396118
    },
    {
      "bits_per_byte": 1.642317195554585,
      "learning_rate": 8.474877549635728e-05,
      "loss_nats": 1.1383675336837769,
      "step": 5740,
      "unclipped_grad_norm": 1.074776530265808
    },
    {
      "bits_per_byte": 1.5380594207923908,
      "learning_rate": 8.430072084256247e-05,
      "loss_nats": 1.0661015510559082,
      "step": 5750,
      "unclipped_grad_norm": 1.1045373678207397
    },
    {
      "bits_per_byte": 1.574528857855802,
      "learning_rate": 8.385404511679161e-05,
      "loss_nats": 1.09138023853302,
      "step": 5760,
      "unclipped_grad_norm": 1.1376982927322388
    },
    {
      "bits_per_byte": 1.5630001728378644,
      "learning_rate": 8.340875595150034e-05,
      "loss_nats": 1.083389163017273,
      "step": 5770,
      "unclipped_grad_norm": 1.1196208000183105
    },
    {
      "bits_per_byte": 1.658872245526639,
      "learning_rate": 8.296486095545203e-05,
      "loss_nats": 1.149842619895935,
      "step": 5780,
      "unclipped_grad_norm": 1.1231145858764648
    },
    {
      "bits_per_byte": 1.4960170579104213,
      "learning_rate": 8.252236771358729e-05,
      "loss_nats": 1.0369600057601929,
      "step": 5790,
      "unclipped_grad_norm": 1.1306540966033936
    },
    {
      "bits_per_byte": 1.632525363327711,
      "learning_rate": 8.208128378689484e-05,
      "loss_nats": 1.1315803527832031,
      "step": 5800,
      "unclipped_grad_norm": 1.1228681802749634
    },
    {
      "bits_per_byte": 1.5271961366506415,
      "learning_rate": 8.16416167122819e-05,
      "loss_nats": 1.058571696281433,
      "step": 5810,
      "unclipped_grad_norm": 1.0998179912567139
    },
    {
      "bits_per_byte": 1.544303422914616,
      "learning_rate": 8.120337400244578e-05,
      "loss_nats": 1.0704295635223389,
      "step": 5820,
      "unclipped_grad_norm": 1.1354378461837769
    },
    {
      "bits_per_byte": 1.5973088198758354,
      "learning_rate": 8.076656314574521e-05,
      "loss_nats": 1.1071701049804688,
      "step": 5830,
      "unclipped_grad_norm": 1.112725853919983
    },
    {
      "bits_per_byte": 1.5755839714188453,
      "learning_rate": 8.03311916060725e-05,
      "loss_nats": 1.092111587524414,
      "step": 5840,
      "unclipped_grad_norm": 1.105390191078186
    },
    {
      "bits_per_byte": 1.551858620767019,
      "learning_rate": 7.989726682272606e-05,
      "loss_nats": 1.0756664276123047,
      "step": 5850,
      "unclipped_grad_norm": 1.0705221891403198
    },
    {
      "bits_per_byte": 1.58110633433805,
      "learning_rate": 7.946479621028318e-05,
      "loss_nats": 1.0959393978118896,
      "step": 5860,
      "unclipped_grad_norm": 1.1176000833511353
    },
    {
      "bits_per_byte": 1.6256357383341469,
      "learning_rate": 7.903378715847343e-05,
      "loss_nats": 1.1268048286437988,
      "step": 5870,
      "unclipped_grad_norm": 1.1718623638153076
    },
    {
      "bits_per_byte": 1.5672180473502748,
      "learning_rate": 7.860424703205224e-05,
      "loss_nats": 1.0863127708435059,
      "step": 5880,
      "unclipped_grad_norm": 1.1114801168441772
    },
    {
      "bits_per_byte": 1.6374564498932493,
      "learning_rate": 7.817618317067534e-05,
      "loss_nats": 1.1349983215332031,
      "step": 5890,
      "unclipped_grad_norm": 1.2559460401535034
    },
    {
      "bits_per_byte": 1.6402117839427224,
      "learning_rate": 7.774960288877295e-05,
      "loss_nats": 1.1369081735610962,
      "step": 5900,
      "unclipped_grad_norm": 1.1818214654922485
    },
    {
      "bits_per_byte": 1.5692250848858438,
      "learning_rate": 7.732451347542522e-05,
      "loss_nats": 1.0877039432525635,
      "step": 5910,
      "unclipped_grad_norm": 1.0690149068832397
    },
    {
      "bits_per_byte": 1.5417845650100792,
      "learning_rate": 7.690092219423722e-05,
      "loss_nats": 1.0686836242675781,
      "step": 5920,
      "unclipped_grad_norm": 1.1236122846603394
    },
    {
      "bits_per_byte": 1.6662781624380931,
      "learning_rate": 7.647883628321536e-05,
      "loss_nats": 1.1549760103225708,
      "step": 5930,
      "unclipped_grad_norm": 1.2468876838684082
    },
    {
      "bits_per_byte": 1.596861664983592,
      "learning_rate": 7.605826295464316e-05,
      "loss_nats": 1.1068601608276367,
      "step": 5940,
      "unclipped_grad_norm": 1.2251354455947876
    },
    {
      "bits_per_byte": 1.5910101272706358,
      "learning_rate": 7.563920939495851e-05,
      "loss_nats": 1.102804183959961,
      "step": 5950,
      "unclipped_grad_norm": 1.11275053024292
    },
    {
      "bits_per_byte": 1.5416916943786134,
      "learning_rate": 7.522168276463047e-05,
      "loss_nats": 1.0686192512512207,
      "step": 5960,
      "unclipped_grad_norm": 1.1575579643249512
    },
    {
      "bits_per_byte": 1.5659371205666486,
      "learning_rate": 7.480569019803719e-05,
      "loss_nats": 1.0854249000549316,
      "step": 5970,
      "unclipped_grad_norm": 1.102811336517334
    },
    {
      "bits_per_byte": 1.5505245513442762,
      "learning_rate": 7.439123880334388e-05,
      "loss_nats": 1.0747417211532593,
      "step": 5980,
      "unclipped_grad_norm": 1.1679552793502808
    },
    {
      "bits_per_byte": 1.5609080038901184,
      "learning_rate": 7.39783356623813e-05,
      "loss_nats": 1.0819389820098877,
      "step": 5990,
      "unclipped_grad_norm": 1.1159348487854004
    },
    {
      "bits_per_byte": 1.6522259759840456,
      "learning_rate": 7.356698783052497e-05,
      "loss_nats": 1.1452357769012451,
      "step": 6000,
      "unclipped_grad_norm": 1.1946688890457153
    },
    {
      "bits_per_byte": 1.610257393659296,
      "learning_rate": 7.315720233657431e-05,
      "loss_nats": 1.116145372390747,
      "step": 6010,
      "unclipped_grad_norm": 1.2001700401306152
    },
    {
      "bits_per_byte": 1.5550891428808262,
      "learning_rate": 7.274898618263286e-05,
      "loss_nats": 1.0779056549072266,
      "step": 6020,
      "unclipped_grad_norm": 1.11931574344635
    },
    {
      "bits_per_byte": 1.5522581364649735,
      "learning_rate": 7.234234634398831e-05,
      "loss_nats": 1.0759433507919312,
      "step": 6030,
      "unclipped_grad_norm": 1.0901384353637695
    },
    {
      "bits_per_byte": 1.5778765001548465,
      "learning_rate": 7.193728976899362e-05,
      "loss_nats": 1.093700647354126,
      "step": 6040,
      "unclipped_grad_norm": 1.1465646028518677
    },
    {
      "bits_per_byte": 1.585459043248737,
      "learning_rate": 7.153382337894797e-05,
      "loss_nats": 1.0989564657211304,
      "step": 6050,
      "unclipped_grad_norm": 1.093497395515442
    },
    {
      "bits_per_byte": 1.580860743112618,
      "learning_rate": 7.113195406797887e-05,
      "loss_nats": 1.0957691669464111,
      "step": 6060,
      "unclipped_grad_norm": 1.1618996858596802
    },
    {
      "bits_per_byte": 1.4692863104347675,
      "learning_rate": 7.073168870292397e-05,
      "loss_nats": 1.0184316635131836,
      "step": 6070,
      "unclipped_grad_norm": 1.095646619796753
    },
    {
      "bits_per_byte": 1.5593341906520728,
      "learning_rate": 7.033303412321404e-05,
      "loss_nats": 1.0808480978012085,
      "step": 6080,
      "unclipped_grad_norm": 1.2226048707962036
    },
    {
      "bits_per_byte": 1.5548665973306097,
      "learning_rate": 6.993599714075586e-05,
      "loss_nats": 1.0777513980865479,
      "step": 6090,
      "unclipped_grad_norm": 1.160715103149414
    },
    {
      "bits_per_byte": 1.6437169623499572,
      "learning_rate": 6.954058453981609e-05,
      "loss_nats": 1.1393377780914307,
      "step": 6100,
      "unclipped_grad_norm": 1.216336965560913
    },
    {
      "bits_per_byte": 1.5953482176559992,
      "learning_rate": 6.914680307690504e-05,
      "loss_nats": 1.1058111190795898,
      "step": 6110,
      "unclipped_grad_norm": 1.1638948917388916
    },
    {
      "bits_per_byte": 1.5315409343593887,
      "learning_rate": 6.875465948066139e-05,
      "loss_nats": 1.0615832805633545,
      "step": 6120,
      "unclipped_grad_norm": 1.1416045427322388
    },
    {
      "bits_per_byte": 1.6181004904692438,
      "learning_rate": 6.836416045173733e-05,
      "loss_nats": 1.121581792831421,
      "step": 6130,
      "unclipped_grad_norm": 1.1373443603515625
    },
    {
      "bits_per_byte": 1.5591868015202834,
      "learning_rate": 6.797531266268377e-05,
      "loss_nats": 1.0807459354400635,
      "step": 6140,
      "unclipped_grad_norm": 1.0817177295684814
    },
    {
      "bits_per_byte": 1.5489426549216403,
      "learning_rate": 6.758812275783663e-05,
      "loss_nats": 1.0736452341079712,
      "step": 6150,
      "unclipped_grad_norm": 1.156706690788269
    },
    {
      "bits_per_byte": 1.6171114182441317,
      "learning_rate": 6.720259735320298e-05,
      "loss_nats": 1.1208962202072144,
      "step": 6160,
      "unclipped_grad_norm": 1.2064051628112793
    },
    {
      "bits_per_byte": 1.5388670513208427,
      "learning_rate": 6.681874303634842e-05,
      "loss_nats": 1.0666613578796387,
      "step": 6170,
      "unclipped_grad_norm": 1.0822783708572388
    },
    {
      "bits_per_byte": 1.5490690621700245,
      "learning_rate": 6.643656636628402e-05,
      "loss_nats": 1.073732852935791,
      "step": 6180,
      "unclipped_grad_norm": 1.109446406364441
    },
    {
      "bits_per_byte": 1.5625440748477764,
      "learning_rate": 6.605607387335466e-05,
      "loss_nats": 1.0830730199813843,
      "step": 6190,
      "unclipped_grad_norm": 1.1676242351531982
    },
    {
      "bits_per_byte": 1.5465116801516945,
      "learning_rate": 6.567727205912724e-05,
      "loss_nats": 1.071960210800171,
      "step": 6200,
      "unclipped_grad_norm": 1.094281792640686
    },
    {
      "bits_per_byte": 1.5807004552820139,
      "learning_rate": 6.530016739627959e-05,
      "loss_nats": 1.0956580638885498,
      "step": 6210,
      "unclipped_grad_norm": 1.2013437747955322
    },
    {
      "bits_per_byte": 1.592342648849521,
      "learning_rate": 6.492476632848993e-05,
      "loss_nats": 1.1037278175354004,
      "step": 6220,
      "unclipped_grad_norm": 1.1159899234771729
    },
    {
      "bits_per_byte": 1.550343453612918,
      "learning_rate": 6.455107527032678e-05,
      "loss_nats": 1.0746161937713623,
      "step": 6230,
      "unclipped_grad_norm": 1.156030535697937
    },
    {
      "bits_per_byte": 1.5936101609863798,
      "learning_rate": 6.417910060713926e-05,
      "loss_nats": 1.1046063899993896,
      "step": 6240,
      "unclipped_grad_norm": 1.2167954444885254
    },
    {
      "bits_per_byte": 1.5477754086702344,
      "learning_rate": 6.380884869494799e-05,
      "loss_nats": 1.07283616065979,
      "step": 6250,
      "unclipped_grad_norm": 1.1424623727798462
    },
    {
      "bits_per_byte": 1.5145149958892723,
      "learning_rate": 6.34403258603367e-05,
      "loss_nats": 1.0497817993164062,
      "step": 6260,
      "unclipped_grad_norm": 1.159928798675537
    },
    {
      "bits_per_byte": 1.558105374611658,
      "learning_rate": 6.30735384003437e-05,
      "loss_nats": 1.0799963474273682,
      "step": 6270,
      "unclipped_grad_norm": 1.1427968740463257
    },
    {
      "bits_per_byte": 1.5868821996846267,
      "learning_rate": 6.270849258235484e-05,
      "loss_nats": 1.099942922592163,
      "step": 6280,
      "unclipped_grad_norm": 1.2086946964263916
    },
    {
      "bits_per_byte": 1.5572024656946284,
      "learning_rate": 6.234519464399581e-05,
      "loss_nats": 1.0793704986572266,
      "step": 6290,
      "unclipped_grad_norm": 1.26358163356781
    },
    {
      "bits_per_byte": 1.525056844456559,
      "learning_rate": 6.198365079302614e-05,
      "loss_nats": 1.057088851928711,
      "step": 6300,
      "unclipped_grad_norm": 1.1142038106918335
    },
    {
      "bits_per_byte": 1.585295831713068,
      "learning_rate": 6.16238672072326e-05,
      "loss_nats": 1.0988433361053467,
      "step": 6310,
      "unclipped_grad_norm": 1.202506422996521
    },
    {
      "bits_per_byte": 1.5589988244828903,
      "learning_rate": 6.126585003432406e-05,
      "loss_nats": 1.0806156396865845,
      "step": 6320,
      "unclipped_grad_norm": 1.1080162525177002
    },
    {
      "bits_per_byte": 1.5721742433828394,
      "learning_rate": 6.090960539182614e-05,
      "loss_nats": 1.0897481441497803,
      "step": 6330,
      "unclipped_grad_norm": 1.1463266611099243
    },
    {
      "bits_per_byte": 1.4897746036320538,
      "learning_rate": 6.0555139366976935e-05,
      "loss_nats": 1.0326330661773682,
      "step": 6340,
      "unclipped_grad_norm": 1.0811859369277954
    },
    {
      "bits_per_byte": 1.5979289893148465,
      "learning_rate": 6.020245801662274e-05,
      "loss_nats": 1.1075999736785889,
      "step": 6350,
      "unclipped_grad_norm": 1.1469974517822266
    },
    {
      "bits_per_byte": 1.6121533304024078,
      "learning_rate": 5.985156736711483e-05,
      "loss_nats": 1.1174595355987549,
      "step": 6360,
      "unclipped_grad_norm": 1.2663604021072388
    },
    {
      "bits_per_byte": 1.522211219515383,
      "learning_rate": 5.950247341420618e-05,
      "loss_nats": 1.0551164150238037,
      "step": 6370,
      "unclipped_grad_norm": 1.136039137840271
    },
    {
      "bits_per_byte": 1.5209657211578345,
      "learning_rate": 5.915518212294935e-05,
      "loss_nats": 1.054253101348877,
      "step": 6380,
      "unclipped_grad_norm": 1.1138842105865479
    },
    {
      "bits_per_byte": 1.6340888576067047,
      "learning_rate": 5.880969942759425e-05,
      "loss_nats": 1.1326640844345093,
      "step": 6390,
      "unclipped_grad_norm": 1.227125883102417
    },
    {
      "bits_per_byte": 1.5932569086215076,
      "learning_rate": 5.846603123148688e-05,
      "loss_nats": 1.1043615341186523,
      "step": 6400,
      "unclipped_grad_norm": 1.1935430765151978
    },
    {
      "bits_per_byte": 1.6221133616618257,
      "learning_rate": 5.812418340696858e-05,
      "loss_nats": 1.1243633031845093,
      "step": 6410,
      "unclipped_grad_norm": 1.289966106414795
    },
    {
      "bits_per_byte": 1.5341647016809516,
      "learning_rate": 5.778416179527541e-05,
      "loss_nats": 1.0634019374847412,
      "step": 6420,
      "unclipped_grad_norm": 1.2202966213226318
    },
    {
      "bits_per_byte": 1.5828500664351481,
      "learning_rate": 5.7445972206438635e-05,
      "loss_nats": 1.097148060798645,
      "step": 6430,
      "unclipped_grad_norm": 1.1407660245895386
    },
    {
      "bits_per_byte": 1.518648426920109,
      "learning_rate": 5.710962041918516e-05,
      "loss_nats": 1.0526468753814697,
      "step": 6440,
      "unclipped_grad_norm": 1.1754008531570435
    },
    {
      "bits_per_byte": 1.5227140967865058,
      "learning_rate": 5.677511218083908e-05,
      "loss_nats": 1.0554649829864502,
      "step": 6450,
      "unclipped_grad_norm": 1.1674573421478271
    },
    {
      "bits_per_byte": 1.5506523344538674,
      "learning_rate": 5.644245320722316e-05,
      "loss_nats": 1.0748302936553955,
      "step": 6460,
      "unclipped_grad_norm": 1.1624079942703247
    },
    {
      "bits_per_byte": 1.5647150118496176,
      "learning_rate": 5.61116491825615e-05,
      "loss_nats": 1.0845777988433838,
      "step": 6470,
      "unclipped_grad_norm": 1.1915463209152222
    },
    {
      "bits_per_byte": 1.5303090226312583,
      "learning_rate": 5.5782705759382104e-05,
      "loss_nats": 1.0607293844223022,
      "step": 6480,
      "unclipped_grad_norm": 1.1169558763504028
    },
    {
      "bits_per_byte": 1.610437975442702,
      "learning_rate": 5.5455628558420546e-05,
      "loss_nats": 1.1162705421447754,
      "step": 6490,
      "unclipped_grad_norm": 1.212812900543213
    },
    {
      "bits_per_byte": 1.6629645727039195,
      "learning_rate": 5.513042316852368e-05,
      "loss_nats": 1.152679204940796,
      "step": 6500,
      "unclipped_grad_norm": 1.249900221824646
    },
    {
      "bits_per_byte": 1.6085310318099353,
      "learning_rate": 5.480709514655443e-05,
      "loss_nats": 1.1149487495422363,
      "step": 6510,
      "unclipped_grad_norm": 1.2940583229064941
    },
    {
      "bits_per_byte": 1.5857588090091908,
      "learning_rate": 5.448565001729653e-05,
      "loss_nats": 1.0991642475128174,
      "step": 6520,
      "unclipped_grad_norm": 1.1899638175964355
    },
    {
      "bits_per_byte": 1.557970024265429,
      "learning_rate": 5.41660932733604e-05,
      "loss_nats": 1.0799025297164917,
      "step": 6530,
      "unclipped_grad_norm": 1.18255615234375
    },
    {
      "bits_per_byte": 1.5442463246745295,
      "learning_rate": 5.384843037508902e-05,
      "loss_nats": 1.070389986038208,
      "step": 6540,
      "unclipped_grad_norm": 1.1805294752120972
    },
    {
      "bits_per_byte": 1.5888496812104973,
      "learning_rate": 5.353266675046498e-05,
      "loss_nats": 1.101306676864624,
      "step": 6550,
      "unclipped_grad_norm": 1.1853868961334229
    },
    {
      "bits_per_byte": 1.598041465968511,
      "learning_rate": 5.321880779501729e-05,
      "loss_nats": 1.107677936553955,
      "step": 6560,
      "unclipped_grad_norm": 1.167456865310669
    },
    {
      "bits_per_byte": 1.5763221209563485,
      "learning_rate": 5.290685887172962e-05,
      "loss_nats": 1.092623233795166,
      "step": 6570,
      "unclipped_grad_norm": 1.2579692602157593
    },
    {
      "bits_per_byte": 1.5598688847136053,
      "learning_rate": 5.2596825310948295e-05,
      "loss_nats": 1.0812187194824219,
      "step": 6580,
      "unclipped_grad_norm": 1.218462347984314
    },
    {
      "bits_per_byte": 1.5745766690327418,
      "learning_rate": 5.228871241029154e-05,
      "loss_nats": 1.0914133787155151,
      "step": 6590,
      "unclipped_grad_norm": 1.1964205503463745
    },
    {
      "bits_per_byte": 1.483007602268555,
      "learning_rate": 5.198252543455865e-05,
      "loss_nats": 1.0279425382614136,
      "step": 6600,
      "unclipped_grad_norm": 1.1824510097503662
    },
    {
      "bits_per_byte": 1.5325055850480782,
      "learning_rate": 5.167826961564027e-05,
      "loss_nats": 1.0622519254684448,
      "step": 6610,
      "unclipped_grad_norm": 1.1718802452087402
    },
    {
      "bits_per_byte": 1.648197798335537,
      "learning_rate": 5.137595015242884e-05,
      "loss_nats": 1.1424436569213867,
      "step": 6620,
      "unclipped_grad_norm": 1.2758471965789795
    },
    {
      "bits_per_byte": 1.5936395700196773,
      "learning_rate": 5.1075572210729925e-05,
      "loss_nats": 1.1046267747879028,
      "step": 6630,
      "unclipped_grad_norm": 1.1752488613128662
    },
    {
      "bits_per_byte": 1.550369594975849,
      "learning_rate": 5.07771409231737e-05,
      "loss_nats": 1.074634313583374,
      "step": 6640,
      "unclipped_grad_norm": 1.1907187700271606
    },
    {
      "bits_per_byte": 1.539030778804464,
      "learning_rate": 5.048066138912756e-05,
      "loss_nats": 1.066774845123291,
      "step": 6650,
      "unclipped_grad_norm": 1.172040343284607
    },
    {
      "bits_per_byte": 1.5824082430050816,
      "learning_rate": 5.018613867460863e-05,
      "loss_nats": 1.096841812133789,
      "step": 6660,
      "unclipped_grad_norm": 1.2395251989364624
    },
    {
      "bits_per_byte": 1.55066678099654,
      "learning_rate": 4.989357781219755e-05,
      "loss_nats": 1.0748403072357178,
      "step": 6670,
      "unclipped_grad_norm": 1.1819844245910645
    },
    {
      "bits_per_byte": 1.567913545190364,
      "learning_rate": 4.960298380095222e-05,
      "loss_nats": 1.0867948532104492,
      "step": 6680,
      "unclipped_grad_norm": 1.207129955291748
    },
    {
      "bits_per_byte": 1.5601953077849429,
      "learning_rate": 4.931436160632256e-05,
      "loss_nats": 1.0814449787139893,
      "step": 6690,
      "unclipped_grad_norm": 1.284845232963562
    },
    {
      "bits_per_byte": 1.5747911313983678,
      "learning_rate": 4.9027716160065485e-05,
      "loss_nats": 1.091562032699585,
      "step": 6700,
      "unclipped_grad_norm": 1.1885749101638794
    },
    {
      "bits_per_byte": 1.5878793550943293,
      "learning_rate": 4.874305236016084e-05,
      "loss_nats": 1.1006340980529785,
      "step": 6710,
      "unclipped_grad_norm": 1.178633689880371
    },
    {
      "bits_per_byte": 1.6366173465396898,
      "learning_rate": 4.846037507072752e-05,
      "loss_nats": 1.1344166994094849,
      "step": 6720,
      "unclipped_grad_norm": 1.265533685684204
    },
    {
      "bits_per_byte": 1.5154492056487592,
      "learning_rate": 4.817968912194049e-05,
      "loss_nats": 1.050429344177246,
      "step": 6730,
      "unclipped_grad_norm": 1.138245940208435
    },
    {
      "bits_per_byte": 1.547962697777024,
      "learning_rate": 4.790099930994813e-05,
      "loss_nats": 1.0729659795761108,
      "step": 6740,
      "unclipped_grad_norm": 1.167292833328247
    },
    {
      "bits_per_byte": 1.5245414124519232,
      "learning_rate": 4.762431039679047e-05,
      "loss_nats": 1.0567315816879272,
      "step": 6750,
      "unclipped_grad_norm": 1.1478474140167236
    },
    {
      "bits_per_byte": 1.5232747602283185,
      "learning_rate": 4.734962711031755e-05,
      "loss_nats": 1.0558536052703857,
      "step": 6760,
      "unclipped_grad_norm": 1.185871958732605
    },
    {
      "bits_per_byte": 1.5451932611501802,
      "learning_rate": 4.707695414410892e-05,
      "loss_nats": 1.0710463523864746,
      "step": 6770,
      "unclipped_grad_norm": 1.2135790586471558
    },
    {
      "bits_per_byte": 1.5261805790972967,
      "learning_rate": 4.680629615739322e-05,
      "loss_nats": 1.0578677654266357,
      "step": 6780,
      "unclipped_grad_norm": 1.2295221090316772
    },
    {
      "bits_per_byte": 1.5602444948230898,
      "learning_rate": 4.653765777496863e-05,
      "loss_nats": 1.0814790725708008,
      "step": 6790,
      "unclipped_grad_norm": 1.174469232559204
    },
    {
      "bits_per_byte": 1.5335944072106906,
      "learning_rate": 4.6271043587123986e-05,
      "loss_nats": 1.0630066394805908,
      "step": 6800,
      "unclipped_grad_norm": 1.1970314979553223
    },
    {
      "bits_per_byte": 1.5348973477736272,
      "learning_rate": 4.600645814956006e-05,
      "loss_nats": 1.0639097690582275,
      "step": 6810,
      "unclipped_grad_norm": 1.162747859954834
    },
    {
      "bits_per_byte": 1.5590328770477613,
      "learning_rate": 4.574390598331204e-05,
      "loss_nats": 1.0806392431259155,
      "step": 6820,
      "unclipped_grad_norm": 1.2186238765716553
    },
    {
      "bits_per_byte": 1.5711721004762618,
      "learning_rate": 4.548339157467199e-05,
      "loss_nats": 1.0890535116195679,
      "step": 6830,
      "unclipped_grad_norm": 1.1893293857574463
    },
    {
      "bits_per_byte": 1.526197777362383,
      "learning_rate": 4.52249193751124e-05,
      "loss_nats": 1.0578796863555908,
      "step": 6840,
      "unclipped_grad_norm": 1.2071653604507446
    },
    {
      "bits_per_byte": 1.624858204769596,
      "learning_rate": 4.4968493801209985e-05,
      "loss_nats": 1.1262658834457397,
      "step": 6850,
      "unclipped_grad_norm": 1.2998149394989014
    },
    {
      "bits_per_byte": 1.574837222748799,
      "learning_rate": 4.471411923457034e-05,
      "loss_nats": 1.0915939807891846,
      "step": 6860,
      "unclipped_grad_norm": 1.2379318475723267
    },
    {
      "bits_per_byte": 1.5010148737444944,
      "learning_rate": 4.446180002175291e-05,
      "loss_nats": 1.0404242277145386,
      "step": 6870,
      "unclipped_grad_norm": 1.1776995658874512
    },
    {
      "bits_per_byte": 1.5287346934452601,
      "learning_rate": 4.421154047419693e-05,
      "loss_nats": 1.0596381425857544,
      "step": 6880,
      "unclipped_grad_norm": 1.2149008512496948
    },
    {
      "bits_per_byte": 1.5161942344922967,
      "learning_rate": 4.396334486814753e-05,
      "loss_nats": 1.05094575881958,
      "step": 6890,
      "unclipped_grad_norm": 1.1354871988296509
    },
    {
      "bits_per_byte": 1.5760503883679853,
      "learning_rate": 4.371721744458285e-05,
      "loss_nats": 1.0924348831176758,
      "step": 6900,
      "unclipped_grad_norm": 1.2098510265350342
    },
    {
      "bits_per_byte": 1.5705858116194704,
      "learning_rate": 4.347316240914145e-05,
      "loss_nats": 1.0886471271514893,
      "step": 6910,
      "unclipped_grad_norm": 1.2706652879714966
    },
    {
      "bits_per_byte": 1.523291786510754,
      "learning_rate": 4.3231183932050546e-05,
      "loss_nats": 1.0558654069900513,
      "step": 6920,
      "unclipped_grad_norm": 1.1377208232879639
    },
    {
      "bits_per_byte": 1.5043128130574397,
      "learning_rate": 4.299128614805464e-05,
      "loss_nats": 1.0427101850509644,
      "step": 6930,
      "unclipped_grad_norm": 1.1997718811035156
    },
    {
      "bits_per_byte": 1.5789452003473081,
      "learning_rate": 4.2753473156344965e-05,
      "loss_nats": 1.0944414138793945,
      "step": 6940,
      "unclipped_grad_norm": 1.2278647422790527
    },
    {
      "bits_per_byte": 1.4992432804579567,
      "learning_rate": 4.251774902048942e-05,
      "loss_nats": 1.039196252822876,
      "step": 6950,
      "unclipped_grad_norm": 1.2732465267181396
    },
    {
      "bits_per_byte": 1.4942526878952196,
      "learning_rate": 4.228411776836306e-05,
      "loss_nats": 1.0357370376586914,
      "step": 6960,
      "unclipped_grad_norm": 1.1388111114501953
    },
    {
      "bits_per_byte": 1.4775614276636824,
      "learning_rate": 4.2052583392079427e-05,
      "loss_nats": 1.024167537689209,
      "step": 6970,
      "unclipped_grad_norm": 1.2496774196624756
    },
    {
      "bits_per_byte": 1.5972561931846714,
      "learning_rate": 4.182314984792214e-05,
      "loss_nats": 1.1071336269378662,
      "step": 6980,
      "unclipped_grad_norm": 1.2329373359680176
    },
    {
      "bits_per_byte": 1.5648880263963856,
      "learning_rate": 4.159582105627748e-05,
      "loss_nats": 1.0846977233886719,
      "step": 6990,
      "unclipped_grad_norm": 1.225949764251709
    },
    {
      "bits_per_byte": 1.5748815942727217,
      "learning_rate": 4.137060090156724e-05,
      "loss_nats": 1.0916247367858887,
      "step": 7000,
      "unclipped_grad_norm": 1.2655893564224243
    },
    {
      "bits_per_byte": 1.4699690815586928,
      "learning_rate": 4.114749323218249e-05,
      "loss_nats": 1.0189049243927002,
      "step": 7010,
      "unclipped_grad_norm": 1.1935946941375732
    },
    {
      "bits_per_byte": 1.5527095909234883,
      "learning_rate": 4.0926501860417706e-05,
      "loss_nats": 1.076256275177002,
      "step": 7020,
      "unclipped_grad_norm": 1.2986186742782593
    },
    {
      "bits_per_byte": 1.5324563980099315,
      "learning_rate": 4.070763056240571e-05,
      "loss_nats": 1.0622178316116333,
      "step": 7030,
      "unclipped_grad_norm": 1.2135052680969238
    },
    {
      "bits_per_byte": 1.5215922539549278,
      "learning_rate": 4.049088307805306e-05,
      "loss_nats": 1.0546873807907104,
      "step": 7040,
      "unclipped_grad_norm": 1.1919881105422974
    },
    {
      "bits_per_byte": 1.5129705916845244,
      "learning_rate": 4.027626311097629e-05,
      "loss_nats": 1.0487112998962402,
      "step": 7050,
      "unclipped_grad_norm": 1.177833914756775
    },
    {
      "bits_per_byte": 1.5002927185935215,
      "learning_rate": 4.006377432843846e-05,
      "loss_nats": 1.0399236679077148,
      "step": 7060,
      "unclipped_grad_norm": 1.2103830575942993
    },
    {
      "bits_per_byte": 1.5379128915738558,
      "learning_rate": 3.985342036128656e-05,
      "loss_nats": 1.065999984741211,
      "step": 7070,
      "unclipped_grad_norm": 1.2223129272460938
    },
    {
      "bits_per_byte": 1.5494862920810175,
      "learning_rate": 3.964520480388956e-05,
      "loss_nats": 1.0740220546722412,
      "step": 7080,
      "unclipped_grad_norm": 1.2125953435897827
    },
    {
      "bits_per_byte": 1.5050088268454815,
      "learning_rate": 3.943913121407678e-05,
      "loss_nats": 1.0431926250457764,
      "step": 7090,
      "unclipped_grad_norm": 1.274321436882019
    },
    {
      "bits_per_byte": 1.5086218383748073,
      "learning_rate": 3.923520311307735e-05,
      "loss_nats": 1.0456969738006592,
      "step": 7100,
      "unclipped_grad_norm": 1.244695782661438
    },
    {
      "bits_per_byte": 1.480645764524256,
      "learning_rate": 3.9033423985459816e-05,
      "loss_nats": 1.0263054370880127,
      "step": 7110,
      "unclipped_grad_norm": 1.1544733047485352
    },
    {
      "bits_per_byte": 1.5449419944972695,
      "learning_rate": 3.88337972790728e-05,
      "loss_nats": 1.070872187614441,
      "step": 7120,
      "unclipped_grad_norm": 1.25104558467865
    },
    {
      "bits_per_byte": 1.5693447848108444,
      "learning_rate": 3.8636326404985845e-05,
      "loss_nats": 1.0877869129180908,
      "step": 7130,
      "unclipped_grad_norm": 1.2125219106674194
    },
    {
      "bits_per_byte": 1.5401648324042534,
      "learning_rate": 3.844101473743143e-05,
      "loss_nats": 1.0675609111785889,
      "step": 7140,
      "unclipped_grad_norm": 1.2392476797103882
    },
    {
      "bits_per_byte": 1.581065058501843,
      "learning_rate": 3.824786561374701e-05,
      "loss_nats": 1.0959107875823975,
      "step": 7150,
      "unclipped_grad_norm": 1.2786449193954468
    },
    {
      "bits_per_byte": 1.568793580414829,
      "learning_rate": 3.805688233431824e-05,
      "loss_nats": 1.0874048471450806,
      "step": 7160,
      "unclipped_grad_norm": 1.2588245868682861
    },
    {
      "bits_per_byte": 1.5794673396753276,
      "learning_rate": 3.786806816252236e-05,
      "loss_nats": 1.0948033332824707,
      "step": 7170,
      "unclipped_grad_norm": 1.2507553100585938
    },
    {
      "bits_per_byte": 1.519351835962138,
      "learning_rate": 3.768142632467269e-05,
      "loss_nats": 1.0531344413757324,
      "step": 7180,
      "unclipped_grad_norm": 1.194468379020691
    },
    {
      "bits_per_byte": 1.5739864245749806,
      "learning_rate": 3.7496960009963236e-05,
      "loss_nats": 1.0910042524337769,
      "step": 7190,
      "unclipped_grad_norm": 1.2753334045410156
    },
    {
      "bits_per_byte": 1.5095894127685614,
      "learning_rate": 3.7314672370414325e-05,
      "loss_nats": 1.0463676452636719,
      "step": 7200,
      "unclipped_grad_norm": 1.1532927751541138
    },
    {
      "bits_per_byte": 1.5104264523303108,
      "learning_rate": 3.713456652081882e-05,
      "loss_nats": 1.0469478368759155,
      "step": 7210,
      "unclipped_grad_norm": 1.1723830699920654
    },
    {
      "bits_per_byte": 1.5078061246617651,
      "learning_rate": 3.695664553868873e-05,
      "loss_nats": 1.0451315641403198,
      "step": 7220,
      "unclipped_grad_norm": 1.254270076751709
    },
    {
      "bits_per_byte": 1.5518125294165879,
      "learning_rate": 3.678091246420277e-05,
      "loss_nats": 1.075634479522705,
      "step": 7230,
      "unclipped_grad_norm": 1.2554386854171753
    },
    {
      "bits_per_byte": 1.5582962753541159,
      "learning_rate": 3.660737030015427e-05,
      "loss_nats": 1.0801286697387695,
      "step": 7240,
      "unclipped_grad_norm": 1.2507786750793457
    },
    {
      "bits_per_byte": 1.571778683285855,
      "learning_rate": 3.6436022011900056e-05,
      "loss_nats": 1.0894739627838135,
      "step": 7250,
      "unclipped_grad_norm": 1.2601356506347656
    },
    {
      "bits_per_byte": 1.5335579468887077,
      "learning_rate": 3.626687052730954e-05,
      "loss_nats": 1.062981367111206,
      "step": 7260,
      "unclipped_grad_norm": 1.203548789024353
    },
    {
      "bits_per_byte": 1.5322409037484004,
      "learning_rate": 3.6099918736714955e-05,
      "loss_nats": 1.0620684623718262,
      "step": 7270,
      "unclipped_grad_norm": 1.2632365226745605
    },
    {
      "bits_per_byte": 1.522663877852454,
      "learning_rate": 3.5935169492861716e-05,
      "loss_nats": 1.0554301738739014,
      "step": 7280,
      "unclipped_grad_norm": 1.226701259613037
    },
    {
      "bits_per_byte": 1.5219511817472784,
      "learning_rate": 3.5772625610859886e-05,
      "loss_nats": 1.054936170578003,
      "step": 7290,
      "unclipped_grad_norm": 1.2358945608139038
    },
    {
      "bits_per_byte": 1.5307491262348163,
      "learning_rate": 3.5612289868135887e-05,
      "loss_nats": 1.0610344409942627,
      "step": 7300,
      "unclipped_grad_norm": 1.1983592510223389
    },
    {
      "bits_per_byte": 1.5145411372522035,
      "learning_rate": 3.545416500438522e-05,
      "loss_nats": 1.049799919128418,
      "step": 7310,
      "unclipped_grad_norm": 1.224237084388733
    },
    {
      "bits_per_byte": 1.4925153191562035,
      "learning_rate": 3.52982537215255e-05,
      "loss_nats": 1.0345327854156494,
      "step": 7320,
      "unclipped_grad_norm": 1.2309412956237793
    },
    {
      "bits_per_byte": 1.55153787312316,
      "learning_rate": 3.5144558683650396e-05,
      "loss_nats": 1.0754441022872925,
      "step": 7330,
      "unclipped_grad_norm": 1.2790515422821045
    },
    {
      "bits_per_byte": 1.5015385609163716,
      "learning_rate": 3.499308251698404e-05,
      "loss_nats": 1.0407872200012207,
      "step": 7340,
      "unclipped_grad_norm": 1.2610660791397095
    },
    {
      "bits_per_byte": 1.5181352306899345,
      "learning_rate": 3.484382780983616e-05,
      "loss_nats": 1.0522911548614502,
      "step": 7350,
      "unclipped_grad_norm": 1.2389053106307983
    },
    {
      "bits_per_byte": 1.5177651240252776,
      "learning_rate": 3.469679711255795e-05,
      "loss_nats": 1.052034616470337,
      "step": 7360,
      "unclipped_grad_norm": 1.2198708057403564
    },
    {
      "bits_per_byte": 1.4782792832483838,
      "learning_rate": 3.4551992937498284e-05,
      "loss_nats": 1.024665117263794,
      "step": 7370,
      "unclipped_grad_norm": 1.2335501909255981
    },
    {
      "bits_per_byte": 1.4776752801785535,
      "learning_rate": 3.440941775896104e-05,
      "loss_nats": 1.0242464542388916,
      "step": 7380,
      "unclipped_grad_norm": 1.1802135705947876
    },
    {
      "bits_per_byte": 1.4876678161589845,
      "learning_rate": 3.4269074013162596e-05,
      "loss_nats": 1.031172752380371,
      "step": 7390,
      "unclipped_grad_norm": 1.2134222984313965
    },
    {
      "bits_per_byte": 1.508026778402822,
      "learning_rate": 3.41309640981904e-05,
      "loss_nats": 1.0452845096588135,
      "step": 7400,
      "unclipped_grad_norm": 1.23512601852417
    },
    {
      "bits_per_byte": 1.5449191208047048,
      "learning_rate": 3.3995090373961795e-05,
      "loss_nats": 1.0708563327789307,
      "step": 7410,
      "unclipped_grad_norm": 1.250113606452942
    },
    {
      "bits_per_byte": 1.5275736385692853,
      "learning_rate": 3.3861455162183876e-05,
      "loss_nats": 1.058833360671997,
      "step": 7420,
      "unclipped_grad_norm": 1.2959712743759155
    },
    {
      "bits_per_byte": 1.4779425412179943,
      "learning_rate": 3.3730060746313664e-05,
      "loss_nats": 1.0244317054748535,
      "step": 7430,
      "unclipped_grad_norm": 1.2527908086776733
    },
    {
      "bits_per_byte": 1.468023269846831,
      "learning_rate": 3.3600909371519225e-05,
      "loss_nats": 1.0175561904907227,
      "step": 7440,
      "unclipped_grad_norm": 1.2249972820281982
    },
    {
      "bits_per_byte": 1.5643605556061893,
      "learning_rate": 3.3474003244641194e-05,
      "loss_nats": 1.0843321084976196,
      "step": 7450,
      "unclipped_grad_norm": 1.2696515321731567
    },
    {
      "bits_per_byte": 1.5085103936170483,
      "learning_rate": 3.334934453415514e-05,
      "loss_nats": 1.0456197261810303,
      "step": 7460,
      "unclipped_grad_norm": 1.2498137950897217
    },
    {
      "bits_per_byte": 1.4827952036947394,
      "learning_rate": 3.3226935370134486e-05,
      "loss_nats": 1.0277953147888184,
      "step": 7470,
      "unclipped_grad_norm": 1.2631312608718872
    },
    {
      "bits_per_byte": 1.5577199614910746,
      "learning_rate": 3.31067778442141e-05,
      "loss_nats": 1.0797291994094849,
      "step": 7480,
      "unclipped_grad_norm": 1.1877262592315674
    },
    {
      "bits_per_byte": 1.530344623039987,
      "learning_rate": 3.298887400955457e-05,
      "loss_nats": 1.0607540607452393,
      "step": 7490,
      "unclipped_grad_norm": 1.268275499343872
    },
    {
      "bits_per_byte": 1.5391859071555423,
      "learning_rate": 3.287322588080714e-05,
      "loss_nats": 1.0668823719024658,
      "step": 7500,
      "unclipped_grad_norm": 1.3035756349563599
    },
    {
      "bits_per_byte": 1.5412744644676202,
      "learning_rate": 3.275983543407924e-05,
      "loss_nats": 1.0683300495147705,
      "step": 7510,
      "unclipped_grad_norm": 1.2108540534973145
    },
    {
      "bits_per_byte": 1.5426088778556648,
      "learning_rate": 3.2648704606900735e-05,
      "loss_nats": 1.069254994392395,
      "step": 7520,
      "unclipped_grad_norm": 1.265081524848938
    },
    {
      "bits_per_byte": 1.5294915890917073,
      "learning_rate": 3.253983529819086e-05,
      "loss_nats": 1.0601627826690674,
      "step": 7530,
      "unclipped_grad_norm": 1.2194392681121826
    },
    {
      "bits_per_byte": 1.5287204188852386,
      "learning_rate": 3.243322936822571e-05,
      "loss_nats": 1.0596282482147217,
      "step": 7540,
      "unclipped_grad_norm": 1.2046699523925781
    },
    {
      "bits_per_byte": 1.5311642923539992,
      "learning_rate": 3.232888863860648e-05,
      "loss_nats": 1.0613222122192383,
      "step": 7550,
      "unclipped_grad_norm": 1.2151964902877808
    },
    {
      "bits_per_byte": 1.4454386801181778,
      "learning_rate": 3.222681489222838e-05,
      "loss_nats": 1.0019017457962036,
      "step": 7560,
      "unclipped_grad_norm": 1.2129703760147095
    },
    {
      "bits_per_byte": 1.4692137337561033,
      "learning_rate": 3.212700987325009e-05,
      "loss_nats": 1.0183813571929932,
      "step": 7570,
      "unclipped_grad_norm": 1.2061578035354614
    },
    {
      "bits_per_byte": 1.4652694836412157,
      "learning_rate": 3.202947528706403e-05,
      "loss_nats": 1.0156474113464355,
      "step": 7580,
      "unclipped_grad_norm": 1.1768887042999268
    },
    {
      "bits_per_byte": 1.5398244787381958,
      "learning_rate": 3.1934212800267116e-05,
      "loss_nats": 1.0673249959945679,
      "step": 7590,
      "unclipped_grad_norm": 1.2310807704925537
    },
    {
      "bits_per_byte": 1.5671884663343265,
      "learning_rate": 3.1841224040632484e-05,
      "loss_nats": 1.0862922668457031,
      "step": 7600,
      "unclipped_grad_norm": 1.2284492254257202
    },
    {
      "bits_per_byte": 1.5395315922837765,
      "learning_rate": 3.175051059708143e-05,
      "loss_nats": 1.067121982574463,
      "step": 7610,
      "unclipped_grad_norm": 1.2651493549346924
    },
    {
      "bits_per_byte": 1.5411196800818436,
      "learning_rate": 3.1662074019656414e-05,
      "loss_nats": 1.0682227611541748,
      "step": 7620,
      "unclipped_grad_norm": 1.2471308708190918
    },
    {
      "bits_per_byte": 1.5165897945892812,
      "learning_rate": 3.157591581949455e-05,
      "loss_nats": 1.0512199401855469,
      "step": 7630,
      "unclipped_grad_norm": 1.253797173500061
    },
    {
      "bits_per_byte": 1.497296092884888,
      "learning_rate": 3.149203746880175e-05,
      "loss_nats": 1.037846565246582,
      "step": 7640,
      "unclipped_grad_norm": 1.318769931793213
    },
    {
      "bits_per_byte": 1.4637429656321574,
      "learning_rate": 3.141044040082763e-05,
      "loss_nats": 1.0145893096923828,
      "step": 7650,
      "unclipped_grad_norm": 1.2420055866241455
    },
    {
      "bits_per_byte": 1.4840704550508872,
      "learning_rate": 3.133112600984091e-05,
      "loss_nats": 1.0286792516708374,
      "step": 7660,
      "unclipped_grad_norm": 1.299368977546692
    },
    {
      "bits_per_byte": 1.5379041204586616,
      "learning_rate": 3.125409565110571e-05,
      "loss_nats": 1.0659939050674438,
      "step": 7670,
      "unclipped_grad_norm": 1.2423371076583862
    },
    {
      "bits_per_byte": 1.5171365274363742,
      "learning_rate": 3.117935064085835e-05,
      "loss_nats": 1.0515989065170288,
      "step": 7680,
      "unclipped_grad_norm": 1.2243461608886719
    },
    {
      "bits_per_byte": 1.5051288707357837,
      "learning_rate": 3.110689225628479e-05,
      "loss_nats": 1.0432758331298828,
      "step": 7690,
      "unclipped_grad_norm": 1.245628833770752
    },
    {
      "bits_per_byte": 1.544223278999314,
      "learning_rate": 3.1036721735498926e-05,
      "loss_nats": 1.0703740119934082,
      "step": 7700,
      "unclipped_grad_norm": 1.2915515899658203
    },
    {
      "bits_per_byte": 1.4916550619365878,
      "learning_rate": 3.096884027752132e-05,
      "loss_nats": 1.0339365005493164,
      "step": 7710,
      "unclipped_grad_norm": 1.201813817024231
    },
    {
      "bits_per_byte": 1.4940050328779773,
      "learning_rate": 3.090324904225885e-05,
      "loss_nats": 1.0355653762817383,
      "step": 7720,
      "unclipped_grad_norm": 1.2110096216201782
    },
    {
      "bits_per_byte": 1.4993191248069873,
      "learning_rate": 3.083994915048472e-05,
      "loss_nats": 1.0392488241195679,
      "step": 7730,
      "unclipped_grad_norm": 1.2087814807891846
    },
    {
      "bits_per_byte": 1.5356127956012164,
      "learning_rate": 3.077894168381942e-05,
      "loss_nats": 1.0644056797027588,
      "step": 7740,
      "unclipped_grad_norm": 1.3031466007232666
    },
    {
      "bits_per_byte": 1.4696118735928507,
      "learning_rate": 3.0720227684712254e-05,
      "loss_nats": 1.0186573266983032,
      "step": 7750,
      "unclipped_grad_norm": 1.249332070350647
    },
    {
      "bits_per_byte": 1.4759148657643217,
      "learning_rate": 3.0663808156423456e-05,
      "loss_nats": 1.0230262279510498,
      "step": 7760,
      "unclipped_grad_norm": 1.2120082378387451
    },
    {
      "bits_per_byte": 1.5360676897127488,
      "learning_rate": 3.060968406300711e-05,
      "loss_nats": 1.0647209882736206,
      "step": 7770,
      "unclipped_grad_norm": 1.2807056903839111
    },
    {
      "bits_per_byte": 1.529660132089553,
      "learning_rate": 3.055785632929463e-05,
      "loss_nats": 1.0602796077728271,
      "step": 7780,
      "unclipped_grad_norm": 1.2968212366104126
    },
    {
      "bits_per_byte": 1.5315347429839576,
      "learning_rate": 3.0508325840878984e-05,
      "loss_nats": 1.0615789890289307,
      "step": 7790,
      "unclipped_grad_norm": 1.2608845233917236
    },
    {
      "bits_per_byte": 1.5176234103209667,
      "learning_rate": 3.046109344409957e-05,
      "loss_nats": 1.051936388015747,
      "step": 7800,
      "unclipped_grad_norm": 1.2756648063659668
    },
    {
      "bits_per_byte": 1.5054119541791038,
      "learning_rate": 3.0416159946027727e-05,
      "loss_nats": 1.0434720516204834,
      "step": 7810,
      "unclipped_grad_norm": 1.2882802486419678
    },
    {
      "bits_per_byte": 1.5431284374439214,
      "learning_rate": 3.0373526114452946e-05,
      "loss_nats": 1.069615125656128,
      "step": 7820,
      "unclipped_grad_norm": 1.3333698511123657
    },
    {
      "bits_per_byte": 1.5271158207526887,
      "learning_rate": 3.0333192677869803e-05,
      "loss_nats": 1.058516025543213,
      "step": 7830,
      "unclipped_grad_norm": 1.229974389076233
    },
    {
      "bits_per_byte": 1.501238623173267,
      "learning_rate": 3.029516032546544e-05,
      "loss_nats": 1.0405793190002441,
      "step": 7840,
      "unclipped_grad_norm": 1.2474827766418457
    },
    {
      "bits_per_byte": 1.4945333635814277,
      "learning_rate": 3.02594297071078e-05,
      "loss_nats": 1.0359315872192383,
      "step": 7850,
      "unclipped_grad_norm": 1.2343677282333374
    },
    {
      "bits_per_byte": 1.534035198744852,
      "learning_rate": 3.0226001433334603e-05,
      "loss_nats": 1.0633121728897095,
      "step": 7860,
      "unclipped_grad_norm": 1.256583571434021
    },
    {
      "bits_per_byte": 1.5241123157380205,
      "learning_rate": 3.019487607534278e-05,
      "loss_nats": 1.056434154510498,
      "step": 7870,
      "unclipped_grad_norm": 1.3308851718902588
    },
    {
      "bits_per_byte": 1.498756569556015,
      "learning_rate": 3.016605416497886e-05,
      "loss_nats": 1.0388588905334473,
      "step": 7880,
      "unclipped_grad_norm": 1.301889181137085
    },
    {
      "bits_per_byte": 1.516417124007815,
      "learning_rate": 3.0139536194729734e-05,
      "loss_nats": 1.051100254058838,
      "step": 7890,
      "unclipped_grad_norm": 1.2672597169876099
    },
    {
      "bits_per_byte": 1.4738032627770283,
      "learning_rate": 3.0115322617714362e-05,
      "loss_nats": 1.0215625762939453,
      "step": 7900,
      "unclipped_grad_norm": 1.2230888605117798
    },
    {
      "bits_per_byte": 1.5374244608454053,
      "learning_rate": 3.0093413847675965e-05,
      "loss_nats": 1.0656614303588867,
      "step": 7910,
      "unclipped_grad_norm": 1.3816039562225342
    },
    {
      "bits_per_byte": 1.5497416863175488,
      "learning_rate": 3.0073810258974985e-05,
      "loss_nats": 1.0741990804672241,
      "step": 7920,
      "unclipped_grad_norm": 1.2244949340820312
    },
    {
      "bits_per_byte": 1.5130077399371107,
      "learning_rate": 3.0056512186582658e-05,
      "loss_nats": 1.0487370491027832,
      "step": 7930,
      "unclipped_grad_norm": 1.2602943181991577
    },
    {
      "bits_per_byte": 1.5360826522033737,
      "learning_rate": 3.0041519926075316e-05,
      "loss_nats": 1.0647313594818115,
      "step": 7940,
      "unclipped_grad_norm": 1.278623342514038
    },
    {
      "bits_per_byte": 1.479992574416279,
      "learning_rate": 3.0028833733629313e-05,
      "loss_nats": 1.0258526802062988,
      "step": 7950,
      "unclipped_grad_norm": 1.2290197610855103
    },
    {
      "bits_per_byte": 1.5513340736818877,
      "learning_rate": 3.0018453826016686e-05,
      "loss_nats": 1.0753028392791748,
      "step": 7960,
      "unclipped_grad_norm": 1.3293349742889404
    },
    {
      "bits_per_byte": 1.5048769161522697,
      "learning_rate": 3.001038038060143e-05,
      "loss_nats": 1.043101191520691,
      "step": 7970,
      "unclipped_grad_norm": 1.2828478813171387
    },
    {
      "bits_per_byte": 1.5449772509406965,
      "learning_rate": 3.0004613535336433e-05,
      "loss_nats": 1.0708966255187988,
      "step": 7980,
      "unclipped_grad_norm": 1.325577974319458
    },
    {
      "bits_per_byte": 1.5031080745881458,
      "learning_rate": 3.000115338876117e-05,
      "loss_nats": 1.0418751239776611,
      "step": 7990,
      "unclipped_grad_norm": 1.3532280921936035
    },
    {
      "bits_per_byte": 1.5250520289423348,
      "learning_rate": 3e-05,
      "loss_nats": 1.0570855140686035,
      "step": 8000,
      "unclipped_grad_norm": 1.3144081830978394
    }
  ]
}