{
  "architecture": "block",
  "batch_size": 32,
  "canonical_sha256_without_self": "6b2a0f3320fece089c65a158888f4aafe0879ff220d9835ed3614be007b68068",
  "depth": 16,
  "diagnostics": [
    {
      "activation_grad_rms_by_block": [
        0.00027713447343558073,
        0.00024001447309274226,
        0.00020889485313091427,
        0.00019106797117274255,
        0.00017709653184283525,
        0.00016345064796041697,
        0.0001520135992905125,
        0.00014396758342627436,
        0.00013824697816744447,
        0.0001333381951553747,
        0.00012879272981081158,
        0.00012578140012919903,
        0.00012338913802523166,
        0.0001212825154652819,
        0.00011928759340662509,
        0.00011723516217898577
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00022927794270799495,
        "first_to_last_ratio": 1.9059069547229377,
        "imbalance_abs_log_ratio": 0.6449579870010378,
        "last_quartile_mean": 0.00012029860226903111,
        "mean": 0.00016006211535568582,
        "normalized": [
          1.7314182860806244,
          1.4995083162520277,
          1.3050861702453052,
          1.1937113960299495,
          1.1064237870985023,
          1.021170110098828,
          0.9497162957812465,
          0.8994482117542517,
          0.8637083038683806,
          0.8330403159990363,
          0.8046421823465957,
          0.7858286756343993,
          0.7708828397715447,
          0.7577215582573746,
          0.7452581339534802,
          0.7324354168284537
        ],
        "population_cv": 0.28697332073733334
      },
      "activation_output_rms_by_block": [
        0.005004490725696087,
        0.007150472607463598,
        0.0056586493737995625,
        0.007886149920523167,
        0.005775847006589174,
        0.008285986259579659,
        0.006597128231078386,
        0.009296777658164501,
        0.007501539774239063,
        0.01081777922809124,
        0.007698386441916227,
        0.011321330443024635,
        0.009009703993797302,
        0.013792803511023521,
        0.00953624676913023,
        0.012954792007803917
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.0064249406568706036,
        "first_to_last_ratio": 0.567404514268178,
        "imbalance_abs_log_ratio": 0.5666828005795043,
        "last_quartile_mean": 0.011323386570438743,
        "mean": 0.008643005246995017,
        "normalized": [
          0.5790220626599802,
          0.8273132322752738,
          0.6547085431617609,
          0.9124314628022479,
          0.6682683674867963,
          0.9586927258271091,
          0.7632910130850682,
          1.0756417926966766,
          0.8679318778438997,
          1.251622429808468,
          0.8907071350665657,
          1.3098835554867694,
          1.04242722714183,
          1.5958342170183197,
          1.103348487778111,
          1.4988758698611242
        ],
        "population_cv": 0.28641618387108014
      },
      "bits_per_byte": 8.092078157393424,
      "branch_output_rms_by_sublayer": [
        0.0033052791841328144,
        0.003737553022801876,
        0.0033692142460495234,
        0.0038969451561570168,
        0.004222586750984192,
        0.003809630637988448,
        0.004224879201501608,
        0.003801452461630106,
        0.004351162351667881,
        0.0037875697016716003,
        0.005037518683820963,
        0.0037579576019197702,
        0.005422801710665226,
        0.003782435320317745,
        0.005750134587287903,
        0.003670723643153906,
        0.006515308283269405,
        0.003754077944904566,
        0.006654165219515562,
        0.0036852110642939806,
        0.006751408334821463,
        0.003720880253240466,
        0.007695567328482866,
        0.003703791182488203,
        0.008140962570905685,
        0.003749529365450144,
        0.009426429867744446,
        0.0036915375385433435,
        0.008798080496490002,
        0.003752112854272127,
        0.008380400948226452,
        0.0036098985001444817
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.007836087849617155,
        0.00707663314616961,
        0.006860545956961424,
        0.006477463491545069,
        0.00630808320659222,
        0.0065403273634858875,
        0.006076989561376891,
        0.0053044433268254945,
        0.006002429844215364,
        0.005811298045810947,
        0.00541486468124986,
        0.005552686577658202,
        0.005724443119797102,
        0.005942467552368739,
        0.006312250195815329,
        0.004848514440357715
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.007062682611073314,
        "first_to_last_ratio": 1.237564932158175,
        "imbalance_abs_log_ratio": 0.21314568451503188,
        "last_quartile_mean": 0.005706918827084721,
        "mean": 0.006130595522490438,
        "normalized": [
          1.2781935818257821,
          1.154314147819503,
          1.119066807097797,
          1.0565798163950184,
          1.0289511326347756,
          1.0668339379905794,
          0.9912559944760846,
          0.865241118479574,
          0.9790940899942118,
          0.947917380047643,
          0.883252640201938,
          0.9057336366895933,
          0.9337499267072272,
          0.9693132633801169,
          1.0296308364592772,
          0.7908716898008789
        ],
        "population_cv": 0.1166547247084935
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.02823692187666893,
        0.014202049933373928,
        0.014330258592963219,
        0.014431006275117397,
        0.0145475585013628,
        0.009805543348193169,
        0.009861846454441547,
        0.009975572116672993,
        0.010043247602880001,
        0.007605585735291243,
        0.007654537446796894,
        0.0077590313740074635,
        0.007819762453436852,
        0.006375181954354048,
        0.006421996746212244,
        0.0065260399132966995,
        0.006563509814441204,
        0.005602568853646517,
        0.0056375316344201565,
        0.0057973419316112995,
        0.005820456892251968,
        0.005056646186858416,
        0.00509054446592927,
        0.005272198468446732,
        0.005295110866427422,
        0.0046798414550721645,
        0.004703808110207319,
        0.004869809839874506,
        0.004889858886599541,
        0.004459173884242773,
        0.00446688337251544,
        0.00453129643574357
      ],
      "loss_nats": 5.609001159667969,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 2.1972246170043945,
        "mean_weights": [
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519
        ],
        "sources": 9
      },
      "step": 0,
      "stream_state_rms_by_sublayer": [
        0.0033052791841328144,
        0.005004490725696087,
        0.005999269895255566,
        0.007150472607463598,
        0.004222586750984192,
        0.0056586493737995625,
        0.006890070624649525,
        0.007886149920523167,
        0.004351162351667881,
        0.005775847006589174,
        0.007411441300064325,
        0.008285986259579659,
        0.005422801710665226,
        0.006597128231078386,
        0.008578632958233356,
        0.009296777658164501,
        0.006515308283269405,
        0.007501539774239063,
        0.010256913490593433,
        0.01081777922809124,
        0.006751408334821463,
        0.007698386441916227,
        0.01081965770572424,
        0.011321330443024635,
        0.008140962570905685,
        0.009009703993797302,
        0.013233783654868603,
        0.013792803511023521,
        0.008798080496490002,
        0.00953624676913023,
        0.01231932733207941,
        0.012954792007803917
      ]
    },
    {
      "activation_grad_rms_by_block": [
        8.295638690469787e-05,
        7.495813042623922e-05,
        6.424964522011578e-05,
        5.6337856221944094e-05,
        4.736588743980974e-05,
        4.000150511274114e-05,
        3.3406991860829294e-05,
        2.9257240385049954e-05,
        2.6081905161845498e-05,
        2.4221646526711993e-05,
        2.3017932107904926e-05,
        2.205361488449853e-05,
        2.1097952412674204e-05,
        2.0617559130187146e-05,
        2.0049723389092833e-05,
        1.9771381630562246e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 6.962550469324924e-05,
        "first_to_last_ratio": 3.415668082811133,
        "imbalance_abs_log_ratio": 1.2283731057520397,
        "last_quartile_mean": 2.0384154140629107e-05,
        "mean": 3.784033492593153e-05,
        "normalized": [
          2.1922741187961536,
          1.980905575306399,
          1.6979142850050797,
          1.4888308026929338,
          1.2517301322127163,
          1.0571128715176512,
          0.8828408079954885,
          0.7731760419752606,
          0.6892620060815551,
          0.640101272203941,
          0.6082909190143296,
          0.5828070741885916,
          0.557551946989138,
          0.5448566766267753,
          0.5298505795030106,
          0.5224948898909759
        ],
        "population_cv": 0.5389270020989329
      },
      "activation_output_rms_by_block": [
        0.00972363818436861,
        0.018413539975881577,
        0.011301152408123016,
        0.020593363791704178,
        0.012286243960261345,
        0.022828614339232445,
        0.014672646299004555,
        0.02904798462986946,
        0.01802690140902996,
        0.03463379293680191,
        0.0215163491666317,
        0.04299422353506088,
        0.024622667580842972,
        0.048123423010110855,
        0.032037004828453064,
        0.06016550958156586
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.015007923590019345,
        "first_to_last_ratio": 0.36394181302547757,
        "imbalance_abs_log_ratio": 1.010761278449072,
        "last_quartile_mean": 0.04123715125024319,
        "mean": 0.0263116909773089,
        "normalized": [
          0.3695558066851058,
          0.6998235116002747,
          0.4295106847320868,
          0.7826697193070493,
          0.4669499946186262,
          0.8676224708978132,
          0.5576474089657784,
          1.1039953553316026,
          0.6851289574879962,
          1.3162891342358027,
          0.8177486268437599,
          1.6340349836176982,
          0.9358071133503911,
          1.8289749241739084,
          1.2175958153385753,
          2.2866454928135314
        ],
        "population_cv": 0.5242218345998277
      },
      "bits_per_byte": 5.094476963164152,
      "branch_output_rms_by_sublayer": [
        0.004319524392485619,
        0.008673875592648983,
        0.004246409051120281,
        0.009991174563765526,
        0.005452096462249756,
        0.010073051787912846,
        0.006387368775904179,
        0.010035603307187557,
        0.0060663470067083836,
        0.01051078736782074,
        0.007258581463247538,
        0.010655198246240616,
        0.007839187048375607,
        0.011261194944381714,
        0.010483811609447002,
        0.011669226922094822,
        0.01079322025179863,
        0.012236819602549076,
        0.012645936571061611,
        0.012139025144279003,
        0.013178177177906036,
        0.012943001464009285,
        0.017010189592838287,
        0.012494352646172047,
        0.016437800601124763,
        0.012577332556247711,
        0.018056757748126984,
        0.01252446137368679,
        0.022778941318392754,
        0.013186860829591751,
        0.022304536774754524,
        0.012990611605346203
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00047768131303286524,
        0.00037801938737084786,
        0.0003757481810571057,
        0.0003849183374966076,
        0.0003205908784645981,
        0.0003135875371373823,
        0.0002702256708410831,
        0.00025052984822069896,
        0.00022846424191684566,
        0.00020301436013832287,
        0.00018877625003321395,
        0.00018901092530252878,
        0.00017529332745812984,
        0.0001779656532508539,
        0.00018745944157007413,
        0.00017444725355961893
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.0004040918047393566,
        "first_to_last_ratio": 2.260129748343848,
        "imbalance_abs_log_ratio": 0.8154222224078733,
        "last_quartile_mean": 0.0001787914189596692,
        "mean": 0.00026848328792817357,
        "normalized": [
          1.779184532188304,
          1.407981071328276,
          1.39952167584311,
          1.4336770845848084,
          1.1940813185749002,
          1.167996487071014,
          1.0064897257715932,
          0.9331301406280542,
          0.8509439961043904,
          0.7561526890740202,
          0.703121045223927,
          0.7039951229779867,
          0.6529021929477618,
          0.6628556087202881,
          0.6982164253747687,
          0.649750883586796
        ],
        "population_cv": 0.3443291255737099
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.6925109028816223,
          "mean_weights": [
            0.516845703125,
            0.4831542670726776
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6916897296905518,
          "mean_weights": [
            0.5256155133247375,
            0.47438448667526245
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6921632289886475,
          "mean_weights": [
            0.5142087936401367,
            0.4857912063598633
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6927341222763062,
          "mean_weights": [
            0.5126981735229492,
            0.4873018264770508
          ],
          "sources": 2
        },
        {
          "entropy_mean": 1.0980398654937744,
          "mean_weights": [
            0.3447725176811218,
            0.324658066034317,
            0.33056944608688354
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.098333477973938,
          "mean_weights": [
            0.3432988226413727,
            0.3276415765285492,
            0.3290596008300781
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0971107482910156,
          "mean_weights": [
            0.354239284992218,
            0.3338475227355957,
            0.3119131922721863
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.098445177078247,
          "mean_weights": [
            0.33804112672805786,
            0.3361448645591736,
            0.3258139491081238
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.3858705759048462,
          "mean_weights": [
            0.25839337706565857,
            0.24941343069076538,
            0.24149224162101746,
            0.2507009506225586
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.38621187210083,
          "mean_weights": [
            0.2517760992050171,
            0.2535240054130554,
            0.24683426320552826,
            0.24786563217639923
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.385884165763855,
          "mean_weights": [
            0.2573123574256897,
            0.2553749680519104,
            0.24531587958335876,
            0.24199676513671875
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3861134052276611,
          "mean_weights": [
            0.2514210641384125,
            0.2551700472831726,
            0.24895039200782776,
            0.24445852637290955
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.6088017225265503,
          "mean_weights": [
            0.20787756145000458,
            0.20764237642288208,
            0.1969079077243805,
            0.1933688223361969,
            0.19420333206653595
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6093868017196655,
          "mean_weights": [
            0.1996297687292099,
            0.200723797082901,
            0.20193718373775482,
            0.19867070019245148,
            0.199038565158844
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.607616901397705,
          "mean_weights": [
            0.20617185533046722,
            0.2148669958114624,
            0.20431196689605713,
            0.19360044598579407,
            0.18104872107505798
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094070672988892,
          "mean_weights": [
            0.1992160826921463,
            0.20051369071006775,
            0.198721244931221,
            0.19902712106704712,
            0.20252184569835663
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.7898081541061401,
          "mean_weights": [
            0.17206060886383057,
            0.18212765455245972,
            0.17207278311252594,
            0.1640574187040329,
            0.15447911620140076,
            0.15520241856575012
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7917404174804688,
          "mean_weights": [
            0.16550754010677338,
            0.16615179181098938,
            0.16667258739471436,
            0.16704785823822021,
            0.16769331693649292,
            0.16692689061164856
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7875205278396606,
          "mean_weights": [
            0.1741587072610855,
            0.18743160367012024,
            0.17662332952022552,
            0.16453337669372559,
            0.15226861834526062,
            0.14498436450958252
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7916977405548096,
          "mean_weights": [
            0.1655852198600769,
            0.16538481414318085,
            0.16465555131435394,
            0.16688036918640137,
            0.16788282990455627,
            0.16961121559143066
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.941636562347412,
          "mean_weights": [
            0.15022630989551544,
            0.1620834320783615,
            0.15417584776878357,
            0.14502081274986267,
            0.1326843798160553,
            0.12827157974243164,
            0.12753766775131226
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9458396434783936,
          "mean_weights": [
            0.14176656305789948,
            0.14105038344860077,
            0.14134296774864197,
            0.14280946552753448,
            0.145027756690979,
            0.14322641491889954,
            0.14477643370628357
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9395363330841064,
          "mean_weights": [
            0.1491325944662094,
            0.16522708535194397,
            0.15644297003746033,
            0.1476437896490097,
            0.13585588335990906,
            0.12666445970535278,
            0.11903323233127594
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9456722736358643,
          "mean_weights": [
            0.14162756502628326,
            0.1389600932598114,
            0.139722540974617,
            0.1427886188030243,
            0.14271582663059235,
            0.14634007215499878,
            0.14784526824951172
          ],
          "sources": 7
        },
        {
          "entropy_mean": 2.0722482204437256,
          "mean_weights": [
            0.13311916589736938,
            0.1474711298942566,
            0.1391790360212326,
            0.13093729317188263,
            0.12214824557304382,
            0.11368396133184433,
            0.1066313236951828,
            0.10682985186576843
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079228401184082,
          "mean_weights": [
            0.12360984086990356,
            0.12237860262393951,
            0.12229418754577637,
            0.12343960255384445,
            0.1245395839214325,
            0.12723875045776367,
            0.12765389680862427,
            0.12884552776813507
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0708298683166504,
          "mean_weights": [
            0.13347145915031433,
            0.14857617020606995,
            0.14165782928466797,
            0.1326693594455719,
            0.12101326882839203,
            0.11364143341779709,
            0.10599445551633835,
            0.10297602415084839
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079227924346924,
          "mean_weights": [
            0.12400860339403152,
            0.12165012210607529,
            0.12233424186706543,
            0.12391965091228485,
            0.12502582371234894,
            0.12579283118247986,
            0.12804390490055084,
            0.12922482192516327
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.1904914379119873,
          "mean_weights": [
            0.1202651709318161,
            0.13243305683135986,
            0.1248447597026825,
            0.11750722676515579,
            0.10865370184183121,
            0.1034422218799591,
            0.09818274527788162,
            0.09678354859352112,
            0.09788758307695389
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1969027519226074,
          "mean_weights": [
            0.10986955463886261,
            0.108255535364151,
            0.10776711255311966,
            0.10903829336166382,
            0.11005579680204391,
            0.11137478053569794,
            0.11346087604761124,
            0.11412093043327332,
            0.1160571351647377
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1876320838928223,
          "mean_weights": [
            0.12014202773571014,
            0.1339646577835083,
            0.12840712070465088,
            0.12060266733169556,
            0.11135218292474747,
            0.10444460809230804,
            0.09650979936122894,
            0.09281164407730103,
            0.09176531434059143
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.028673440217971802,
        0.01493624784052372,
        0.015959296375513077,
        0.015908053144812584,
        0.017468810081481934,
        0.011937789618968964,
        0.013394257053732872,
        0.013926870189607143,
        0.015381631441414356,
        0.011804917827248573,
        0.013283023610711098,
        0.013736946508288383,
        0.015293780714273453,
        0.01275910809636116,
        0.014138885773718357,
        0.014833617955446243,
        0.01643991842865944,
        0.014291302300989628,
        0.01566910557448864,
        0.016281748190522194,
        0.01796886697411537,
        0.016028419137001038,
        0.017591506242752075,
        0.01816963590681553,
        0.020232705399394035,
        0.018053309991955757,
        0.019960105419158936,
        0.01998855359852314,
        0.02254442311823368,
        0.020374728366732597,
        0.022746482864022255,
        0.022396961227059364
      ],
      "loss_nats": 3.531222343444824,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 2.1935343742370605,
        "mean_weights": [
          0.0999753400683403,
          0.09610259532928467,
          0.10262835025787354,
          0.1083034798502922,
          0.11600279062986374,
          0.11817552894353867,
          0.12095586210489273,
          0.11944535374641418,
          0.11841070652008057
        ],
        "sources": 9
      },
      "step": 100,
      "stream_state_rms_by_sublayer": [
        0.004319524392485619,
        0.00972363818436861,
        0.010906110517680645,
        0.018413539975881577,
        0.005452096462249756,
        0.011301152408123016,
        0.013437297195196152,
        0.020593363791704178,
        0.0060663470067083836,
        0.012286243960261345,
        0.014960306696593761,
        0.022828614339232445,
        0.007839187048375607,
        0.014672646299004555,
        0.020714404061436653,
        0.02904798462986946,
        0.01079322025179863,
        0.01802690140902996,
        0.026133937761187553,
        0.03463379293680191,
        0.013178177177906036,
        0.0215163491666317,
        0.03426522761583328,
        0.04299422353506088,
        0.016437800601124763,
        0.024622667580842972,
        0.03885738551616669,
        0.048123423010110855,
        0.022778941318392754,
        0.032037004828453064,
        0.050110094249248505,
        0.06016550958156586
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.00015777233056724072,
        0.00015884912863839418,
        0.00014271712279878557,
        0.0001244439627043903,
        0.00010494918387848884,
        9.392325591761619e-05,
        8.538185647921637e-05,
        7.606096187373623e-05,
        7.664377335458994e-05,
        6.531321560032666e-05,
        6.149107503006235e-05,
        5.037344453739934e-05,
        4.3381573050282896e-05,
        3.9420847315341234e-05,
        3.5541354009183124e-05,
        3.5066059353994206e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.0001459456361772027,
        "first_to_last_ratio": 3.805378902507801,
        "imbalance_abs_log_ratio": 1.3364155665156117,
        "last_quartile_mean": 3.8352458432200365e-05,
        "mean": 8.445807156931551e-05,
        "normalized": [
          1.8680550909542795,
          1.8808045896243943,
          1.6897985017531016,
          1.4734407309105797,
          1.2426187566022755,
          1.1120696242813515,
          1.0109377930698158,
          0.9005765874172524,
          0.907477188745515,
          0.7733211804004252,
          0.7280662923921495,
          0.5964313842526867,
          0.513646265468887,
          0.46675050214695213,
          0.42081654658683526,
          0.4151889653934991
        ],
        "population_cv": 0.4877143313799778
      },
      "activation_output_rms_by_block": [
        0.025262881070375443,
        0.04275903478264809,
        0.027615655213594437,
        0.03926205262541771,
        0.023017436265945435,
        0.03517371043562889,
        0.029550904408097267,
        0.04735429957509041,
        0.028369002044200897,
        0.04305245727300644,
        0.030713967978954315,
        0.05133935809135437,
        0.032636724412441254,
        0.06983363628387451,
        0.0380333848297596,
        0.08374716341495514
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.03372490592300892,
        "first_to_last_ratio": 0.601556641750376,
        "imbalance_abs_log_ratio": 0.508234580502337,
        "last_quartile_mean": 0.056062727235257626,
        "mean": 0.04048260429408401,
        "normalized": [
          0.6240428823910242,
          1.056232313317272,
          0.6821610342304507,
          0.9698499716125064,
          0.5685759764548824,
          0.868859873246076,
          0.7299654981044719,
          1.1697443976451536,
          0.7007701836106094,
          1.0634804263148154,
          0.7586954573335775,
          1.2681831860026092,
          0.8061913254234652,
          1.7250282560027828,
          0.939499458914941,
          2.0687197593953623
        ],
        "population_cv": 0.3940384688106746
      },
      "bits_per_byte": 3.0824777281177833,
      "branch_output_rms_by_sublayer": [
        0.014225655235350132,
        0.01885763369500637,
        0.016379276290535927,
        0.015279849991202354,
        0.023593612015247345,
        0.011461553163826466,
        0.02286384254693985,
        0.009474686346948147,
        0.01935836300253868,
        0.010116237215697765,
        0.017970046028494835,
        0.009506942704319954,
        0.022477619349956512,
        0.013764957897365093,
        0.025794917717576027,
        0.012601078487932682,
        0.02379908226430416,
        0.014191905967891216,
        0.0178237184882164,
        0.0165114663541317,
        0.023045850917696953,
        0.019447455182671547,
        0.020932989194989204,
        0.02472713589668274,
        0.017093855887651443,
        0.028764601796865463,
        0.021727578714489937,
        0.03578802943229675,
        0.008686712011694908,
        0.036414168775081635,
        0.008637740276753902,
        0.04455644637346268
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0006504433508330581,
        0.00048523602227392907,
        0.0003980638932136608,
        0.0004387300445530515,
        0.000365155189968218,
        0.0002788779263831531,
        0.00029885208568229324,
        0.00026779671325771547,
        0.0002902285411340871,
        0.00023612702925164195,
        0.00026277969610552434,
        0.0002548116213942568,
        0.0001837802586554031,
        0.00016271499692659642,
        0.00013374164438063855,
        0.00014095740133328614
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.0004931183277184248,
        "first_to_last_ratio": 3.1752920249892207,
        "imbalance_abs_log_ratio": 1.1553996044896453,
        "last_quartile_mean": 0.00015529857532398106,
        "mean": 0.0003030185259591571,
        "normalized": [
          2.1465464818501867,
          1.6013411085609086,
          1.3136619022010376,
          1.4478654173513685,
          1.20505896071,
          0.9203329251913204,
          0.9862502127100129,
          0.8837635006310174,
          0.9577914096684838,
          0.7792494815431475,
          0.8672067005597657,
          0.8409110320489186,
          0.6064984247206531,
          0.536980359242221,
          0.4413645797968972,
          0.46517750321406204
        ],
        "population_cv": 0.440031122144889
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.6679806113243103,
          "mean_weights": [
            0.4191037714481354,
            0.580896258354187
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6774038076400757,
          "mean_weights": [
            0.4775225520133972,
            0.522477388381958
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.686615526676178,
          "mean_weights": [
            0.49171629548072815,
            0.5082836747169495
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6786561012268066,
          "mean_weights": [
            0.48390668630599976,
            0.516093373298645
          ],
          "sources": 2
        },
        {
          "entropy_mean": 1.0939385890960693,
          "mean_weights": [
            0.32296979427337646,
            0.3461979627609253,
            0.33083218336105347
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0876491069793701,
          "mean_weights": [
            0.3022151589393616,
            0.3410899341106415,
            0.35669487714767456
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.094268560409546,
          "mean_weights": [
            0.3278208076953888,
            0.3347434997558594,
            0.33743569254875183
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0785255432128906,
          "mean_weights": [
            0.30184268951416016,
            0.3319981098175049,
            0.3661591410636902
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.3751009702682495,
          "mean_weights": [
            0.2335343211889267,
            0.2411448359489441,
            0.23957431316375732,
            0.2857465147972107
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3803684711456299,
          "mean_weights": [
            0.23392163217067719,
            0.24959179759025574,
            0.25211477279663086,
            0.264371782541275
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.377173662185669,
          "mean_weights": [
            0.2529916763305664,
            0.22492024302482605,
            0.24044713377952576,
            0.2816409468650818
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3786447048187256,
          "mean_weights": [
            0.23216357827186584,
            0.24167919158935547,
            0.2545858919620514,
            0.2715713381767273
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.6062697172164917,
          "mean_weights": [
            0.20799711346626282,
            0.1897224485874176,
            0.18646025657653809,
            0.19469907879829407,
            0.22112110257148743
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6025722026824951,
          "mean_weights": [
            0.20026245713233948,
            0.18544277548789978,
            0.1946466565132141,
            0.19751906394958496,
            0.22212904691696167
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6060729026794434,
          "mean_weights": [
            0.20172038674354553,
            0.20138651132583618,
            0.18545961380004883,
            0.19016142189502716,
            0.2212720513343811
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.601726770401001,
          "mean_weights": [
            0.20073023438453674,
            0.1886606216430664,
            0.19257165491580963,
            0.19004762172698975,
            0.22798986732959747
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.788693904876709,
          "mean_weights": [
            0.16298983991146088,
            0.17133885622024536,
            0.15878048539161682,
            0.15357322990894318,
            0.17383432388305664,
            0.17948326468467712
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7880157232284546,
          "mean_weights": [
            0.16720382869243622,
            0.15951547026634216,
            0.1605137139558792,
            0.15542668104171753,
            0.17394030094146729,
            0.1834000051021576
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7797719240188599,
          "mean_weights": [
            0.15782463550567627,
            0.17797376215457916,
            0.15211433172225952,
            0.14025422930717468,
            0.16258476674556732,
            0.20924828946590424
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7884608507156372,
          "mean_weights": [
            0.16717270016670227,
            0.17295360565185547,
            0.1665436029434204,
            0.1547294557094574,
            0.1560850888490677,
            0.18251556158065796
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.927689790725708,
          "mean_weights": [
            0.1226624995470047,
            0.15493766963481903,
            0.12924720346927643,
            0.12375974655151367,
            0.1207568496465683,
            0.16289259493350983,
            0.18574345111846924
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9359079599380493,
          "mean_weights": [
            0.1374971866607666,
            0.15181128680706024,
            0.14001096785068512,
            0.13405972719192505,
            0.11918757855892181,
            0.14624495804309845,
            0.17118829488754272
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9246108531951904,
          "mean_weights": [
            0.12061166763305664,
            0.16360490024089813,
            0.1358117014169693,
            0.1262153834104538,
            0.1155187338590622,
            0.15234649181365967,
            0.18589113652706146
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.937231183052063,
          "mean_weights": [
            0.1359395533800125,
            0.16017206013202667,
            0.1499013900756836,
            0.1417488306760788,
            0.12114370614290237,
            0.13951142132282257,
            0.15158303081989288
          ],
          "sources": 7
        },
        {
          "entropy_mean": 2.052294969558716,
          "mean_weights": [
            0.0988478884100914,
            0.15235479176044464,
            0.12090037763118744,
            0.11009317636489868,
            0.09369668364524841,
            0.12056069076061249,
            0.13413047790527344,
            0.1694159209728241
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.061406135559082,
          "mean_weights": [
            0.1149308905005455,
            0.14702443778514862,
            0.13390076160430908,
            0.12344712018966675,
            0.0966324657201767,
            0.11953848600387573,
            0.1299745738506317,
            0.1345512568950653
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0511577129364014,
          "mean_weights": [
            0.09969542920589447,
            0.16962948441505432,
            0.1319108009338379,
            0.11863738298416138,
            0.09143088757991791,
            0.12074249982833862,
            0.14481820166110992,
            0.12313534319400787
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.069282054901123,
          "mean_weights": [
            0.12384641170501709,
            0.13645249605178833,
            0.13362589478492737,
            0.12747827172279358,
            0.10400287806987762,
            0.11596200615167618,
            0.12676866352558136,
            0.13186340034008026
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.165989875793457,
          "mean_weights": [
            0.08691617846488953,
            0.1522500216960907,
            0.1198691874742508,
            0.10617777705192566,
            0.08211278915405273,
            0.11005747318267822,
            0.12842655181884766,
            0.08939606696367264,
            0.12479399144649506
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1907525062561035,
          "mean_weights": [
            0.1032973974943161,
            0.12799270451068878,
            0.11672255396842957,
            0.11367214471101761,
            0.10446079820394516,
            0.11008220911026001,
            0.12035597860813141,
            0.1041758805513382,
            0.09924033284187317
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1538312435150146,
          "mean_weights": [
            0.09009353071451187,
            0.16798722743988037,
            0.13133394718170166,
            0.11392836272716522,
            0.08597822487354279,
            0.10755141079425812,
            0.13270318508148193,
            0.09302861988544464,
            0.07739552855491638
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.04356693848967552,
        0.020052054896950722,
        0.025759555399417877,
        0.02808709815144539,
        0.03107464499771595,
        0.023514779284596443,
        0.025064069777727127,
        0.026588447391986847,
        0.02779453620314598,
        0.02102271094918251,
        0.022771678864955902,
        0.023088837042450905,
        0.024504151195287704,
        0.02003801427781582,
        0.021661879494786263,
        0.021999599412083626,
        0.02357223629951477,
        0.01831224001944065,
        0.01920042186975479,
        0.01775558479130268,
        0.01933184079825878,
        0.014507194049656391,
        0.0161737073212862,
        0.015147161670029163,
        0.017492808401584625,
        0.01415062416344881,
        0.01712540164589882,
        0.017588311806321144,
        0.021003147587180138,
        0.01741618849337101,
        0.020855708047747612,
        0.02042423188686371
      ],
      "loss_nats": 2.136610746383667,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 2.1821343898773193,
        "mean_weights": [
          0.12100096046924591,
          0.09246665984392166,
          0.10230658948421478,
          0.10203823447227478,
          0.09980043023824692,
          0.10532397031784058,
          0.11232466995716095,
          0.1358221024274826,
          0.12891638278961182
        ],
        "sources": 9
      },
      "step": 500,
      "stream_state_rms_by_sublayer": [
        0.014225655235350132,
        0.025262881070375443,
        0.03417503461241722,
        0.04275903478264809,
        0.023593612015247345,
        0.027615655213594437,
        0.035288337618112564,
        0.03926205262541771,
        0.01935836300253868,
        0.023017436265945435,
        0.030542491003870964,
        0.03517371043562889,
        0.022477619349956512,
        0.029550904408097267,
        0.04085070639848709,
        0.04735429957509041,
        0.02379908226430416,
        0.028369002044200897,
        0.03469245135784149,
        0.04305245727300644,
        0.023045850917696953,
        0.030713967978954315,
        0.037714507430791855,
        0.05133935809135437,
        0.017093855887651443,
        0.032636724412441254,
        0.041826508939266205,
        0.06983363628387451,
        0.008686712011694908,
        0.0380333848297596,
        0.04225953668355942,
        0.08374716341495514
      ]
    },
    {
      "activation_grad_rms_by_block": [
        7.445879600709304e-05,
        7.514825847465545e-05,
        8.149984205374494e-05,
        7.616886432515457e-05,
        7.93620856711641e-05,
        7.107412966433913e-05,
        6.609321280848235e-05,
        6.663892418146133e-05,
        0.00013160602247808129,
        0.00010466711682965979,
        0.00017213824321515858,
        0.0001231937640113756,
        0.00011169029312441126,
        7.821891631465405e-05,
        5.0732542149489745e-05,
        4.2012750782305375e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 7.6818940215162e-05,
        "first_to_last_ratio": 1.087107257387618,
        "imbalance_abs_log_ratio": 0.08352027612092149,
        "last_quartile_mean": 7.066362559271511e-05,
        "mean": 8.779398513070191e-05,
        "normalized": [
          0.8481081693266763,
          0.8559613550151489,
          0.9283078098392884,
          0.8675863638238926,
          0.9039581191469443,
          0.8095557976839601,
          0.752821650709751,
          0.7590374680253287,
          1.499032334415107,
          1.1921900648869996,
          1.9607065672994624,
          1.403214170401,
          1.2721861635297007,
          0.8909370749967309,
          0.5778589737550069,
          0.47853791714500205
        ],
        "population_cv": 0.365612445298668
      },
      "activation_output_rms_by_block": [
        0.10262583196163177,
        0.17122715711593628,
        0.1563396155834198,
        0.23706908524036407,
        0.04811444878578186,
        0.07098451256752014,
        0.05486894026398659,
        0.08282698690891266,
        0.05263432487845421,
        0.09547916799783707,
        0.05760422348976135,
        0.09091533720493317,
        0.05383981764316559,
        0.1016460508108139,
        0.07459317892789841,
        0.18281812965869904
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.16681542247533798,
        "first_to_last_ratio": 1.6160480792916094,
        "imbalance_abs_log_ratio": 0.4799837116932076,
        "last_quartile_mean": 0.10322429426014423,
        "mean": 0.10209917556494474,
        "normalized": [
          1.005158282559804,
          1.6770669906831872,
          1.531252477978855,
          2.3219490650006835,
          0.4712520793586283,
          0.6952505950683928,
          0.5374082597668455,
          0.8112405066016116,
          0.5155215464494499,
          0.9351610085930936,
          0.5641987133688422,
          0.8904610316574242,
          0.5273286228341615,
          0.9955619156411052,
          0.7305953110311854,
          1.7905935934067303
        ],
        "population_cv": 0.5286889875537891
      },
      "bits_per_byte": 2.0839361382492645,
      "branch_output_rms_by_sublayer": [
        0.05184043571352959,
        0.0848206952214241,
        0.0633782222867012,
        0.04741951450705528,
        0.15336942672729492,
        0.017866143956780434,
        0.08626767247915268,
        0.013284381479024887,
        0.043669264763593674,
        0.012495848350226879,
        0.03683627396821976,
        0.02719804272055626,
        0.04707624390721321,
        0.01943410001695156,
        0.031199999153614044,
        0.02732040360569954,
        0.04137895256280899,
        0.029010025784373283,
        0.04094765707850456,
        0.04834925755858421,
        0.03695840761065483,
        0.041233282536268234,
        0.026400581002235413,
        0.05263383314013481,
        0.021495802327990532,
        0.04698725789785385,
        0.018388623371720314,
        0.06700151413679123,
        0.015488670207560062,
        0.0718570277094841,
        0.015049737878143787,
        0.12199355661869049
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0003583741677462985,
        0.0002652231972340971,
        0.00033417461006948805,
        0.0002275253539728081,
        0.0001893834795064905,
        0.00020636019253143534,
        0.0002096112726133392,
        0.0001923796657115476,
        0.00047635112247231166,
        0.000565001794959564,
        0.0009549288128880833,
        0.0007263576654831343,
        0.0005471283815528999,
        0.0004183078461688176,
        0.0002607600761312772,
        0.0002482750049477455
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00029632433225567294,
        "first_to_last_ratio": 0.8038795478406102,
        "imbalance_abs_log_ratio": 0.21830583714537038,
        "last_quartile_mean": 0.00036861782720018504,
        "mean": 0.0003862589152493336,
        "normalized": [
          0.9278081452565691,
          0.6866461504529756,
          0.8651570148323317,
          0.5890488089470723,
          0.49030189862217616,
          0.5342535392308757,
          0.5426703807678671,
          0.49805883596852335,
          1.2332430493282536,
          1.462754056032209,
          2.472250542804088,
          1.8804942405387945,
          1.4164809146210218,
          1.0829726632945058,
          0.6750914110628468,
          0.6427683482398892
        ],
        "population_cv": 0.550745478202196
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.4576239287853241,
          "mean_weights": [
            0.8160475492477417,
            0.18395249545574188
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.622429370880127,
          "mean_weights": [
            0.626032829284668,
            0.37396717071533203
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.5034195184707642,
          "mean_weights": [
            0.7919335961341858,
            0.20806637406349182
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6750276684761047,
          "mean_weights": [
            0.4605996012687683,
            0.5394004583358765
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.9881983399391174,
          "mean_weights": [
            0.5066143870353699,
            0.2128569334745407,
            0.2805286943912506
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0713173151016235,
          "mean_weights": [
            0.2439693659543991,
            0.33285707235336304,
            0.42317354679107666
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0050503015518188,
          "mean_weights": [
            0.46526122093200684,
            0.19465598464012146,
            0.3400827646255493
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.091313123703003,
          "mean_weights": [
            0.29163461923599243,
            0.3559022545814514,
            0.35246312618255615
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.3032760620117188,
          "mean_weights": [
            0.25388818979263306,
            0.13041767477989197,
            0.27188149094581604,
            0.34381261467933655
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3785908222198486,
          "mean_weights": [
            0.2572174668312073,
            0.24623332917690277,
            0.2131483256816864,
            0.28340083360671997
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.2377310991287231,
          "mean_weights": [
            0.36895960569381714,
            0.12289825826883316,
            0.12992563843727112,
            0.3782165050506592
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.370632529258728,
          "mean_weights": [
            0.2267947793006897,
            0.24114219844341278,
            0.21443253755569458,
            0.31763046979904175
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.5590848922729492,
          "mean_weights": [
            0.22265344858169556,
            0.11684703826904297,
            0.1529940664768219,
            0.24088799953460693,
            0.26661744713783264
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5987765789031982,
          "mean_weights": [
            0.19780412316322327,
            0.19732677936553955,
            0.1696196049451828,
            0.187029629945755,
            0.24821987748146057
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5600793361663818,
          "mean_weights": [
            0.19789859652519226,
            0.13498559594154358,
            0.15099790692329407,
            0.2153477668762207,
            0.300770103931427
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6049537658691406,
          "mean_weights": [
            0.21525533497333527,
            0.2069544643163681,
            0.18156641721725464,
            0.19566501677036285,
            0.20055876672267914
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.7536540031433105,
          "mean_weights": [
            0.1706119179725647,
            0.12355935573577881,
            0.11797749251127243,
            0.17735670506954193,
            0.15967294573783875,
            0.250821590423584
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7486846446990967,
          "mean_weights": [
            0.1656067669391632,
            0.16547001898288727,
            0.14594469964504242,
            0.13893218338489532,
            0.11710066348314285,
            0.26694566011428833
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.502744197845459,
          "mean_weights": [
            0.14790554344654083,
            0.12157414853572845,
            0.05193060636520386,
            0.11843410134315491,
            0.09838350117206573,
            0.4617721140384674
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7703553438186646,
          "mean_weights": [
            0.17112275958061218,
            0.18443143367767334,
            0.1734921634197235,
            0.14492332935333252,
            0.11953528970479965,
            0.2064950168132782
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.6120144128799438,
          "mean_weights": [
            0.08869323134422302,
            0.07363369315862656,
            0.0430244579911232,
            0.08643164485692978,
            0.0696420818567276,
            0.21115350723266602,
            0.4274213910102844
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.8458226919174194,
          "mean_weights": [
            0.12519317865371704,
            0.13179585337638855,
            0.1208491250872612,
            0.10168728232383728,
            0.08638136088848114,
            0.13217970728874207,
            0.3019134998321533
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.5814783573150635,
          "mean_weights": [
            0.09110043942928314,
            0.07703252136707306,
            0.04712430387735367,
            0.08346284925937653,
            0.06837025284767151,
            0.1662226915359497,
            0.4666869342327118
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.864192008972168,
          "mean_weights": [
            0.13281965255737305,
            0.1363598257303238,
            0.09730516374111176,
            0.10794492065906525,
            0.09621791541576385,
            0.14952389895915985,
            0.27982860803604126
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.7858508825302124,
          "mean_weights": [
            0.0760645866394043,
            0.06870019435882568,
            0.041292984038591385,
            0.06624549627304077,
            0.05510660633444786,
            0.12923449277877808,
            0.3349960446357727,
            0.22835959494113922
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.9164016246795654,
          "mean_weights": [
            0.10010629147291183,
            0.09289439767599106,
            0.06040576845407486,
            0.08482401818037033,
            0.06471984833478928,
            0.12172460556030273,
            0.20661231875419617,
            0.26871275901794434
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.7764818668365479,
          "mean_weights": [
            0.07597766816616058,
            0.07146216928958893,
            0.04444725811481476,
            0.06381020694971085,
            0.052670422941446304,
            0.10254879295825958,
            0.2695360779762268,
            0.3195474147796631
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.9026113748550415,
          "mean_weights": [
            0.09874680638313293,
            0.09077301621437073,
            0.052286967635154724,
            0.08586552739143372,
            0.061087287962436676,
            0.1283089518547058,
            0.24689394235610962,
            0.2360374927520752
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0007686614990234,
          "mean_weights": [
            0.0738106444478035,
            0.07652983069419861,
            0.04215084761381149,
            0.06585545837879181,
            0.053016483783721924,
            0.10676231980323792,
            0.23121783137321472,
            0.19090119004249573,
            0.1597554087638855
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.05625581741333,
          "mean_weights": [
            0.08243454247713089,
            0.09636277705430984,
            0.04735179618000984,
            0.07971908152103424,
            0.057041361927986145,
            0.1233949214220047,
            0.21250130236148834,
            0.17240752279758453,
            0.12878668308258057
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.023190498352051,
          "mean_weights": [
            0.08014722913503647,
            0.08487560600042343,
            0.04445280879735947,
            0.06905653327703476,
            0.057849444448947906,
            0.1067991554737091,
            0.20276135206222534,
            0.2105080783367157,
            0.14354978501796722
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.05538630113005638,
        0.04719197377562523,
        0.05779172107577324,
        0.05863569676876068,
        0.10090170800685883,
        0.07870100438594818,
        0.1107865646481514,
        0.10731016099452972,
        0.12961514294147491,
        0.08656422793865204,
        0.08832070231437683,
        0.06243300065398216,
        0.09176014363765717,
        0.06557498127222061,
        0.0774947926402092,
        0.06959337741136551,
        0.08217275887727737,
        0.054640792310237885,
        0.061008501797914505,
        0.04349073767662048,
        0.07030727714300156,
        0.028683170676231384,
        0.04603966698050499,
        0.03691008314490318,
        0.04661448672413826,
        0.035560719668865204,
        0.03472456708550453,
        0.035963866859674454,
        0.04182035103440285,
        0.0347055159509182,
        0.03997879475355148,
        0.04123349115252495
      ],
      "loss_nats": 1.444474458694458,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 1.9153306484222412,
        "mean_weights": [
          0.0914863795042038,
          0.05659400671720505,
          0.05275696888566017,
          0.06721317768096924,
          0.05635443329811096,
          0.0678911805152893,
          0.08835235983133316,
          0.2647843658924103,
          0.25456711649894714
        ],
        "sources": 9
      },
      "step": 2000,
      "stream_state_rms_by_sublayer": [
        0.05184043571352959,
        0.10262583196163177,
        0.1408688724040985,
        0.17122715711593628,
        0.15336942672729492,
        0.1563396155834198,
        0.2344740480184555,
        0.23706908524036407,
        0.043669264763593674,
        0.04811444878578186,
        0.059994228184223175,
        0.07098451256752014,
        0.04707624390721321,
        0.05486894026398659,
        0.069931261241436,
        0.08282698690891266,
        0.04137895256280899,
        0.05263432487845421,
        0.07298346608877182,
        0.09547916799783707,
        0.03695840761065483,
        0.05760422348976135,
        0.0662730410695076,
        0.09091533720493317,
        0.021495802327990532,
        0.05383981764316559,
        0.058252494782209396,
        0.1016460508108139,
        0.015488670207560062,
        0.07459317892789841,
        0.07778036594390869,
        0.18281812965869904
      ]
    },
    {
      "activation_grad_rms_by_block": [
        5.599572978098877e-05,
        5.6567834690213203e-05,
        7.252454088302329e-05,
        6.816006498411298e-05,
        6.961339386180043e-05,
        6.354664219543338e-05,
        5.091843559057452e-05,
        5.004410559195094e-05,
        0.00011432960309321061,
        8.808868005871773e-05,
        0.0001649916812311858,
        0.00011419985094107687,
        0.0001274511159863323,
        8.248462108895183e-05,
        5.212613905314356e-05,
        4.046529647894204e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 6.331204258458456e-05,
        "first_to_last_ratio": 0.8371088393670095,
        "imbalance_abs_log_ratio": 0.1778011818604578,
        "last_quartile_mean": 7.563179315184243e-05,
        "mean": 7.946923346935364e-05,
        "normalized": [
          0.7046214910652542,
          0.711820565276094,
          0.9126115569114117,
          0.8576912348147676,
          0.8759791786420843,
          0.7996382929745935,
          0.6407314298584572,
          0.6297293104160852,
          1.438664978910366,
          1.1084626869175487,
          2.076170538310438,
          1.4370322444989567,
          1.6037793548805563,
          1.0379440883969395,
          0.6559285496725645,
          0.5091944984538828
        ],
        "population_cv": 0.418216483754139
      },
      "activation_output_rms_by_block": [
        0.16897788643836975,
        0.2546124756336212,
        0.24958282709121704,
        0.3488057255744934,
        0.05895345285534859,
        0.09218393266201019,
        0.05982525274157524,
        0.10435724258422852,
        0.07404384016990662,
        0.13799543678760529,
        0.072679303586483,
        0.12079861760139465,
        0.06610427051782608,
        0.11929895728826523,
        0.08813745528459549,
        0.2320432960987091
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.25549472868442535,
        "first_to_last_ratio": 2.0213831070680737,
        "imbalance_abs_log_ratio": 0.7037819835779293,
        "last_quartile_mean": 0.12639599479734898,
        "mean": 0.1405249983072281,
        "normalized": [
          1.2024756340429583,
          1.8118660644063134,
          1.776074222364032,
          2.4821613931772033,
          0.419522886073689,
          0.6559966822448885,
          0.425726763652258,
          0.7426240444143152,
          0.5269086714950565,
          0.9819992061904,
          0.5171983950327836,
          0.8596236901372816,
          0.47040933153618064,
          0.8489518500291555,
          0.6272012549105437,
          1.6512599102929408
        ],
        "population_cv": 0.5965464726510664
      },
      "bits_per_byte": 1.9646943750306483,
      "branch_output_rms_by_sublayer": [
        0.07875782251358032,
        0.1447884738445282,
        0.07949890196323395,
        0.06182052940130234,
        0.24504326283931732,
        0.02123360149562359,
        0.10607437044382095,
        0.01836485043168068,
        0.053805794566869736,
        0.015964502468705177,
        0.03685670718550682,
        0.03905382379889488,
        0.05105716362595558,
        0.02389056608080864,
        0.045697249472141266,
        0.039935220032930374,
        0.059221696108579636,
        0.040779970586299896,
        0.054663438349962234,
        0.080495685338974,
        0.04156494140625,
        0.05664549022912979,
        0.0355471596121788,
        0.07668386399745941,
        0.026702450588345528,
        0.05748681351542473,
        0.024266516789793968,
        0.08028528094291687,
        0.019828684628009796,
        0.08536289632320404,
        0.0202633123844862,
        0.16591297090053558
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00033760869044818646,
        0.0001988249533593431,
        0.00022180290878374173,
        0.00015787488110689858,
        0.0001349156990162873,
        0.00018159413720443885,
        0.00013882426062051804,
        0.00016817749051027943,
        0.0004737077931871952,
        0.0005939057773758512,
        0.0010653831085878915,
        0.0008603315741687424,
        0.000762827758322017,
        0.0005453805470031765,
        0.00031779802289983946,
        0.00029451637801964313
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00022902785842454247,
        "first_to_last_ratio": 0.47701150875195986,
        "imbalance_abs_log_ratio": 0.7402146610222566,
        "last_quartile_mean": 0.000480130676561169,
        "mean": 0.00040334212378837815,
        "normalized": [
          0.8370280973313858,
          0.49294368634717844,
          0.5499125821534951,
          0.3914167943185893,
          0.33449444295353004,
          0.45022358562210574,
          0.34418488036066147,
          0.41695989729681016,
          1.1744565351565805,
          1.4724615713271154,
          2.6413881559935133,
          2.1330070018179734,
          1.89126727245144,
          1.3521537048516206,
          0.7879118102392371,
          0.7301899817787623
        ],
        "population_cv": 0.6909894412084847
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.27112114429473877,
          "mean_weights": [
            0.9155579805374146,
            0.08444204181432724
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.5555346012115479,
          "mean_weights": [
            0.7120839953422546,
            0.28791600465774536
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.4612463414669037,
          "mean_weights": [
            0.8216869831085205,
            0.1783129870891571
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6770275831222534,
          "mean_weights": [
            0.48782873153686523,
            0.5121712684631348
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.9962381720542908,
          "mean_weights": [
            0.465767502784729,
            0.19465303421020508,
            0.3395794630050659
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0414454936981201,
          "mean_weights": [
            0.22931429743766785,
            0.28697463870048523,
            0.4837110638618469
          ],
          "sources": 3
        },
        {
          "entropy_mean": 0.989799976348877,
          "mean_weights": [
            0.40070968866348267,
            0.17750780284404755,
            0.4217824935913086
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.085202932357788,
          "mean_weights": [
            0.2848654091358185,
            0.3230235278606415,
            0.39211106300354004
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.2675609588623047,
          "mean_weights": [
            0.23358023166656494,
            0.10898500680923462,
            0.35628458857536316,
            0.3011501431465149
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3760833740234375,
          "mean_weights": [
            0.2731572389602661,
            0.2245836853981018,
            0.22189702093601227,
            0.280362069606781
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.2111725807189941,
          "mean_weights": [
            0.41297993063926697,
            0.11292652785778046,
            0.1222275048494339,
            0.3518660068511963
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3647139072418213,
          "mean_weights": [
            0.23418807983398438,
            0.22505050897598267,
            0.2112433910369873,
            0.32951802015304565
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.5535553693771362,
          "mean_weights": [
            0.21889275312423706,
            0.10524161159992218,
            0.1673421710729599,
            0.2413085401058197,
            0.26721489429473877
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5950126647949219,
          "mean_weights": [
            0.2038668543100357,
            0.17965060472488403,
            0.17214898765087128,
            0.19203421473503113,
            0.25229936838150024
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5395398139953613,
          "mean_weights": [
            0.2043805718421936,
            0.12131387740373611,
            0.1313788890838623,
            0.23064203560352325,
            0.3122846484184265
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6000796556472778,
          "mean_weights": [
            0.22543779015541077,
            0.20209266245365143,
            0.18844717741012573,
            0.20001280307769775,
            0.18400955200195312
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.7217694520950317,
          "mean_weights": [
            0.1735105812549591,
            0.1090376153588295,
            0.10670766234397888,
            0.18005813658237457,
            0.14383366703987122,
            0.28685232996940613
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7343032360076904,
          "mean_weights": [
            0.16843369603157043,
            0.15050029754638672,
            0.15599164366722107,
            0.14163106679916382,
            0.10262320935726166,
            0.2808200716972351
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.4446122646331787,
          "mean_weights": [
            0.1531161069869995,
            0.1061544418334961,
            0.04045919328927994,
            0.12559477984905243,
            0.08803897351026535,
            0.4866364896297455
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7605862617492676,
          "mean_weights": [
            0.1779545247554779,
            0.16784369945526123,
            0.1857004165649414,
            0.14843404293060303,
            0.10648007690906525,
            0.21358723938465118
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.5263525247573853,
          "mean_weights": [
            0.08794503659009933,
            0.0601879358291626,
            0.033082835376262665,
            0.08346094191074371,
            0.05888189375400543,
            0.20315150916576385,
            0.4732898473739624
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.7944886684417725,
          "mean_weights": [
            0.12276870757341385,
            0.11536462604999542,
            0.1304849088191986,
            0.09876619279384613,
            0.07423829287290573,
            0.11369137465953827,
            0.3446859121322632
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.4918596744537354,
          "mean_weights": [
            0.09284307807683945,
            0.06384406983852386,
            0.03825954347848892,
            0.08267752081155777,
            0.05671031028032303,
            0.1521964967250824,
            0.5134689807891846
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.830867052078247,
          "mean_weights": [
            0.13634362816810608,
            0.12310001999139786,
            0.09669966250658035,
            0.1085672378540039,
            0.08431269228458405,
            0.13993649184703827,
            0.3110402822494507
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.6549372673034668,
          "mean_weights": [
            0.06966839730739594,
            0.049915459007024765,
            0.029968995600938797,
            0.05769501253962517,
            0.040842454880476,
            0.10442843288183212,
            0.30951738357543945,
            0.33796387910842896
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.859694480895996,
          "mean_weights": [
            0.09671731293201447,
            0.07844845205545425,
            0.05707312002778053,
            0.0829043835401535,
            0.06170058995485306,
            0.10576222091913223,
            0.19003936648368835,
            0.3273545503616333
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.677255392074585,
          "mean_weights": [
            0.07509785890579224,
            0.05865459889173508,
            0.03770594298839569,
            0.059113647788763046,
            0.042673639953136444,
            0.08797852694988251,
            0.24025270342826843,
            0.39852309226989746
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.8695217370986938,
          "mean_weights": [
            0.0984041839838028,
            0.08118931949138641,
            0.05154065042734146,
            0.08640529215335846,
            0.05755901336669922,
            0.11663993448019028,
            0.23645231127738953,
            0.27180927991867065
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.8854753971099854,
          "mean_weights": [
            0.06882007420063019,
            0.0593036487698555,
            0.03391250967979431,
            0.060251303017139435,
            0.038275185972452164,
            0.09127645194530487,
            0.20795494318008423,
            0.23203802108764648,
            0.20816785097122192
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.032858371734619,
          "mean_weights": [
            0.08241330087184906,
            0.08549442887306213,
            0.041513457894325256,
            0.08080939948558807,
            0.054553937166929245,
            0.12166906148195267,
            0.2086121290922165,
            0.17801183462142944,
            0.14692245423793793
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.9540549516677856,
          "mean_weights": [
            0.07821957767009735,
            0.07357673346996307,
            0.037884071469306946,
            0.06657841801643372,
            0.044617801904678345,
            0.09479035437107086,
            0.1742672622203827,
            0.2624226212501526,
            0.16764315962791443
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.058635130524635315,
        0.05468355491757393,
        0.07153555005788803,
        0.0688173845410347,
        0.13985061645507812,
        0.1219802051782608,
        0.16997404396533966,
        0.17499428987503052,
        0.1912512183189392,
        0.14618010818958282,
        0.12251991033554077,
        0.0794699639081955,
        0.12536334991455078,
        0.09043660014867783,
        0.10440115630626678,
        0.08707059919834137,
        0.11446695029735565,
        0.06907524168491364,
        0.0854191705584526,
        0.06048329547047615,
        0.10043052583932877,
        0.03875526040792465,
        0.06678051501512527,
        0.05217510461807251,
        0.06635956466197968,
        0.04468905180692673,
        0.04571816325187683,
        0.04713807255029678,
        0.05554596707224846,
        0.0446290485560894,
        0.052154671400785446,
        0.054206155240535736
      ],
      "loss_nats": 1.3618223667144775,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 1.836374282836914,
        "mean_weights": [
          0.08612492680549622,
          0.0513404943048954,
          0.04786331206560135,
          0.0643724650144577,
          0.0475543811917305,
          0.059286996722221375,
          0.07609974592924118,
          0.2768789529800415,
          0.2904787063598633
        ],
        "sources": 9
      },
      "step": 4000,
      "stream_state_rms_by_sublayer": [
        0.07875782251358032,
        0.16897788643836975,
        0.21303832530975342,
        0.2546124756336212,
        0.24504326283931732,
        0.24958282709121704,
        0.345289409160614,
        0.3488057255744934,
        0.053805794566869736,
        0.05895345285534859,
        0.07887569069862366,
        0.09218393266201019,
        0.05105716362595558,
        0.05982525274157524,
        0.08438026905059814,
        0.10435724258422852,
        0.059221696108579636,
        0.07404384016990662,
        0.09824100881814957,
        0.13799543678760529,
        0.04156494140625,
        0.072679303586483,
        0.08400582522153854,
        0.12079861760139465,
        0.026702450588345528,
        0.06610427051782608,
        0.07228080183267593,
        0.11929895728826523,
        0.019828684628009796,
        0.08813745528459549,
        0.09191317111253738,
        0.2320432960987091
      ]
    },
    {
      "activation_grad_rms_by_block": [
        7.052410364849493e-05,
        7.120969530660659e-05,
        9.769770258571953e-05,
        9.110000246437266e-05,
        9.818621765589342e-05,
        8.837205677991733e-05,
        7.198825915111229e-05,
        6.996823503868654e-05,
        0.00017213664250448346,
        0.00013030946138314903,
        0.0002552570658735931,
        0.00017121800919994712,
        0.00020642777963075787,
        0.00012888041965197772,
        9.31880422285758e-05,
        5.8820565755013376e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 8.263287600129843e-05,
        "first_to_last_ratio": 0.6782682211585493,
        "imbalance_abs_log_ratio": 0.3882124628447938,
        "last_quartile_mean": 0.00012182920181658119,
        "mean": 0.0001172052661786438,
        "normalized": [
          0.6017144617120052,
          0.6075639570500958,
          0.8335606903260565,
          0.7772688500661601,
          0.8377287203651615,
          0.7539939088165287,
          0.6142066947861206,
          0.5969717685896574,
          1.4686766910465736,
          1.1118055155007445,
          2.1778634543991506,
          1.4608388750977905,
          1.7612500390223207,
          1.0996128745233913,
          0.7950840885130448,
          0.5018594101851985
        ],
        "population_cv": 0.46767509062432555
      },
      "activation_output_rms_by_block": [
        0.17702145874500275,
        0.2651609778404236,
        0.2740711271762848,
        0.3786091208457947,
        0.05278070271015167,
        0.09138669073581696,
        0.05598878860473633,
        0.10357969254255295,
        0.06921031326055527,
        0.1325109452009201,
        0.06760682165622711,
        0.11279971897602081,
        0.060408592224121094,
        0.10416899621486664,
        0.073752261698246,
        0.19092032313346863
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.27371567115187645,
        "first_to_last_ratio": 2.5506400527811586,
        "imbalance_abs_log_ratio": 0.9363443287654816,
        "last_quartile_mean": 0.10731254331767559,
        "mean": 0.13812353322282434,
        "normalized": [
          1.2816169309789325,
          1.9197378727103604,
          1.984246426234597,
          2.7410906165788056,
          0.38212679243445447,
          0.661630126333285,
          0.40535300030599286,
          0.7499061899571856,
          0.5010754622740751,
          0.9593654470679528,
          0.4894663500039642,
          0.8166582214056853,
          0.43735191834883375,
          0.7541726871902314,
          0.5339586960845188,
          1.3822432620911254
        ],
        "population_cv": 0.6672200194546598
      },
      "bits_per_byte": 1.8951255009474826,
      "branch_output_rms_by_sublayer": [
        0.08229005336761475,
        0.1513136327266693,
        0.08296028524637222,
        0.06130215525627136,
        0.2681557536125183,
        0.023455867543816566,
        0.11001982539892197,
        0.01849420741200447,
        0.047267504036426544,
        0.01671545021235943,
        0.035008542239665985,
        0.04324181750416756,
        0.04728955402970314,
        0.02490479126572609,
        0.04628673940896988,
        0.041959211230278015,
        0.055535849183797836,
        0.03813145309686661,
        0.04947454109787941,
        0.083168163895607,
        0.037920478731393814,
        0.05371610447764397,
        0.03159874305129051,
        0.07563052326440811,
        0.023936081677675247,
        0.05285703018307686,
        0.021295147016644478,
        0.07199715077877045,
        0.018087781965732574,
        0.0706687644124031,
        0.01923547312617302,
        0.13923443853855133
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0004973569892103084,
        0.0002771506747073045,
        0.0003595853430528865,
        0.00025621221416812444,
        0.0002736875635381804,
        0.00034154014112545435,
        0.0002723491519532143,
        0.0002954443301970981,
        0.0007531989220686374,
        0.0008991377204894639,
        0.0016884293947290697,
        0.0013942943414501116,
        0.0012489939262520248,
        0.0008593069460315814,
        0.000558781441282585,
        0.00045362324645161216
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00034757630528465595,
        "first_to_last_ratio": 0.44550989973264,
        "imbalance_abs_log_ratio": 0.8085358106960822,
        "last_quartile_mean": 0.0007801763900044509,
        "mean": 0.0006518182716692285,
        "normalized": [
          0.7630301432585446,
          0.42519623452339683,
          0.5516650248727019,
          0.39307307773376104,
          0.41988323346214174,
          0.523980618479459,
          0.417829882638547,
          0.45326181090398177,
          1.1555351465367567,
          1.3794300644363342,
          2.5903376264755575,
          2.139084468865058,
          1.916168939317244,
          1.3183228874989945,
          0.8572656913277571,
          0.6959351496697651
        ],
        "population_cv": 0.6710654904340576
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.2534913420677185,
          "mean_weights": [
            0.9232981204986572,
            0.07670190930366516
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.5290617346763611,
          "mean_weights": [
            0.7383309602737427,
            0.26166898012161255
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.4635298252105713,
          "mean_weights": [
            0.8211046457290649,
            0.17889533936977386
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.680541455745697,
          "mean_weights": [
            0.4997203052043915,
            0.5002796649932861
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.9984116554260254,
          "mean_weights": [
            0.45704665780067444,
            0.20008087158203125,
            0.3428725004196167
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0224952697753906,
          "mean_weights": [
            0.22097927331924438,
            0.2672557532787323,
            0.5117650032043457
          ],
          "sources": 3
        },
        {
          "entropy_mean": 0.9886391162872314,
          "mean_weights": [
            0.4091557264328003,
            0.17683157324790955,
            0.41401270031929016
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.083122730255127,
          "mean_weights": [
            0.2858731150627136,
            0.31187689304351807,
            0.4022499620914459
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.262911081314087,
          "mean_weights": [
            0.24318832159042358,
            0.1005454957485199,
            0.3312486708164215,
            0.325017511844635
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3697116374969482,
          "mean_weights": [
            0.2760387659072876,
            0.21072158217430115,
            0.21145658195018768,
            0.30178308486938477
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.1750009059906006,
          "mean_weights": [
            0.40358343720436096,
            0.10160854458808899,
            0.10028939694166183,
            0.3945186138153076
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.355452299118042,
          "mean_weights": [
            0.23967716097831726,
            0.20835967361927032,
            0.20370042324066162,
            0.348262757062912
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.5273280143737793,
          "mean_weights": [
            0.2132018804550171,
            0.09696971625089645,
            0.14153459668159485,
            0.2435910701751709,
            0.3047027587890625
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5883060693740845,
          "mean_weights": [
            0.2065391093492508,
            0.16525766253471375,
            0.16744214296340942,
            0.19437497854232788,
            0.26638609170913696
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.4673986434936523,
          "mean_weights": [
            0.19910070300102234,
            0.10532978177070618,
            0.08866854012012482,
            0.23074698448181152,
            0.37615397572517395
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6001981496810913,
          "mean_weights": [
            0.22463412582874298,
            0.18846800923347473,
            0.19619622826576233,
            0.2022438496351242,
            0.18845780193805695
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.658735752105713,
          "mean_weights": [
            0.16469183564186096,
            0.09157001227140427,
            0.085207998752594,
            0.16954544186592102,
            0.1403937190771103,
            0.34859102964401245
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7217588424682617,
          "mean_weights": [
            0.1685413271188736,
            0.1354280561208725,
            0.15638956427574158,
            0.13712063431739807,
            0.10325069725513458,
            0.29926973581314087
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.383854627609253,
          "mean_weights": [
            0.14582018554210663,
            0.09375129640102386,
            0.03237149864435196,
            0.11780930310487747,
            0.08738291263580322,
            0.522864818572998
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.758126974105835,
          "mean_weights": [
            0.18006329238414764,
            0.15370121598243713,
            0.18850785493850708,
            0.14673401415348053,
            0.10883604735136032,
            0.2221575826406479
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.4768933057785034,
          "mean_weights": [
            0.08639779686927795,
            0.05247560143470764,
            0.02685363218188286,
            0.07686029374599457,
            0.05649450421333313,
            0.2057446390390396,
            0.49517351388931274
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.754528522491455,
          "mean_weights": [
            0.1207476407289505,
            0.10044647008180618,
            0.12684977054595947,
            0.0954352393746376,
            0.07363080978393555,
            0.10486453771591187,
            0.37802553176879883
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.434983491897583,
          "mean_weights": [
            0.09186187386512756,
            0.054585013538599014,
            0.030214153230190277,
            0.07775386422872543,
            0.05580390244722366,
            0.14904344081878662,
            0.5407377481460571
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.814563274383545,
          "mean_weights": [
            0.13872049748897552,
            0.10858511924743652,
            0.09594539552927017,
            0.10938869416713715,
            0.08644531667232513,
            0.13206249475479126,
            0.32885247468948364
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.5878477096557617,
          "mean_weights": [
            0.06617997586727142,
            0.04098459705710411,
            0.023611580953001976,
            0.05256733298301697,
            0.03869973123073578,
            0.09328385442495346,
            0.30499327182769775,
            0.37967967987060547
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.8095037937164307,
          "mean_weights": [
            0.0948307067155838,
            0.0694318562746048,
            0.0552619993686676,
            0.07500732690095901,
            0.061008021235466,
            0.09275255352258682,
            0.18348734080791473,
            0.3682202100753784
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.5925970077514648,
          "mean_weights": [
            0.07132496684789658,
            0.04804965853691101,
            0.030455447733402252,
            0.054331667721271515,
            0.038998860865831375,
            0.07944463938474655,
            0.22967059910297394,
            0.4477241635322571
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.8567713499069214,
          "mean_weights": [
            0.09893427789211273,
            0.07625022530555725,
            0.05004327371716499,
            0.08632857352495193,
            0.05865295231342316,
            0.10661771893501282,
            0.2290721982717514,
            0.2941007912158966
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.7706098556518555,
          "mean_weights": [
            0.0600857213139534,
            0.04508119821548462,
            0.026483699679374695,
            0.05021750181913376,
            0.03222868591547012,
            0.07324853539466858,
            0.17921699583530426,
            0.23399141430854797,
            0.2994462847709656
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.0186803340911865,
          "mean_weights": [
            0.08094437420368195,
            0.07421140372753143,
            0.03662773221731186,
            0.07851459085941315,
            0.05622554570436478,
            0.11645162105560303,
            0.2003173828125,
            0.18591775000095367,
            0.17078962922096252
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.8739352226257324,
          "mean_weights": [
            0.07170331478118896,
            0.05929408222436905,
            0.03108443133533001,
            0.05872419476509094,
            0.03831540048122406,
            0.07797900587320328,
            0.1519450843334198,
            0.2740318179130554,
            0.23692265152931213
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.061437346041202545,
        0.05765428766608238,
        0.07246167957782745,
        0.07205748558044434,
        0.1429983228445053,
        0.13230633735656738,
        0.18651185929775238,
        0.18575440347194672,
        0.20683544874191284,
        0.14580051600933075,
        0.1242423728108406,
        0.07512427866458893,
        0.12742595374584198,
        0.08523519337177277,
        0.10536280274391174,
        0.07712213695049286,
        0.12031000852584839,
        0.06152798980474472,
        0.08708149939775467,
        0.05821346119046211,
        0.10479474812746048,
        0.03660533204674721,
        0.06689487397670746,
        0.049551647156476974,
        0.06684304773807526,
        0.04078952595591545,
        0.04357287660241127,
        0.04368526116013527,
        0.05231116712093353,
        0.03767167031764984,
        0.04748373478651047,
        0.0490146204829216
      ],
      "loss_nats": 1.3136008977890015,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 1.685392141342163,
        "mean_weights": [
          0.08037909120321274,
          0.03995431959629059,
          0.0371873639523983,
          0.05237603187561035,
          0.03903363645076752,
          0.04661020264029503,
          0.0628754124045372,
          0.26517099142074585,
          0.37641292810440063
        ],
        "sources": 9
      },
      "step": 8000,
      "stream_state_rms_by_sublayer": [
        0.08229005336761475,
        0.17702145874500275,
        0.22416120767593384,
        0.2651609778404236,
        0.2681557536125183,
        0.2740711271762848,
        0.37491148710250854,
        0.3786091208457947,
        0.047267504036426544,
        0.05278070271015167,
        0.07517243921756744,
        0.09138669073581696,
        0.04728955402970314,
        0.05598878860473633,
        0.08201033622026443,
        0.10357969254255295,
        0.055535849183797836,
        0.06921031326055527,
        0.0903589129447937,
        0.1325109452009201,
        0.037920478731393814,
        0.06760682165622711,
        0.07624635845422745,
        0.11279971897602081,
        0.023936081677675247,
        0.060408592224121094,
        0.0652509555220604,
        0.10416899621486664,
        0.018087781965732574,
        0.073752261698246,
        0.0771600753068924,
        0.19092032313346863
      ]
    }
  ],
  "environment": {
    "autocast": "cuda-bfloat16-forward-fp32-cross-entropy",
    "compile": false,
    "compute_capability": [
      12,
      0
    ],
    "cublas_workspace_config": ":4096:8",
    "cuda": "12.8",
    "deterministic_algorithms": true,
    "gpu": "NVIDIA GeForce RTX 5090",
    "python": "3.10.14",
    "torch": "2.11.0+cu128"
  },
  "evaluations": [
    {
      "bits_per_byte": 8.073347956844582,
      "cross_entropy_nats": 5.596018373966217,
      "step": 0
    },
    {
      "bits_per_byte": 5.112499455104693,
      "cross_entropy_nats": 3.5437145829200745,
      "step": 100
    },
    {
      "bits_per_byte": 3.1188867992707414,
      "cross_entropy_nats": 2.1618475914001465,
      "step": 500
    },
    {
      "bits_per_byte": 1.962477669145846,
      "cross_entropy_nats": 1.3602858632802963,
      "step": 2000
    },
    {
      "bits_per_byte": 1.8161008917328916,
      "cross_entropy_nats": 1.2588252127170563,
      "step": 4000
    },
    {
      "bits_per_byte": 1.737390666804078,
      "cross_entropy_nats": 1.20426744222641,
      "step": 8000
    }
  ],
  "gradient_gate": null,
  "hashes": {
    "final_mixer_parameters": "0bff629038a5dd94843a123dbdfc2bdaca027f8a9025a2a7688adc4ec0647b31",
    "final_model_state": "efa098925d29c4e5158cd73c7764d7014b6c4ee53df36a146dbd0cff04321b62",
    "final_optimizer_state": "b86655a1b88951524b7f3bfca00df30a0958104385aa444bf7b109fbe1d9e51b",
    "final_public_parameters": "87107ee0633f1863f2fcaf1d5f91af597c67acce2c9777fec3486aa4fd1463ef",
    "initial_mixer_parameters": "64fe0b90f5ea84ed88b6d93838558443c700c2b89ade2455ed526bd8db5d506f",
    "initial_public_parameter_elements": 9541824,
    "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe",
    "initial_public_parameter_tensors": 115,
    "initial_public_parameters": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1"
  },
  "manifest": {
    "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716",
    "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371",
    "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
    "input_gate_tensor_hashes": {
      "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434",
      "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94",
      "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc"
    },
    "path": "experiments/k3/attnres_gradient/manifest.json",
    "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338"
  },
  "model": {
    "attnres_aggregation_groups": 8,
    "context": 256,
    "d_ff": 768,
    "d_head": 32,
    "d_model": 192,
    "heads": 6,
    "layers": 16,
    "parameters": {
      "core": 9541824,
      "embedding": 98304,
      "mixer": 12672,
      "total": 9554496
    },
    "sublayers": 32,
    "sublayers_per_attnres_group": 4,
    "transformer_blocks_per_attnres_group": 2,
    "vocabulary": 256
  },
  "optimizer": {
    "betas": [
      0.9,
      0.95
    ],
    "epsilon": 1e-08,
    "grad_clip": 1.0,
    "min_lr": 3e-05,
    "name": "AdamW",
    "peak_lr": 0.0003,
    "warmup_steps": 400,
    "weight_decay_ndim_ge_2": 0.1
  },
  "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
  "run_kind": "formal",
  "schema_version": 1,
  "seed": 2026073001,
  "steps": 8000,
  "target_bytes_seen": 65536000,
  "timing": {
    "mean_ms": 54.63475258274364,
    "measured_steps": 7980,
    "median_ms": 54.539098986424506,
    "p95_ms": 55.062895112496335,
    "peak_allocated_bytes": 6536921088,
    "peak_reserved_bytes": 6889144320,
    "warmup_steps_excluded": 20
  },
  "training_history": [
    {
      "bits_per_byte": 8.084436624250287,
      "learning_rate": 7.499999999999999e-07,
      "loss_nats": 5.603704452514648,
      "step": 1,
      "unclipped_grad_norm": 18.78145980834961
    },
    {
      "bits_per_byte": 7.484355446790379,
      "learning_rate": 7.499999999999999e-06,
      "loss_nats": 5.187759876251221,
      "step": 10,
      "unclipped_grad_norm": 12.867269515991211
    },
    {
      "bits_per_byte": 6.795359238834398,
      "learning_rate": 1.4999999999999999e-05,
      "loss_nats": 4.710184097290039,
      "step": 20,
      "unclipped_grad_norm": 4.549929141998291
    },
    {
      "bits_per_byte": 6.560912489178218,
      "learning_rate": 2.2499999999999998e-05,
      "loss_nats": 4.547677993774414,
      "step": 30,
      "unclipped_grad_norm": 2.9244894981384277
    },
    {
      "bits_per_byte": 6.418922236764676,
      "learning_rate": 2.9999999999999997e-05,
      "loss_nats": 4.449257850646973,
      "step": 40,
      "unclipped_grad_norm": 2.8415465354919434
    },
    {
      "bits_per_byte": 6.252045406840682,
      "learning_rate": 3.75e-05,
      "loss_nats": 4.333587646484375,
      "step": 50,
      "unclipped_grad_norm": 2.8266561031341553
    },
    {
      "bits_per_byte": 5.950088587414815,
      "learning_rate": 4.4999999999999996e-05,
      "loss_nats": 4.124287128448486,
      "step": 60,
      "unclipped_grad_norm": 2.6689200401306152
    },
    {
      "bits_per_byte": 5.70102021674758,
      "learning_rate": 5.2499999999999995e-05,
      "loss_nats": 3.951646089553833,
      "step": 70,
      "unclipped_grad_norm": 2.5050199031829834
    },
    {
      "bits_per_byte": 5.519442653793135,
      "learning_rate": 5.9999999999999995e-05,
      "loss_nats": 3.8257861137390137,
      "step": 80,
      "unclipped_grad_norm": 2.352375030517578
    },
    {
      "bits_per_byte": 5.302846327435307,
      "learning_rate": 6.749999999999999e-05,
      "loss_nats": 3.6756529808044434,
      "step": 90,
      "unclipped_grad_norm": 2.2169463634490967
    },
    {
      "bits_per_byte": 5.139077911943008,
      "learning_rate": 7.5e-05,
      "loss_nats": 3.5621373653411865,
      "step": 100,
      "unclipped_grad_norm": 2.0478744506835938
    },
    {
      "bits_per_byte": 4.944703463903163,
      "learning_rate": 8.249999999999999e-05,
      "loss_nats": 3.4274072647094727,
      "step": 110,
      "unclipped_grad_norm": 1.971948504447937
    },
    {
      "bits_per_byte": 4.730622899762156,
      "learning_rate": 8.999999999999999e-05,
      "loss_nats": 3.279017925262451,
      "step": 120,
      "unclipped_grad_norm": 1.6964443922042847
    },
    {
      "bits_per_byte": 4.549973699157069,
      "learning_rate": 9.75e-05,
      "loss_nats": 3.153801441192627,
      "step": 130,
      "unclipped_grad_norm": 1.537685751914978
    },
    {
      "bits_per_byte": 4.410896832849143,
      "learning_rate": 0.00010499999999999999,
      "loss_nats": 3.057400703430176,
      "step": 140,
      "unclipped_grad_norm": 1.355125904083252
    },
    {
      "bits_per_byte": 4.261557417798932,
      "learning_rate": 0.0001125,
      "loss_nats": 2.9538865089416504,
      "step": 150,
      "unclipped_grad_norm": 1.2107940912246704
    },
    {
      "bits_per_byte": 4.165243693662742,
      "learning_rate": 0.00011999999999999999,
      "loss_nats": 2.887126922607422,
      "step": 160,
      "unclipped_grad_norm": 0.8971752524375916
    },
    {
      "bits_per_byte": 4.038897327136983,
      "learning_rate": 0.00012749999999999998,
      "loss_nats": 2.7995502948760986,
      "step": 170,
      "unclipped_grad_norm": 0.8723403215408325
    },
    {
      "bits_per_byte": 3.967892225936462,
      "learning_rate": 0.00013499999999999997,
      "loss_nats": 2.750333309173584,
      "step": 180,
      "unclipped_grad_norm": 0.7337902784347534
    },
    {
      "bits_per_byte": 3.8789046187617258,
      "learning_rate": 0.0001425,
      "loss_nats": 2.6886518001556396,
      "step": 190,
      "unclipped_grad_norm": 0.8901073336601257
    },
    {
      "bits_per_byte": 3.824914105176364,
      "learning_rate": 0.00015,
      "loss_nats": 2.651228427886963,
      "step": 200,
      "unclipped_grad_norm": 0.642342746257782
    },
    {
      "bits_per_byte": 3.7744440764541642,
      "learning_rate": 0.0001575,
      "loss_nats": 2.6162452697753906,
      "step": 210,
      "unclipped_grad_norm": 0.6421008110046387
    },
    {
      "bits_per_byte": 3.68308276473222,
      "learning_rate": 0.00016499999999999997,
      "loss_nats": 2.5529184341430664,
      "step": 220,
      "unclipped_grad_norm": 0.7395935654640198
    },
    {
      "bits_per_byte": 3.6743831943209755,
      "learning_rate": 0.0001725,
      "loss_nats": 2.5468883514404297,
      "step": 230,
      "unclipped_grad_norm": 0.6173842549324036
    },
    {
      "bits_per_byte": 3.575872219837362,
      "learning_rate": 0.00017999999999999998,
      "loss_nats": 2.4786057472229004,
      "step": 240,
      "unclipped_grad_norm": 0.8246023058891296
    },
    {
      "bits_per_byte": 3.6039487315560175,
      "learning_rate": 0.0001875,
      "loss_nats": 2.4980669021606445,
      "step": 250,
      "unclipped_grad_norm": 0.7212417721748352
    },
    {
      "bits_per_byte": 3.6017153648519127,
      "learning_rate": 0.000195,
      "loss_nats": 2.496518850326538,
      "step": 260,
      "unclipped_grad_norm": 0.6610386967658997
    },
    {
      "bits_per_byte": 3.6077832567396544,
      "learning_rate": 0.00020249999999999996,
      "loss_nats": 2.5007247924804688,
      "step": 270,
      "unclipped_grad_norm": 0.8672931790351868
    },
    {
      "bits_per_byte": 3.524929582790591,
      "learning_rate": 0.00020999999999999998,
      "loss_nats": 2.4432950019836426,
      "step": 280,
      "unclipped_grad_norm": 0.7419575452804565
    },
    {
      "bits_per_byte": 3.446865969633359,
      "learning_rate": 0.00021749999999999997,
      "loss_nats": 2.3891854286193848,
      "step": 290,
      "unclipped_grad_norm": 0.8831741809844971
    },
    {
      "bits_per_byte": 3.491855255237863,
      "learning_rate": 0.000225,
      "loss_nats": 2.4203696250915527,
      "step": 300,
      "unclipped_grad_norm": 0.9954257607460022
    },
    {
      "bits_per_byte": 3.448469191904702,
      "learning_rate": 0.00023249999999999996,
      "loss_nats": 2.390296697616577,
      "step": 310,
      "unclipped_grad_norm": 1.0782777070999146
    },
    {
      "bits_per_byte": 3.4360496367552953,
      "learning_rate": 0.00023999999999999998,
      "loss_nats": 2.381688117980957,
      "step": 320,
      "unclipped_grad_norm": 0.9428723454475403
    },
    {
      "bits_per_byte": 3.415760155502406,
      "learning_rate": 0.0002475,
      "loss_nats": 2.367624521255493,
      "step": 330,
      "unclipped_grad_norm": 1.0516632795333862
    },
    {
      "bits_per_byte": 3.5107169365232886,
      "learning_rate": 0.00025499999999999996,
      "loss_nats": 2.433443546295166,
      "step": 340,
      "unclipped_grad_norm": 1.1474740505218506
    },
    {
      "bits_per_byte": 3.4490202243180663,
      "learning_rate": 0.0002625,
      "loss_nats": 2.390678644180298,
      "step": 350,
      "unclipped_grad_norm": 1.2037273645401
    },
    {
      "bits_per_byte": 3.401095538828636,
      "learning_rate": 0.00026999999999999995,
      "loss_nats": 2.357459783554077,
      "step": 360,
      "unclipped_grad_norm": 1.0446844100952148
    },
    {
      "bits_per_byte": 3.376710806693402,
      "learning_rate": 0.00027749999999999997,
      "loss_nats": 2.34055757522583,
      "step": 370,
      "unclipped_grad_norm": 0.8781102895736694
    },
    {
      "bits_per_byte": 3.422498435763211,
      "learning_rate": 0.000285,
      "loss_nats": 2.3722951412200928,
      "step": 380,
      "unclipped_grad_norm": 0.9067438244819641
    },
    {
      "bits_per_byte": 3.35829834409203,
      "learning_rate": 0.0002925,
      "loss_nats": 2.3277950286865234,
      "step": 390,
      "unclipped_grad_norm": 1.0105773210525513
    },
    {
      "bits_per_byte": 3.298973960572809,
      "learning_rate": 0.0003,
      "loss_nats": 2.2866744995117188,
      "step": 400,
      "unclipped_grad_norm": 1.1294934749603271
    },
    {
      "bits_per_byte": 3.2943352445137375,
      "learning_rate": 0.0002999988466112388,
      "loss_nats": 2.283459186553955,
      "step": 410,
      "unclipped_grad_norm": 1.2423999309539795
    },
    {
      "bits_per_byte": 3.324363071389082,
      "learning_rate": 0.00029999538646466355,
      "loss_nats": 2.3042728900909424,
      "step": 420,
      "unclipped_grad_norm": 1.016541838645935
    },
    {
      "bits_per_byte": 3.29597423917646,
      "learning_rate": 0.0002999896196193985,
      "loss_nats": 2.284595251083374,
      "step": 430,
      "unclipped_grad_norm": 1.1856372356414795
    },
    {
      "bits_per_byte": 3.3349637380229638,
      "learning_rate": 0.00029998154617398326,
      "loss_nats": 2.3116207122802734,
      "step": 440,
      "unclipped_grad_norm": 0.8890857696533203
    },
    {
      "bits_per_byte": 3.2500166912475987,
      "learning_rate": 0.0002999711662663707,
      "loss_nats": 2.252739906311035,
      "step": 450,
      "unclipped_grad_norm": 1.1027717590332031
    },
    {
      "bits_per_byte": 3.273506081702441,
      "learning_rate": 0.0002999584800739247,
      "loss_nats": 2.269021511077881,
      "step": 460,
      "unclipped_grad_norm": 0.9078090190887451
    },
    {
      "bits_per_byte": 3.199572459923028,
      "learning_rate": 0.0002999434878134173,
      "loss_nats": 2.2177746295928955,
      "step": 470,
      "unclipped_grad_norm": 1.019633173942566
    },
    {
      "bits_per_byte": 3.1991851549932853,
      "learning_rate": 0.000299926189741025,
      "loss_nats": 2.217506170272827,
      "step": 480,
      "unclipped_grad_norm": 0.8520210981369019
    },
    {
      "bits_per_byte": 3.1319430341936423,
      "learning_rate": 0.000299906586152324,
      "loss_nats": 2.1708974838256836,
      "step": 490,
      "unclipped_grad_norm": 2.670640707015991
    },
    {
      "bits_per_byte": 3.0465694704441373,
      "learning_rate": 0.0002998846773822856,
      "loss_nats": 2.1117210388183594,
      "step": 500,
      "unclipped_grad_norm": 0.961890697479248
    },
    {
      "bits_per_byte": 3.080116406321437,
      "learning_rate": 0.0002998604638052702,
      "loss_nats": 2.1349740028381348,
      "step": 510,
      "unclipped_grad_norm": 1.4813798666000366
    },
    {
      "bits_per_byte": 2.9958545384271087,
      "learning_rate": 0.0002998339458350211,
      "loss_nats": 2.076568126678467,
      "step": 520,
      "unclipped_grad_norm": 1.3385756015777588
    },
    {
      "bits_per_byte": 3.0340161127923593,
      "learning_rate": 0.0002998051239246572,
      "loss_nats": 2.1030197143554688,
      "step": 530,
      "unclipped_grad_norm": 1.0723164081573486
    },
    {
      "bits_per_byte": 2.9825217554016206,
      "learning_rate": 0.00029977399856666537,
      "loss_nats": 2.067326545715332,
      "step": 540,
      "unclipped_grad_norm": 1.0559418201446533
    },
    {
      "bits_per_byte": 2.964598755424604,
      "learning_rate": 0.0002997405702928922,
      "loss_nats": 2.054903268814087,
      "step": 550,
      "unclipped_grad_norm": 1.3542453050613403
    },
    {
      "bits_per_byte": 2.9389846913010023,
      "learning_rate": 0.0002997048396745345,
      "loss_nats": 2.037148952484131,
      "step": 560,
      "unclipped_grad_norm": 1.3118830919265747
    },
    {
      "bits_per_byte": 2.888880297694537,
      "learning_rate": 0.0002996668073221302,
      "loss_nats": 2.0024192333221436,
      "step": 570,
      "unclipped_grad_norm": 1.1606483459472656
    },
    {
      "bits_per_byte": 2.9506829512126904,
      "learning_rate": 0.000299626473885547,
      "loss_nats": 2.045257568359375,
      "step": 580,
      "unclipped_grad_norm": 1.3442156314849854
    },
    {
      "bits_per_byte": 2.8292920927888927,
      "learning_rate": 0.00029958384005397226,
      "loss_nats": 1.961115837097168,
      "step": 590,
      "unclipped_grad_norm": 0.9916635751724243
    },
    {
      "bits_per_byte": 2.904369399196543,
      "learning_rate": 0.0002995389065559004,
      "loss_nats": 2.013155460357666,
      "step": 600,
      "unclipped_grad_norm": 1.5107635259628296
    },
    {
      "bits_per_byte": 2.8103640262002334,
      "learning_rate": 0.000299491674159121,
      "loss_nats": 1.947995901107788,
      "step": 610,
      "unclipped_grad_norm": 1.3907541036605835
    },
    {
      "bits_per_byte": 2.8376170849865576,
      "learning_rate": 0.00029944214367070535,
      "loss_nats": 1.966886281967163,
      "step": 620,
      "unclipped_grad_norm": 1.3457273244857788
    },
    {
      "bits_per_byte": 2.772177686333259,
      "learning_rate": 0.00029939031593699284,
      "loss_nats": 1.9215271472930908,
      "step": 630,
      "unclipped_grad_norm": 1.2430039644241333
    },
    {
      "bits_per_byte": 2.7475067750669893,
      "learning_rate": 0.00029933619184357654,
      "loss_nats": 1.9044265747070312,
      "step": 640,
      "unclipped_grad_norm": 1.7454813718795776
    },
    {
      "bits_per_byte": 2.664434339186028,
      "learning_rate": 0.00029927977231528774,
      "loss_nats": 1.8468451499938965,
      "step": 650,
      "unclipped_grad_norm": 1.3669214248657227
    },
    {
      "bits_per_byte": 2.607502578305611,
      "learning_rate": 0.00029922105831618057,
      "loss_nats": 1.8073830604553223,
      "step": 660,
      "unclipped_grad_norm": 2.4166994094848633
    },
    {
      "bits_per_byte": 2.640132846653812,
      "learning_rate": 0.0002991600508495153,
      "loss_nats": 1.830000638961792,
      "step": 670,
      "unclipped_grad_norm": 1.6203631162643433
    },
    {
      "bits_per_byte": 2.577274563624662,
      "learning_rate": 0.00029909675095774114,
      "loss_nats": 1.7864305973052979,
      "step": 680,
      "unclipped_grad_norm": 1.443710207939148
    },
    {
      "bits_per_byte": 2.648628101675831,
      "learning_rate": 0.0002990311597224787,
      "loss_nats": 1.8358891010284424,
      "step": 690,
      "unclipped_grad_norm": 1.1127547025680542
    },
    {
      "bits_per_byte": 2.5587637269469963,
      "learning_rate": 0.00029896327826450103,
      "loss_nats": 1.7735998630523682,
      "step": 700,
      "unclipped_grad_norm": 1.3421220779418945
    },
    {
      "bits_per_byte": 2.584466534118443,
      "learning_rate": 0.0002988931077437152,
      "loss_nats": 1.7914156913757324,
      "step": 710,
      "unclipped_grad_norm": 1.5519825220108032
    },
    {
      "bits_per_byte": 2.61796909847182,
      "learning_rate": 0.00029882064935914164,
      "loss_nats": 1.8146378993988037,
      "step": 720,
      "unclipped_grad_norm": 2.2321839332580566
    },
    {
      "bits_per_byte": 2.574501859327452,
      "learning_rate": 0.0002987459043488943,
      "loss_nats": 1.7845087051391602,
      "step": 730,
      "unclipped_grad_norm": 1.835390567779541
    },
    {
      "bits_per_byte": 2.509336257054327,
      "learning_rate": 0.0002986688739901591,
      "loss_nats": 1.7393393516540527,
      "step": 740,
      "unclipped_grad_norm": 1.5465850830078125
    },
    {
      "bits_per_byte": 2.4645351204698667,
      "learning_rate": 0.0002985895595991723,
      "loss_nats": 1.7082855701446533,
      "step": 750,
      "unclipped_grad_norm": 1.1801629066467285
    },
    {
      "bits_per_byte": 2.4095216540771736,
      "learning_rate": 0.00029850796253119824,
      "loss_nats": 1.6701531410217285,
      "step": 760,
      "unclipped_grad_norm": 1.296120524406433
    },
    {
      "bits_per_byte": 2.584656058999694,
      "learning_rate": 0.00029842408418050544,
      "loss_nats": 1.7915470600128174,
      "step": 770,
      "unclipped_grad_norm": 2.048929452896118
    },
    {
      "bits_per_byte": 2.4472219709728096,
      "learning_rate": 0.0002983379259803436,
      "loss_nats": 1.6962850093841553,
      "step": 780,
      "unclipped_grad_norm": 1.45075261592865
    },
    {
      "bits_per_byte": 2.4007569022412523,
      "learning_rate": 0.0002982494894029186,
      "loss_nats": 1.664077877998352,
      "step": 790,
      "unclipped_grad_norm": 1.5680437088012695
    },
    {
      "bits_per_byte": 2.3632249563608103,
      "learning_rate": 0.0002981587759593675,
      "loss_nats": 1.6380627155303955,
      "step": 800,
      "unclipped_grad_norm": 1.358572006225586
    },
    {
      "bits_per_byte": 2.4340216145884854,
      "learning_rate": 0.00029806578719973283,
      "loss_nats": 1.6871352195739746,
      "step": 810,
      "unclipped_grad_norm": 1.6542795896530151
    },
    {
      "bits_per_byte": 2.3610528154804133,
      "learning_rate": 0.00029797052471293596,
      "loss_nats": 1.6365571022033691,
      "step": 820,
      "unclipped_grad_norm": 1.4101909399032593
    },
    {
      "bits_per_byte": 2.429125956449025,
      "learning_rate": 0.0002978729901267499,
      "loss_nats": 1.683741807937622,
      "step": 830,
      "unclipped_grad_norm": 1.4574730396270752
    },
    {
      "bits_per_byte": 2.509321810511654,
      "learning_rate": 0.0002977731851077716,
      "loss_nats": 1.7393293380737305,
      "step": 840,
      "unclipped_grad_norm": 1.71798837184906
    },
    {
      "bits_per_byte": 2.3876714302677042,
      "learning_rate": 0.0002976711113613935,
      "loss_nats": 1.6550077199935913,
      "step": 850,
      "unclipped_grad_norm": 1.7788947820663452
    },
    {
      "bits_per_byte": 2.402646303643631,
      "learning_rate": 0.0002975667706317743,
      "loss_nats": 1.665387511253357,
      "step": 860,
      "unclipped_grad_norm": 1.3990782499313354
    },
    {
      "bits_per_byte": 2.3188210996998437,
      "learning_rate": 0.00029746016470180915,
      "loss_nats": 1.6072843074798584,
      "step": 870,
      "unclipped_grad_norm": 1.5185577869415283
    },
    {
      "bits_per_byte": 2.342377219423221,
      "learning_rate": 0.00029735129539309926,
      "loss_nats": 1.6236121654510498,
      "step": 880,
      "unclipped_grad_norm": 1.4461203813552856
    },
    {
      "bits_per_byte": 2.2667829331670766,
      "learning_rate": 0.0002972401645659207,
      "loss_nats": 1.571214199066162,
      "step": 890,
      "unclipped_grad_norm": 1.2836962938308716
    },
    {
      "bits_per_byte": 2.284366783356592,
      "learning_rate": 0.00029712677411919283,
      "loss_nats": 1.583402395248413,
      "step": 900,
      "unclipped_grad_norm": 1.337744116783142
    },
    {
      "bits_per_byte": 2.3096057533186616,
      "learning_rate": 0.0002970111259904454,
      "loss_nats": 1.6008967161178589,
      "step": 910,
      "unclipped_grad_norm": 1.3567826747894287
    },
    {
      "bits_per_byte": 2.2867041995644684,
      "learning_rate": 0.0002968932221557859,
      "loss_nats": 1.5850225687026978,
      "step": 920,
      "unclipped_grad_norm": 1.568986177444458
    },
    {
      "bits_per_byte": 2.304558234498489,
      "learning_rate": 0.0002967730646298655,
      "loss_nats": 1.597398042678833,
      "step": 930,
      "unclipped_grad_norm": 1.5530638694763184
    },
    {
      "bits_per_byte": 2.297734650842856,
      "learning_rate": 0.0002966506554658449,
      "loss_nats": 1.5926682949066162,
      "step": 940,
      "unclipped_grad_norm": 1.5203993320465088
    },
    {
      "bits_per_byte": 2.280084587332759,
      "learning_rate": 0.00029652599675535876,
      "loss_nats": 1.5804342031478882,
      "step": 950,
      "unclipped_grad_norm": 1.4065279960632324
    },
    {
      "bits_per_byte": 2.325858113825197,
      "learning_rate": 0.0002963990906284808,
      "loss_nats": 1.6121619939804077,
      "step": 960,
      "unclipped_grad_norm": 1.3770049810409546
    },
    {
      "bits_per_byte": 2.231583072032335,
      "learning_rate": 0.0002962699392536863,
      "loss_nats": 1.5468155145645142,
      "step": 970,
      "unclipped_grad_norm": 1.3417105674743652
    },
    {
      "bits_per_byte": 2.3223450242160224,
      "learning_rate": 0.0002961385448378161,
      "loss_nats": 1.609726905822754,
      "step": 980,
      "unclipped_grad_norm": 1.444604754447937
    },
    {
      "bits_per_byte": 2.2731906627729233,
      "learning_rate": 0.00029600490962603817,
      "loss_nats": 1.5756556987762451,
      "step": 990,
      "unclipped_grad_norm": 1.3634493350982666
    },
    {
      "bits_per_byte": 2.2346108266007745,
      "learning_rate": 0.00029586903590180956,
      "loss_nats": 1.5489141941070557,
      "step": 1000,
      "unclipped_grad_norm": 1.291415810585022
    },
    {
      "bits_per_byte": 2.2324534762283514,
      "learning_rate": 0.00029573092598683735,
      "loss_nats": 1.5474188327789307,
      "step": 1010,
      "unclipped_grad_norm": 1.811213493347168
    },
    {
      "bits_per_byte": 2.198316295893291,
      "learning_rate": 0.00029559058224103895,
      "loss_nats": 1.523756742477417,
      "step": 1020,
      "unclipped_grad_norm": 1.5226773023605347
    },
    {
      "bits_per_byte": 2.1998723949182977,
      "learning_rate": 0.0002954480070625017,
      "loss_nats": 1.5248353481292725,
      "step": 1030,
      "unclipped_grad_norm": 1.3728047609329224
    },
    {
      "bits_per_byte": 2.256233517363152,
      "learning_rate": 0.00029530320288744205,
      "loss_nats": 1.5639019012451172,
      "step": 1040,
      "unclipped_grad_norm": 1.3665989637374878
    },
    {
      "bits_per_byte": 2.2423672441546874,
      "learning_rate": 0.0002951561721901638,
      "loss_nats": 1.554290533065796,
      "step": 1050,
      "unclipped_grad_norm": 1.274046778678894
    },
    {
      "bits_per_byte": 2.1489992388624755,
      "learning_rate": 0.00029500691748301594,
      "loss_nats": 1.4895727634429932,
      "step": 1060,
      "unclipped_grad_norm": 1.2373967170715332
    },
    {
      "bits_per_byte": 2.2260787672914706,
      "learning_rate": 0.00029485544131634963,
      "loss_nats": 1.5430002212524414,
      "step": 1070,
      "unclipped_grad_norm": 1.3982683420181274
    },
    {
      "bits_per_byte": 2.206438348562938,
      "learning_rate": 0.0002947017462784745,
      "loss_nats": 1.5293865203857422,
      "step": 1080,
      "unclipped_grad_norm": 1.334183692932129
    },
    {
      "bits_per_byte": 2.1702291212502822,
      "learning_rate": 0.00029454583499561476,
      "loss_nats": 1.5042881965637207,
      "step": 1090,
      "unclipped_grad_norm": 1.4794831275939941
    },
    {
      "bits_per_byte": 2.2378767771406594,
      "learning_rate": 0.00029438771013186413,
      "loss_nats": 1.551177978515625,
      "step": 1100,
      "unclipped_grad_norm": 1.3411470651626587
    },
    {
      "bits_per_byte": 2.087035609583114,
      "learning_rate": 0.0002942273743891401,
      "loss_nats": 1.4466228485107422,
      "step": 1110,
      "unclipped_grad_norm": 1.452073574066162
    },
    {
      "bits_per_byte": 2.1415479185311934,
      "learning_rate": 0.00029406483050713824,
      "loss_nats": 1.484407901763916,
      "step": 1120,
      "unclipped_grad_norm": 1.1817387342453003
    },
    {
      "bits_per_byte": 2.079985696758946,
      "learning_rate": 0.00029390008126328504,
      "loss_nats": 1.4417362213134766,
      "step": 1130,
      "unclipped_grad_norm": 1.5416163206100464
    },
    {
      "bits_per_byte": 2.2251825656978244,
      "learning_rate": 0.00029373312947269045,
      "loss_nats": 1.5423790216445923,
      "step": 1140,
      "unclipped_grad_norm": 1.5179954767227173
    },
    {
      "bits_per_byte": 2.0668831985029628,
      "learning_rate": 0.00029356397798809995,
      "loss_nats": 1.4326542615890503,
      "step": 1150,
      "unclipped_grad_norm": 1.186457633972168
    },
    {
      "bits_per_byte": 2.104958265594829,
      "learning_rate": 0.0002933926296998457,
      "loss_nats": 1.4590458869934082,
      "step": 1160,
      "unclipped_grad_norm": 1.5302215814590454
    },
    {
      "bits_per_byte": 2.146566716248672,
      "learning_rate": 0.0002932190875357972,
      "loss_nats": 1.487886667251587,
      "step": 1170,
      "unclipped_grad_norm": 1.187764048576355
    },
    {
      "bits_per_byte": 2.1225221657968443,
      "learning_rate": 0.00029304335446131126,
      "loss_nats": 1.4712202548980713,
      "step": 1180,
      "unclipped_grad_norm": 1.1978200674057007
    },
    {
      "bits_per_byte": 2.061979457178911,
      "learning_rate": 0.00029286543347918116,
      "loss_nats": 1.4292552471160889,
      "step": 1190,
      "unclipped_grad_norm": 1.333662509918213
    },
    {
      "bits_per_byte": 2.04061714814994,
      "learning_rate": 0.0002926853276295856,
      "loss_nats": 1.4144480228424072,
      "step": 1200,
      "unclipped_grad_norm": 1.397094964981079
    },
    {
      "bits_per_byte": 2.105093787923709,
      "learning_rate": 0.0002925030399900368,
      "loss_nats": 1.4591398239135742,
      "step": 1210,
      "unclipped_grad_norm": 1.355148196220398
    },
    {
      "bits_per_byte": 2.113284805636352,
      "learning_rate": 0.00029231857367532726,
      "loss_nats": 1.4648174047470093,
      "step": 1220,
      "unclipped_grad_norm": 1.2704886198043823
    },
    {
      "bits_per_byte": 2.0731149898569767,
      "learning_rate": 0.0002921319318374776,
      "loss_nats": 1.4369738101959229,
      "step": 1230,
      "unclipped_grad_norm": 1.2257503271102905
    },
    {
      "bits_per_byte": 2.02175856255223,
      "learning_rate": 0.00029194311766568174,
      "loss_nats": 1.4013762474060059,
      "step": 1240,
      "unclipped_grad_norm": 1.311407208442688
    },
    {
      "bits_per_byte": 2.10808387829161,
      "learning_rate": 0.000291752134386253,
      "loss_nats": 1.461212396621704,
      "step": 1250,
      "unclipped_grad_norm": 1.183249831199646
    },
    {
      "bits_per_byte": 2.0709103442555663,
      "learning_rate": 0.00029155898526256856,
      "loss_nats": 1.4354456663131714,
      "step": 1260,
      "unclipped_grad_norm": 1.2188260555267334
    },
    {
      "bits_per_byte": 2.073173119992968,
      "learning_rate": 0.0002913636735950141,
      "loss_nats": 1.437014102935791,
      "step": 1270,
      "unclipped_grad_norm": 1.2204123735427856
    },
    {
      "bits_per_byte": 1.983587529141182,
      "learning_rate": 0.0002911662027209272,
      "loss_nats": 1.3749181032180786,
      "step": 1280,
      "unclipped_grad_norm": 1.4762879610061646
    },
    {
      "bits_per_byte": 2.0899118474361438,
      "learning_rate": 0.00029096657601454013,
      "loss_nats": 1.4486165046691895,
      "step": 1290,
      "unclipped_grad_norm": 1.5250427722930908
    },
    {
      "bits_per_byte": 2.0451232655851967,
      "learning_rate": 0.00029076479688692266,
      "loss_nats": 1.4175714254379272,
      "step": 1300,
      "unclipped_grad_norm": 1.2546813488006592
    },
    {
      "bits_per_byte": 2.037817958524497,
      "learning_rate": 0.00029056086878592323,
      "loss_nats": 1.4125077724456787,
      "step": 1310,
      "unclipped_grad_norm": 1.2255067825317383
    },
    {
      "bits_per_byte": 2.0185500542177333,
      "learning_rate": 0.00029035479519611045,
      "loss_nats": 1.3991522789001465,
      "step": 1320,
      "unclipped_grad_norm": 1.1987844705581665
    },
    {
      "bits_per_byte": 2.097532054730573,
      "learning_rate": 0.0002901465796387134,
      "loss_nats": 1.4538984298706055,
      "step": 1330,
      "unclipped_grad_norm": 1.270599603652954
    },
    {
      "bits_per_byte": 2.036339251692379,
      "learning_rate": 0.00028993622567156154,
      "loss_nats": 1.411482810974121,
      "step": 1340,
      "unclipped_grad_norm": 1.31504225730896
    },
    {
      "bits_per_byte": 2.0528777913299496,
      "learning_rate": 0.0002897237368890237,
      "loss_nats": 1.4229464530944824,
      "step": 1350,
      "unclipped_grad_norm": 1.3823131322860718
    },
    {
      "bits_per_byte": 2.0989251142025616,
      "learning_rate": 0.0002895091169219469,
      "loss_nats": 1.4548640251159668,
      "step": 1360,
      "unclipped_grad_norm": 1.3448033332824707
    },
    {
      "bits_per_byte": 2.081831758533307,
      "learning_rate": 0.00028929236943759425,
      "loss_nats": 1.4430158138275146,
      "step": 1370,
      "unclipped_grad_norm": 1.1824978590011597
    },
    {
      "bits_per_byte": 2.037543130248418,
      "learning_rate": 0.00028907349813958225,
      "loss_nats": 1.4123172760009766,
      "step": 1380,
      "unclipped_grad_norm": 1.241772174835205
    },
    {
      "bits_per_byte": 2.0465340392702243,
      "learning_rate": 0.0002888525067678175,
      "loss_nats": 1.4185492992401123,
      "step": 1390,
      "unclipped_grad_norm": 1.1579062938690186
    },
    {
      "bits_per_byte": 1.983804227281269,
      "learning_rate": 0.00028862939909843273,
      "loss_nats": 1.3750683069229126,
      "step": 1400,
      "unclipped_grad_norm": 1.2093477249145508
    },
    {
      "bits_per_byte": 2.004750510277803,
      "learning_rate": 0.0002884041789437225,
      "loss_nats": 1.389587163925171,
      "step": 1410,
      "unclipped_grad_norm": 1.0995537042617798
    },
    {
      "bits_per_byte": 2.044002282666873,
      "learning_rate": 0.0002881768501520778,
      "loss_nats": 1.4167944192886353,
      "step": 1420,
      "unclipped_grad_norm": 1.377259612083435
    },
    {
      "bits_per_byte": 2.072302027866348,
      "learning_rate": 0.0002879474166079205,
      "loss_nats": 1.4364103078842163,
      "step": 1430,
      "unclipped_grad_norm": 1.2491836547851562
    },
    {
      "bits_per_byte": 2.010299358525241,
      "learning_rate": 0.00028771588223163694,
      "loss_nats": 1.3934333324432373,
      "step": 1440,
      "unclipped_grad_norm": 1.1953747272491455
    },
    {
      "bits_per_byte": 1.9653831655473537,
      "learning_rate": 0.00028748225097951054,
      "loss_nats": 1.3622997999191284,
      "step": 1450,
      "unclipped_grad_norm": 1.1474745273590088
    },
    {
      "bits_per_byte": 2.0178239434657903,
      "learning_rate": 0.000287246526843655,
      "loss_nats": 1.398648977279663,
      "step": 1460,
      "unclipped_grad_norm": 1.309095859527588
    },
    {
      "bits_per_byte": 1.9352492253764266,
      "learning_rate": 0.00028700871385194533,
      "loss_nats": 1.3414125442504883,
      "step": 1470,
      "unclipped_grad_norm": 1.2327831983566284
    },
    {
      "bits_per_byte": 2.02495382822261,
      "learning_rate": 0.00028676881606794945,
      "loss_nats": 1.4035910367965698,
      "step": 1480,
      "unclipped_grad_norm": 1.2556331157684326
    },
    {
      "bits_per_byte": 2.0395678819970264,
      "learning_rate": 0.00028652683759085854,
      "loss_nats": 1.413720726966858,
      "step": 1490,
      "unclipped_grad_norm": 1.1320099830627441
    },
    {
      "bits_per_byte": 2.0169458000504847,
      "learning_rate": 0.0002862827825554171,
      "loss_nats": 1.3980402946472168,
      "step": 1500,
      "unclipped_grad_norm": 1.2224371433258057
    },
    {
      "bits_per_byte": 2.0172811662196675,
      "learning_rate": 0.00028603665513185245,
      "loss_nats": 1.3982727527618408,
      "step": 1510,
      "unclipped_grad_norm": 1.1516478061676025
    },
    {
      "bits_per_byte": 1.959494651564462,
      "learning_rate": 0.000285788459525803,
      "loss_nats": 1.3582181930541992,
      "step": 1520,
      "unclipped_grad_norm": 0.9697362780570984
    },
    {
      "bits_per_byte": 2.05711750763902,
      "learning_rate": 0.00028553819997824707,
      "loss_nats": 1.4258852005004883,
      "step": 1530,
      "unclipped_grad_norm": 1.1719471216201782
    },
    {
      "bits_per_byte": 2.0050322178599163,
      "learning_rate": 0.00028528588076542964,
      "loss_nats": 1.389782428741455,
      "step": 1540,
      "unclipped_grad_norm": 1.2557746171951294
    },
    {
      "bits_per_byte": 2.0500259750133427,
      "learning_rate": 0.00028503150619878996,
      "loss_nats": 1.4209697246551514,
      "step": 1550,
      "unclipped_grad_norm": 1.1871017217636108
    },
    {
      "bits_per_byte": 1.9591589414299777,
      "learning_rate": 0.0002847750806248876,
      "loss_nats": 1.357985496520996,
      "step": 1560,
      "unclipped_grad_norm": 1.1881691217422485
    },
    {
      "bits_per_byte": 2.048241998975943,
      "learning_rate": 0.000284516608425328,
      "loss_nats": 1.4197331666946411,
      "step": 1570,
      "unclipped_grad_norm": 1.3408379554748535
    },
    {
      "bits_per_byte": 2.0003424229535387,
      "learning_rate": 0.000284256094016688,
      "loss_nats": 1.3865317106246948,
      "step": 1580,
      "unclipped_grad_norm": 1.160094141960144
    },
    {
      "bits_per_byte": 1.9972685770346676,
      "learning_rate": 0.0002839935418504399,
      "loss_nats": 1.3844010829925537,
      "step": 1590,
      "unclipped_grad_norm": 1.1695513725280762
    },
    {
      "bits_per_byte": 2.0280931995314595,
      "learning_rate": 0.000283728956412876,
      "loss_nats": 1.4057670831680298,
      "step": 1600,
      "unclipped_grad_norm": 1.1151546239852905
    },
    {
      "bits_per_byte": 1.9648931869750457,
      "learning_rate": 0.00028346234222503136,
      "loss_nats": 1.3619601726531982,
      "step": 1610,
      "unclipped_grad_norm": 1.1052403450012207
    },
    {
      "bits_per_byte": 1.934925382044852,
      "learning_rate": 0.00028319370384260675,
      "loss_nats": 1.3411880731582642,
      "step": 1620,
      "unclipped_grad_norm": 1.0654687881469727
    },
    {
      "bits_per_byte": 1.9529704897039815,
      "learning_rate": 0.00028292304585589105,
      "loss_nats": 1.3536959886550903,
      "step": 1630,
      "unclipped_grad_norm": 1.1012754440307617
    },
    {
      "bits_per_byte": 1.9931929321745208,
      "learning_rate": 0.0002826503728896824,
      "loss_nats": 1.3815760612487793,
      "step": 1640,
      "unclipped_grad_norm": 1.143923282623291
    },
    {
      "bits_per_byte": 2.059868542122221,
      "learning_rate": 0.0002823756896032095,
      "loss_nats": 1.4277920722961426,
      "step": 1650,
      "unclipped_grad_norm": 1.2584717273712158
    },
    {
      "bits_per_byte": 1.992070745377641,
      "learning_rate": 0.0002820990006900518,
      "loss_nats": 1.3807982206344604,
      "step": 1660,
      "unclipped_grad_norm": 1.177032232284546
    },
    {
      "bits_per_byte": 2.009216383772758,
      "learning_rate": 0.00028182031087805947,
      "loss_nats": 1.392682671546936,
      "step": 1670,
      "unclipped_grad_norm": 1.1299933195114136
    },
    {
      "bits_per_byte": 2.027195106128654,
      "learning_rate": 0.0002815396249292725,
      "loss_nats": 1.4051445722579956,
      "step": 1680,
      "unclipped_grad_norm": 1.2568069696426392
    },
    {
      "bits_per_byte": 2.0647518175108197,
      "learning_rate": 0.0002812569476398391,
      "loss_nats": 1.4311769008636475,
      "step": 1690,
      "unclipped_grad_norm": 1.1674096584320068
    },
    {
      "bits_per_byte": 1.9602546428986247,
      "learning_rate": 0.00028097228383993446,
      "loss_nats": 1.3587449789047241,
      "step": 1700,
      "unclipped_grad_norm": 1.178646206855774
    },
    {
      "bits_per_byte": 1.8971180919403792,
      "learning_rate": 0.0002806856383936774,
      "loss_nats": 1.3149820566177368,
      "step": 1710,
      "unclipped_grad_norm": 1.0156482458114624
    },
    {
      "bits_per_byte": 1.8941140709777584,
      "learning_rate": 0.00028039701619904774,
      "loss_nats": 1.3128998279571533,
      "step": 1720,
      "unclipped_grad_norm": 1.0928010940551758
    },
    {
      "bits_per_byte": 1.8690832000232327,
      "learning_rate": 0.0002801064221878024,
      "loss_nats": 1.295549750328064,
      "step": 1730,
      "unclipped_grad_norm": 1.0898301601409912
    },
    {
      "bits_per_byte": 2.0020670649763908,
      "learning_rate": 0.00027981386132539134,
      "loss_nats": 1.38772714138031,
      "step": 1740,
      "unclipped_grad_norm": 1.1916359663009644
    },
    {
      "bits_per_byte": 1.9965376507685006,
      "learning_rate": 0.0002795193386108724,
      "loss_nats": 1.383894443511963,
      "step": 1750,
      "unclipped_grad_norm": 1.1260547637939453
    },
    {
      "bits_per_byte": 2.010018682839033,
      "learning_rate": 0.00027922285907682625,
      "loss_nats": 1.3932387828826904,
      "step": 1760,
      "unclipped_grad_norm": 1.2356810569763184
    },
    {
      "bits_per_byte": 2.000046096846102,
      "learning_rate": 0.00027892442778927005,
      "loss_nats": 1.3863263130187988,
      "step": 1770,
      "unclipped_grad_norm": 1.1323615312576294
    },
    {
      "bits_per_byte": 1.8857641412957182,
      "learning_rate": 0.00027862404984757116,
      "loss_nats": 1.3071120977401733,
      "step": 1780,
      "unclipped_grad_norm": 1.1074084043502808
    },
    {
      "bits_per_byte": 1.98148022572016,
      "learning_rate": 0.0002783217303843597,
      "loss_nats": 1.373457431793213,
      "step": 1790,
      "unclipped_grad_norm": 1.1702499389648438
    },
    {
      "bits_per_byte": 1.8942642118319617,
      "learning_rate": 0.00027801747456544134,
      "loss_nats": 1.3130038976669312,
      "step": 1800,
      "unclipped_grad_norm": 1.2193527221679688
    },
    {
      "bits_per_byte": 1.9583697130451683,
      "learning_rate": 0.00027771128758970843,
      "loss_nats": 1.3574384450912476,
      "step": 1810,
      "unclipped_grad_norm": 1.4279322624206543
    },
    {
      "bits_per_byte": 1.9039834673801714,
      "learning_rate": 0.00027740317468905165,
      "loss_nats": 1.3197407722473145,
      "step": 1820,
      "unclipped_grad_norm": 1.2399523258209229
    },
    {
      "bits_per_byte": 1.902950023631137,
      "learning_rate": 0.0002770931411282704,
      "loss_nats": 1.3190244436264038,
      "step": 1830,
      "unclipped_grad_norm": 1.066636562347412
    },
    {
      "bits_per_byte": 1.9139311158887273,
      "learning_rate": 0.0002767811922049827,
      "loss_nats": 1.3266359567642212,
      "step": 1840,
      "unclipped_grad_norm": 1.129875898361206
    },
    {
      "bits_per_byte": 1.9166544611651397,
      "learning_rate": 0.00027646733324953503,
      "loss_nats": 1.3285236358642578,
      "step": 1850,
      "unclipped_grad_norm": 1.1898164749145508
    },
    {
      "bits_per_byte": 1.9000117000411458,
      "learning_rate": 0.00027615156962491097,
      "loss_nats": 1.3169877529144287,
      "step": 1860,
      "unclipped_grad_norm": 1.1322225332260132
    },
    {
      "bits_per_byte": 1.9091849106728662,
      "learning_rate": 0.0002758339067266396,
      "loss_nats": 1.3233461380004883,
      "step": 1870,
      "unclipped_grad_norm": 1.1632918119430542
    },
    {
      "bits_per_byte": 1.9066440389890191,
      "learning_rate": 0.0002755143499827035,
      "loss_nats": 1.321584939956665,
      "step": 1880,
      "unclipped_grad_norm": 1.0685805082321167
    },
    {
      "bits_per_byte": 1.915992327959318,
      "learning_rate": 0.0002751929048534456,
      "loss_nats": 1.3280646800994873,
      "step": 1890,
      "unclipped_grad_norm": 1.1415059566497803
    },
    {
      "bits_per_byte": 1.9305143710155228,
      "learning_rate": 0.0002748695768314763,
      "loss_nats": 1.3381305932998657,
      "step": 1900,
      "unclipped_grad_norm": 1.0947742462158203
    },
    {
      "bits_per_byte": 1.863847876148318,
      "learning_rate": 0.00027454437144157947,
      "loss_nats": 1.2919209003448486,
      "step": 1910,
      "unclipped_grad_norm": 1.0498101711273193
    },
    {
      "bits_per_byte": 1.8709641742757193,
      "learning_rate": 0.00027421729424061787,
      "loss_nats": 1.2968535423278809,
      "step": 1920,
      "unclipped_grad_norm": 1.227995753288269
    },
    {
      "bits_per_byte": 1.9694514151535134,
      "learning_rate": 0.0002738883508174385,
      "loss_nats": 1.3651196956634521,
      "step": 1930,
      "unclipped_grad_norm": 1.1106438636779785
    },
    {
      "bits_per_byte": 1.8401163341210607,
      "learning_rate": 0.0002735575467927768,
      "loss_nats": 1.2754714488983154,
      "step": 1940,
      "unclipped_grad_norm": 1.1273587942123413
    },
    {
      "bits_per_byte": 1.9228790292478175,
      "learning_rate": 0.0002732248878191609,
      "loss_nats": 1.3328381776809692,
      "step": 1950,
      "unclipped_grad_norm": 1.0416061878204346
    },
    {
      "bits_per_byte": 1.840930843955547,
      "learning_rate": 0.0002728903795808148,
      "loss_nats": 1.276036024093628,
      "step": 1960,
      "unclipped_grad_norm": 1.0370304584503174
    },
    {
      "bits_per_byte": 1.9309775202942963,
      "learning_rate": 0.0002725540277935614,
      "loss_nats": 1.338451623916626,
      "step": 1970,
      "unclipped_grad_norm": 1.147440791130066
    },
    {
      "bits_per_byte": 1.9639140897436835,
      "learning_rate": 0.00027221583820472456,
      "loss_nats": 1.3612815141677856,
      "step": 1980,
      "unclipped_grad_norm": 1.2305821180343628
    },
    {
      "bits_per_byte": 1.9535571225260744,
      "learning_rate": 0.0002718758165930314,
      "loss_nats": 1.354102611541748,
      "step": 1990,
      "unclipped_grad_norm": 1.3296585083007812
    },
    {
      "bits_per_byte": 1.92549075778382,
      "learning_rate": 0.0002715339687685131,
      "loss_nats": 1.3346484899520874,
      "step": 2000,
      "unclipped_grad_norm": 1.0916969776153564
    },
    {
      "bits_per_byte": 1.880516778635243,
      "learning_rate": 0.00027119030057240576,
      "loss_nats": 1.3034749031066895,
      "step": 2010,
      "unclipped_grad_norm": 1.0219805240631104
    },
    {
      "bits_per_byte": 1.9332846675556208,
      "learning_rate": 0.0002708448178770506,
      "loss_nats": 1.3400508165359497,
      "step": 2020,
      "unclipped_grad_norm": 1.1069045066833496
    },
    {
      "bits_per_byte": 1.9929170720025369,
      "learning_rate": 0.00027049752658579376,
      "loss_nats": 1.3813848495483398,
      "step": 2030,
      "unclipped_grad_norm": 1.0479846000671387
    },
    {
      "bits_per_byte": 1.9184877962233373,
      "learning_rate": 0.00027014843263288514,
      "loss_nats": 1.3297944068908691,
      "step": 2040,
      "unclipped_grad_norm": 1.0204336643218994
    },
    {
      "bits_per_byte": 1.9692321372736634,
      "learning_rate": 0.00026979754198337724,
      "loss_nats": 1.364967703819275,
      "step": 2050,
      "unclipped_grad_norm": 1.0853239297866821
    },
    {
      "bits_per_byte": 1.9303957029864274,
      "learning_rate": 0.00026944486063302305,
      "loss_nats": 1.3380483388900757,
      "step": 2060,
      "unclipped_grad_norm": 1.0730464458465576
    },
    {
      "bits_per_byte": 1.889977716241857,
      "learning_rate": 0.0002690903946081738,
      "loss_nats": 1.3100327253341675,
      "step": 2070,
      "unclipped_grad_norm": 1.0691337585449219
    },
    {
      "bits_per_byte": 1.8844801188243765,
      "learning_rate": 0.00026873414996567596,
      "loss_nats": 1.3062220811843872,
      "step": 2080,
      "unclipped_grad_norm": 1.0554076433181763
    },
    {
      "bits_per_byte": 1.8957609768424206,
      "learning_rate": 0.00026837613279276736,
      "loss_nats": 1.3140413761138916,
      "step": 2090,
      "unclipped_grad_norm": 1.1196725368499756
    },
    {
      "bits_per_byte": 1.8758003264179814,
      "learning_rate": 0.0002680163492069739,
      "loss_nats": 1.3002057075500488,
      "step": 2100,
      "unclipped_grad_norm": 1.0327236652374268
    },
    {
      "bits_per_byte": 1.914194593309849,
      "learning_rate": 0.00026765480535600414,
      "loss_nats": 1.326818585395813,
      "step": 2110,
      "unclipped_grad_norm": 1.0782309770584106
    },
    {
      "bits_per_byte": 1.8761728408397502,
      "learning_rate": 0.0002672915074176452,
      "loss_nats": 1.3004639148712158,
      "step": 2120,
      "unclipped_grad_norm": 1.0318117141723633
    },
    {
      "bits_per_byte": 1.8463771905430704,
      "learning_rate": 0.00026692646159965624,
      "loss_nats": 1.279811143875122,
      "step": 2130,
      "unclipped_grad_norm": 1.1898446083068848
    },
    {
      "bits_per_byte": 1.984834747325239,
      "learning_rate": 0.0002665596741396633,
      "loss_nats": 1.3757826089859009,
      "step": 2140,
      "unclipped_grad_norm": 1.1598726511001587
    },
    {
      "bits_per_byte": 1.8562880347643413,
      "learning_rate": 0.000266191151305052,
      "loss_nats": 1.286680817604065,
      "step": 2150,
      "unclipped_grad_norm": 1.0410785675048828
    },
    {
      "bits_per_byte": 1.8990754264898486,
      "learning_rate": 0.00026582089939286075,
      "loss_nats": 1.3163387775421143,
      "step": 2160,
      "unclipped_grad_norm": 1.0785906314849854
    },
    {
      "bits_per_byte": 1.9583294691048665,
      "learning_rate": 0.0002654489247296732,
      "loss_nats": 1.3574105501174927,
      "step": 2170,
      "unclipped_grad_norm": 1.1421548128128052
    },
    {
      "bits_per_byte": 1.8472800994601,
      "learning_rate": 0.00026507523367151007,
      "loss_nats": 1.2804369926452637,
      "step": 2180,
      "unclipped_grad_norm": 1.0929888486862183
    },
    {
      "bits_per_byte": 1.8586782496460328,
      "learning_rate": 0.00026469983260372037,
      "loss_nats": 1.2883375883102417,
      "step": 2190,
      "unclipped_grad_norm": 1.058180570602417
    },
    {
      "bits_per_byte": 1.8910505440179393,
      "learning_rate": 0.0002643227279408727,
      "loss_nats": 1.3107763528823853,
      "step": 2200,
      "unclipped_grad_norm": 0.9802089333534241
    },
    {
      "bits_per_byte": 1.8399546704292495,
      "learning_rate": 0.0002639439261266453,
      "loss_nats": 1.2753593921661377,
      "step": 2210,
      "unclipped_grad_norm": 1.060943365097046
    },
    {
      "bits_per_byte": 1.8476158095945843,
      "learning_rate": 0.000263563433633716,
      "loss_nats": 1.2806696891784668,
      "step": 2220,
      "unclipped_grad_norm": 1.0172474384307861
    },
    {
      "bits_per_byte": 1.9309195621409554,
      "learning_rate": 0.00026318125696365156,
      "loss_nats": 1.3384114503860474,
      "step": 2230,
      "unclipped_grad_norm": 1.1571786403656006
    },
    {
      "bits_per_byte": 1.848552427111183,
      "learning_rate": 0.000262797402646797,
      "loss_nats": 1.2813189029693604,
      "step": 2240,
      "unclipped_grad_norm": 1.085301160812378
    },
    {
      "bits_per_byte": 1.864913308670413,
      "learning_rate": 0.00026241187724216334,
      "loss_nats": 1.2926594018936157,
      "step": 2250,
      "unclipped_grad_norm": 1.0694295167922974
    },
    {
      "bits_per_byte": 1.8611869605741684,
      "learning_rate": 0.0002620246873373162,
      "loss_nats": 1.290076494216919,
      "step": 2260,
      "unclipped_grad_norm": 1.1977293491363525
    },
    {
      "bits_per_byte": 1.8337584794839643,
      "learning_rate": 0.00026163583954826264,
      "loss_nats": 1.2710645198822021,
      "step": 2270,
      "unclipped_grad_norm": 1.1360434293746948
    },
    {
      "bits_per_byte": 1.8587281246147829,
      "learning_rate": 0.0002612453405193386,
      "loss_nats": 1.2883721590042114,
      "step": 2280,
      "unclipped_grad_norm": 1.113946795463562
    },
    {
      "bits_per_byte": 1.8907026231152437,
      "learning_rate": 0.000260853196923095,
      "loss_nats": 1.310535192489624,
      "step": 2290,
      "unclipped_grad_norm": 1.1474530696868896
    },
    {
      "bits_per_byte": 1.8085284526197394,
      "learning_rate": 0.0002604594154601839,
      "loss_nats": 1.253576397895813,
      "step": 2300,
      "unclipped_grad_norm": 0.96898353099823
    },
    {
      "bits_per_byte": 1.8446171200941404,
      "learning_rate": 0.0002600640028592441,
      "loss_nats": 1.2785911560058594,
      "step": 2310,
      "unclipped_grad_norm": 0.9442379474639893
    },
    {
      "bits_per_byte": 1.8195852172437095,
      "learning_rate": 0.0002596669658767859,
      "loss_nats": 1.2612403631210327,
      "step": 2320,
      "unclipped_grad_norm": 0.9633188843727112
    },
    {
      "bits_per_byte": 1.8903022475040352,
      "learning_rate": 0.000259268311297076,
      "loss_nats": 1.3102576732635498,
      "step": 2330,
      "unclipped_grad_norm": 1.0167381763458252
    },
    {
      "bits_per_byte": 1.8286817236131452,
      "learning_rate": 0.00025886804593202114,
      "loss_nats": 1.2675455808639526,
      "step": 2340,
      "unclipped_grad_norm": 1.1636918783187866
    },
    {
      "bits_per_byte": 1.8687012265556664,
      "learning_rate": 0.00025846617662105193,
      "loss_nats": 1.2952849864959717,
      "step": 2350,
      "unclipped_grad_norm": 1.0884090662002563
    },
    {
      "bits_per_byte": 1.8708572010668825,
      "learning_rate": 0.00025806271023100637,
      "loss_nats": 1.2967793941497803,
      "step": 2360,
      "unclipped_grad_norm": 1.0467369556427002
    },
    {
      "bits_per_byte": 1.8979811008824086,
      "learning_rate": 0.00025765765365601165,
      "loss_nats": 1.3155802488327026,
      "step": 2370,
      "unclipped_grad_norm": 1.0231177806854248
    },
    {
      "bits_per_byte": 1.853912954355926,
      "learning_rate": 0.00025725101381736707,
      "loss_nats": 1.2850345373153687,
      "step": 2380,
      "unclipped_grad_norm": 1.0531508922576904
    },
    {
      "bits_per_byte": 1.909142430958103,
      "learning_rate": 0.00025684279766342567,
      "loss_nats": 1.3233166933059692,
      "step": 2390,
      "unclipped_grad_norm": 1.0615708827972412
    },
    {
      "bits_per_byte": 1.8518789155441715,
      "learning_rate": 0.00025643301216947504,
      "loss_nats": 1.2836246490478516,
      "step": 2400,
      "unclipped_grad_norm": 1.0747002363204956
    },
    {
      "bits_per_byte": 1.8430525939192415,
      "learning_rate": 0.00025602166433761866,
      "loss_nats": 1.277506709098816,
      "step": 2410,
      "unclipped_grad_norm": 1.035906434059143
    },
    {
      "bits_per_byte": 1.8218237434273397,
      "learning_rate": 0.0002556087611966561,
      "loss_nats": 1.2627919912338257,
      "step": 2420,
      "unclipped_grad_norm": 0.9908418655395508
    },
    {
      "bits_per_byte": 1.8454151195941437,
      "learning_rate": 0.00025519430980196277,
      "loss_nats": 1.279144287109375,
      "step": 2430,
      "unclipped_grad_norm": 1.1172900199890137
    },
    {
      "bits_per_byte": 1.9008382486611926,
      "learning_rate": 0.0002547783172353695,
      "loss_nats": 1.3175606727600098,
      "step": 2440,
      "unclipped_grad_norm": 1.0292961597442627
    },
    {
      "bits_per_byte": 1.8510478953752025,
      "learning_rate": 0.0002543607906050415,
      "loss_nats": 1.2830486297607422,
      "step": 2450,
      "unclipped_grad_norm": 1.0116122961044312
    },
    {
      "bits_per_byte": 1.8498311181203482,
      "learning_rate": 0.0002539417370453568,
      "loss_nats": 1.2822052240371704,
      "step": 2460,
      "unclipped_grad_norm": 0.9883264303207397
    },
    {
      "bits_per_byte": 1.849598081628429,
      "learning_rate": 0.00025352116371678464,
      "loss_nats": 1.282043695449829,
      "step": 2470,
      "unclipped_grad_norm": 1.0902615785598755
    },
    {
      "bits_per_byte": 1.8701792454571813,
      "learning_rate": 0.0002530990778057627,
      "loss_nats": 1.296309471130371,
      "step": 2480,
      "unclipped_grad_norm": 1.4108390808105469
    },
    {
      "bits_per_byte": 1.81769770765049,
      "learning_rate": 0.00025267548652457475,
      "loss_nats": 1.259932041168213,
      "step": 2490,
      "unclipped_grad_norm": 1.0036945343017578
    },
    {
      "bits_per_byte": 1.8526576529872785,
      "learning_rate": 0.00025225039711122696,
      "loss_nats": 1.2841644287109375,
      "step": 2500,
      "unclipped_grad_norm": 1.0499781370162964
    },
    {
      "bits_per_byte": 1.8174889207123426,
      "learning_rate": 0.00025182381682932464,
      "loss_nats": 1.2597873210906982,
      "step": 2510,
      "unclipped_grad_norm": 1.112528681755066
    },
    {
      "bits_per_byte": 1.8559390819657409,
      "learning_rate": 0.0002513957529679477,
      "loss_nats": 1.2864389419555664,
      "step": 2520,
      "unclipped_grad_norm": 1.043527603149414
    },
    {
      "bits_per_byte": 1.8664243482408935,
      "learning_rate": 0.00025096621284152656,
      "loss_nats": 1.2937067747116089,
      "step": 2530,
      "unclipped_grad_norm": 1.047589898109436
    },
    {
      "bits_per_byte": 1.8385335777851701,
      "learning_rate": 0.0002505352037897168,
      "loss_nats": 1.2743743658065796,
      "step": 2540,
      "unclipped_grad_norm": 1.114875078201294
    },
    {
      "bits_per_byte": 1.8303919190933249,
      "learning_rate": 0.0002501027331772739,
      "loss_nats": 1.2687309980392456,
      "step": 2550,
      "unclipped_grad_norm": 1.026410460472107
    },
    {
      "bits_per_byte": 1.8414952910156785,
      "learning_rate": 0.00024966880839392746,
      "loss_nats": 1.2764272689819336,
      "step": 2560,
      "unclipped_grad_norm": 1.0519986152648926
    },
    {
      "bits_per_byte": 1.8285018297603426,
      "learning_rate": 0.0002492334368542548,
      "loss_nats": 1.2674208879470825,
      "step": 2570,
      "unclipped_grad_norm": 0.9845532178878784
    },
    {
      "bits_per_byte": 1.8621820521920607,
      "learning_rate": 0.00024879662599755416,
      "loss_nats": 1.2907662391662598,
      "step": 2580,
      "unclipped_grad_norm": 1.1312247514724731
    },
    {
      "bits_per_byte": 1.821957545929711,
      "learning_rate": 0.00024835838328771803,
      "loss_nats": 1.2628847360610962,
      "step": 2590,
      "unclipped_grad_norm": 0.9848816990852356
    },
    {
      "bits_per_byte": 1.8336463467956017,
      "learning_rate": 0.0002479187162131051,
      "loss_nats": 1.270986795425415,
      "step": 2600,
      "unclipped_grad_norm": 1.0540134906768799
    },
    {
      "bits_per_byte": 1.762311382870726,
      "learning_rate": 0.00024747763228641264,
      "loss_nats": 1.221541166305542,
      "step": 2610,
      "unclipped_grad_norm": 0.9237543940544128
    },
    {
      "bits_per_byte": 1.8236500271968519,
      "learning_rate": 0.00024703513904454795,
      "loss_nats": 1.2640578746795654,
      "step": 2620,
      "unclipped_grad_norm": 0.9258099794387817
    },
    {
      "bits_per_byte": 1.8442078013850869,
      "learning_rate": 0.00024659124404849956,
      "loss_nats": 1.2783074378967285,
      "step": 2630,
      "unclipped_grad_norm": 1.1065376996994019
    },
    {
      "bits_per_byte": 1.6990233151970702,
      "learning_rate": 0.00024614595488320833,
      "loss_nats": 1.1776732206344604,
      "step": 2640,
      "unclipped_grad_norm": 0.975441038608551
    },
    {
      "bits_per_byte": 1.7897238695744,
      "learning_rate": 0.00024569927915743745,
      "loss_nats": 1.2405420541763306,
      "step": 2650,
      "unclipped_grad_norm": 1.024839997291565
    },
    {
      "bits_per_byte": 1.829891621561965,
      "learning_rate": 0.0002452512245036427,
      "loss_nats": 1.2683842182159424,
      "step": 2660,
      "unclipped_grad_norm": 1.0799146890640259
    },
    {
      "bits_per_byte": 1.8374299651145836,
      "learning_rate": 0.00024480179857784177,
      "loss_nats": 1.2736093997955322,
      "step": 2670,
      "unclipped_grad_norm": 0.9827171564102173
    },
    {
      "bits_per_byte": 1.757754702536116,
      "learning_rate": 0.00024435100905948387,
      "loss_nats": 1.218382716178894,
      "step": 2680,
      "unclipped_grad_norm": 0.9623468518257141
    },
    {
      "bits_per_byte": 1.9223347041578367,
      "learning_rate": 0.00024389886365131793,
      "loss_nats": 1.332460880279541,
      "step": 2690,
      "unclipped_grad_norm": 1.0313293933868408
    },
    {
      "bits_per_byte": 1.8426768118271062,
      "learning_rate": 0.00024344537007926163,
      "loss_nats": 1.2772462368011475,
      "step": 2700,
      "unclipped_grad_norm": 1.154735803604126
    },
    {
      "bits_per_byte": 1.7467827253590213,
      "learning_rate": 0.00024299053609226878,
      "loss_nats": 1.2107775211334229,
      "step": 2710,
      "unclipped_grad_norm": 0.9442266225814819
    },
    {
      "bits_per_byte": 1.7668878412101854,
      "learning_rate": 0.00024253436946219733,
      "loss_nats": 1.2247133255004883,
      "step": 2720,
      "unclipped_grad_norm": 0.9805414080619812
    },
    {
      "bits_per_byte": 1.8017586995338266,
      "learning_rate": 0.0002420768779836764,
      "loss_nats": 1.2488839626312256,
      "step": 2730,
      "unclipped_grad_norm": 1.027875542640686
    },
    {
      "bits_per_byte": 1.7992866209103247,
      "learning_rate": 0.00024161806947397305,
      "loss_nats": 1.2471704483032227,
      "step": 2740,
      "unclipped_grad_norm": 1.0912469625473022
    },
    {
      "bits_per_byte": 1.8291928560515094,
      "learning_rate": 0.00024115795177285886,
      "loss_nats": 1.2678998708724976,
      "step": 2750,
      "unclipped_grad_norm": 1.0327590703964233
    },
    {
      "bits_per_byte": 1.831356913747316,
      "learning_rate": 0.0002406965327424758,
      "loss_nats": 1.269399881362915,
      "step": 2760,
      "unclipped_grad_norm": 1.0535632371902466
    },
    {
      "bits_per_byte": 1.7175185014535075,
      "learning_rate": 0.00024023382026720197,
      "loss_nats": 1.190493106842041,
      "step": 2770,
      "unclipped_grad_norm": 0.9521624445915222
    },
    {
      "bits_per_byte": 1.7419930085324922,
      "learning_rate": 0.00023976982225351698,
      "loss_nats": 1.2074575424194336,
      "step": 2780,
      "unclipped_grad_norm": 0.9253908395767212
    },
    {
      "bits_per_byte": 1.829489182158946,
      "learning_rate": 0.0002393045466298666,
      "loss_nats": 1.2681052684783936,
      "step": 2790,
      "unclipped_grad_norm": 1.0019073486328125
    },
    {
      "bits_per_byte": 1.900268298156233,
      "learning_rate": 0.00023883800134652763,
      "loss_nats": 1.3171656131744385,
      "step": 2800,
      "unclipped_grad_norm": 1.1026051044464111
    },
    {
      "bits_per_byte": 1.8251356133350047,
      "learning_rate": 0.0002383701943754715,
      "loss_nats": 1.265087604522705,
      "step": 2810,
      "unclipped_grad_norm": 1.095402479171753
    },
    {
      "bits_per_byte": 1.8078448215825598,
      "learning_rate": 0.00023790113371022884,
      "loss_nats": 1.2531025409698486,
      "step": 2820,
      "unclipped_grad_norm": 1.0277965068817139
    },
    {
      "bits_per_byte": 1.8903005276775264,
      "learning_rate": 0.0002374308273657521,
      "loss_nats": 1.3102564811706543,
      "step": 2830,
      "unclipped_grad_norm": 1.0477416515350342
    },
    {
      "bits_per_byte": 1.7718302786306808,
      "learning_rate": 0.00023695928337827911,
      "loss_nats": 1.2281391620635986,
      "step": 2840,
      "unclipped_grad_norm": 0.9842959642410278
    },
    {
      "bits_per_byte": 1.7997145137456714,
      "learning_rate": 0.00023648650980519564,
      "loss_nats": 1.247467041015625,
      "step": 2850,
      "unclipped_grad_norm": 1.0457379817962646
    },
    {
      "bits_per_byte": 1.7627234533021934,
      "learning_rate": 0.00023601251472489757,
      "loss_nats": 1.2218267917633057,
      "step": 2860,
      "unclipped_grad_norm": 1.0218563079833984
    },
    {
      "bits_per_byte": 1.8575106594293251,
      "learning_rate": 0.00023553730623665305,
      "loss_nats": 1.2875282764434814,
      "step": 2870,
      "unclipped_grad_norm": 1.1067496538162231
    },
    {
      "bits_per_byte": 1.8563188196588458,
      "learning_rate": 0.00023506089246046403,
      "loss_nats": 1.2867021560668945,
      "step": 2880,
      "unclipped_grad_norm": 1.0256601572036743
    },
    {
      "bits_per_byte": 1.76771198207312,
      "learning_rate": 0.00023458328153692736,
      "loss_nats": 1.2252845764160156,
      "step": 2890,
      "unclipped_grad_norm": 1.0549262762069702
    },
    {
      "bits_per_byte": 1.7222432088380106,
      "learning_rate": 0.0002341044816270961,
      "loss_nats": 1.19376802444458,
      "step": 2900,
      "unclipped_grad_norm": 1.0038959980010986
    },
    {
      "bits_per_byte": 1.7887707417233183,
      "learning_rate": 0.00023362450091233953,
      "loss_nats": 1.2398813962936401,
      "step": 2910,
      "unclipped_grad_norm": 1.020102620124817
    },
    {
      "bits_per_byte": 1.7699561836842288,
      "learning_rate": 0.00023314334759420388,
      "loss_nats": 1.2268401384353638,
      "step": 2920,
      "unclipped_grad_norm": 0.9710083603858948
    },
    {
      "bits_per_byte": 1.7845655939270717,
      "learning_rate": 0.0002326610298942717,
      "loss_nats": 1.236966609954834,
      "step": 2930,
      "unclipped_grad_norm": 1.063584566116333
    },
    {
      "bits_per_byte": 1.8044884081683212,
      "learning_rate": 0.00023217755605402175,
      "loss_nats": 1.2507760524749756,
      "step": 2940,
      "unclipped_grad_norm": 1.1001172065734863
    },
    {
      "bits_per_byte": 1.788769881810064,
      "learning_rate": 0.00023169293433468802,
      "loss_nats": 1.2398808002471924,
      "step": 2950,
      "unclipped_grad_norm": 1.052484393119812
    },
    {
      "bits_per_byte": 1.8266576597951407,
      "learning_rate": 0.00023120717301711857,
      "loss_nats": 1.2661426067352295,
      "step": 2960,
      "unclipped_grad_norm": 1.0507400035858154
    },
    {
      "bits_per_byte": 1.7942014378896132,
      "learning_rate": 0.00023072028040163408,
      "loss_nats": 1.2436456680297852,
      "step": 2970,
      "unclipped_grad_norm": 1.016513466835022
    },
    {
      "bits_per_byte": 1.776544667056132,
      "learning_rate": 0.00023023226480788593,
      "loss_nats": 1.2314069271087646,
      "step": 2980,
      "unclipped_grad_norm": 1.0853108167648315
    },
    {
      "bits_per_byte": 1.797620281006115,
      "learning_rate": 0.00022974313457471416,
      "loss_nats": 1.2460154294967651,
      "step": 2990,
      "unclipped_grad_norm": 1.025924563407898
    },
    {
      "bits_per_byte": 1.8018763356670169,
      "learning_rate": 0.0002292528980600049,
      "loss_nats": 1.2489655017852783,
      "step": 3000,
      "unclipped_grad_norm": 1.066794514656067
    },
    {
      "bits_per_byte": 1.8036527444677788,
      "learning_rate": 0.00022876156364054758,
      "loss_nats": 1.2501968145370483,
      "step": 3010,
      "unclipped_grad_norm": 0.9892268180847168
    },
    {
      "bits_per_byte": 1.7646690930314042,
      "learning_rate": 0.0002282691397118918,
      "loss_nats": 1.2231754064559937,
      "step": 3020,
      "unclipped_grad_norm": 1.1210085153579712
    },
    {
      "bits_per_byte": 1.8358253669820335,
      "learning_rate": 0.00022777563468820376,
      "loss_nats": 1.2724971771240234,
      "step": 3030,
      "unclipped_grad_norm": 1.0379256010055542
    },
    {
      "bits_per_byte": 1.763304066731506,
      "learning_rate": 0.00022728105700212282,
      "loss_nats": 1.222229242324829,
      "step": 3040,
      "unclipped_grad_norm": 1.0503100156784058
    },
    {
      "bits_per_byte": 1.7804524568490367,
      "learning_rate": 0.00022678541510461698,
      "loss_nats": 1.2341156005859375,
      "step": 3050,
      "unclipped_grad_norm": 1.0255935192108154
    },
    {
      "bits_per_byte": 1.779026720673384,
      "learning_rate": 0.00022628871746483876,
      "loss_nats": 1.2331273555755615,
      "step": 3060,
      "unclipped_grad_norm": 1.0667622089385986
    },
    {
      "bits_per_byte": 1.7089128335696342,
      "learning_rate": 0.0002257909725699804,
      "loss_nats": 1.184528112411499,
      "step": 3070,
      "unclipped_grad_norm": 0.9528610110282898
    },
    {
      "bits_per_byte": 1.7514905784437398,
      "learning_rate": 0.00022529218892512886,
      "loss_nats": 1.214040756225586,
      "step": 3080,
      "unclipped_grad_norm": 0.9759883880615234
    },
    {
      "bits_per_byte": 1.8129533942437885,
      "learning_rate": 0.00022479237505312034,
      "loss_nats": 1.256643533706665,
      "step": 3090,
      "unclipped_grad_norm": 1.074378490447998
    },
    {
      "bits_per_byte": 1.7348117009630646,
      "learning_rate": 0.00022429153949439496,
      "loss_nats": 1.2024798393249512,
      "step": 3100,
      "unclipped_grad_norm": 1.0595066547393799
    },
    {
      "bits_per_byte": 1.8111885082806345,
      "learning_rate": 0.00022378969080685053,
      "loss_nats": 1.255420207977295,
      "step": 3110,
      "unclipped_grad_norm": 1.0304162502288818
    },
    {
      "bits_per_byte": 1.7739528885076294,
      "learning_rate": 0.0002232868375656965,
      "loss_nats": 1.2296104431152344,
      "step": 3120,
      "unclipped_grad_norm": 0.9790826439857483
    },
    {
      "bits_per_byte": 1.8333825254091782,
      "learning_rate": 0.00022278298836330733,
      "loss_nats": 1.2708039283752441,
      "step": 3130,
      "unclipped_grad_norm": 1.0250012874603271
    },
    {
      "bits_per_byte": 1.807222588351738,
      "learning_rate": 0.00022227815180907572,
      "loss_nats": 1.252671241760254,
      "step": 3140,
      "unclipped_grad_norm": 1.065406322479248
    },
    {
      "bits_per_byte": 1.7667856835155729,
      "learning_rate": 0.00022177233652926546,
      "loss_nats": 1.2246425151824951,
      "step": 3150,
      "unclipped_grad_norm": 1.1064149141311646
    },
    {
      "bits_per_byte": 1.77101886448391,
      "learning_rate": 0.00022126555116686416,
      "loss_nats": 1.227576732635498,
      "step": 3160,
      "unclipped_grad_norm": 0.9769210815429688
    },
    {
      "bits_per_byte": 1.6904848925296843,
      "learning_rate": 0.00022075780438143532,
      "loss_nats": 1.1717548370361328,
      "step": 3170,
      "unclipped_grad_norm": 1.003720998764038
    },
    {
      "bits_per_byte": 1.7897881910858227,
      "learning_rate": 0.00022024910484897065,
      "loss_nats": 1.2405866384506226,
      "step": 3180,
      "unclipped_grad_norm": 0.9989861249923706
    },
    {
      "bits_per_byte": 1.7613923075845153,
      "learning_rate": 0.00021973946126174158,
      "loss_nats": 1.2209041118621826,
      "step": 3190,
      "unclipped_grad_norm": 1.0040642023086548
    },
    {
      "bits_per_byte": 1.7417902409871249,
      "learning_rate": 0.00021922888232815086,
      "loss_nats": 1.2073169946670532,
      "step": 3200,
      "unclipped_grad_norm": 0.9894367456436157
    },
    {
      "bits_per_byte": 1.7268853645500994,
      "learning_rate": 0.0002187173767725837,
      "loss_nats": 1.1969857215881348,
      "step": 3210,
      "unclipped_grad_norm": 1.0016205310821533
    },
    {
      "bits_per_byte": 1.7365394386736324,
      "learning_rate": 0.0002182049533352588,
      "loss_nats": 1.2036774158477783,
      "step": 3220,
      "unclipped_grad_norm": 0.9552999138832092
    },
    {
      "bits_per_byte": 1.8281382584364185,
      "learning_rate": 0.00021769162077207884,
      "loss_nats": 1.2671688795089722,
      "step": 3230,
      "unclipped_grad_norm": 1.0519078969955444
    },
    {
      "bits_per_byte": 1.7008661093010653,
      "learning_rate": 0.00021717738785448102,
      "loss_nats": 1.178950548171997,
      "step": 3240,
      "unclipped_grad_norm": 0.956534206867218
    },
    {
      "bits_per_byte": 1.8618967329742793,
      "learning_rate": 0.00021666226336928708,
      "loss_nats": 1.290568470954895,
      "step": 3250,
      "unclipped_grad_norm": 1.118624210357666
    },
    {
      "bits_per_byte": 1.7141249397866825,
      "learning_rate": 0.0002161462561185532,
      "loss_nats": 1.188140869140625,
      "step": 3260,
      "unclipped_grad_norm": 0.9860926270484924
    },
    {
      "bits_per_byte": 1.7454612106697915,
      "learning_rate": 0.00021562937491941957,
      "loss_nats": 1.2098615169525146,
      "step": 3270,
      "unclipped_grad_norm": 0.968685507774353
    },
    {
      "bits_per_byte": 1.8017138120619516,
      "learning_rate": 0.00021511162860395975,
      "loss_nats": 1.2488528490066528,
      "step": 3280,
      "unclipped_grad_norm": 1.0125019550323486
    },
    {
      "bits_per_byte": 1.7954340375483469,
      "learning_rate": 0.0002145930260190298,
      "loss_nats": 1.2445000410079956,
      "step": 3290,
      "unclipped_grad_norm": 1.0358432531356812
    },
    {
      "bits_per_byte": 1.7138430602219183,
      "learning_rate": 0.00021407357602611698,
      "loss_nats": 1.1879454851150513,
      "step": 3300,
      "unclipped_grad_norm": 1.0642298460006714
    },
    {
      "bits_per_byte": 1.7570346111769533,
      "learning_rate": 0.0002135532875011885,
      "loss_nats": 1.217883586883545,
      "step": 3310,
      "unclipped_grad_norm": 1.017012596130371
    },
    {
      "bits_per_byte": 1.753239298037713,
      "learning_rate": 0.0002130321693345397,
      "loss_nats": 1.2152528762817383,
      "step": 3320,
      "unclipped_grad_norm": 0.9849429130554199
    },
    {
      "bits_per_byte": 1.7669095110241941,
      "learning_rate": 0.00021251023043064224,
      "loss_nats": 1.2247283458709717,
      "step": 3330,
      "unclipped_grad_norm": 0.9692639708518982
    },
    {
      "bits_per_byte": 1.7337540076602584,
      "learning_rate": 0.00021198747970799188,
      "loss_nats": 1.2017467021942139,
      "step": 3340,
      "unclipped_grad_norm": 0.9712040424346924
    },
    {
      "bits_per_byte": 1.7289699662612077,
      "learning_rate": 0.00021146392609895618,
      "loss_nats": 1.1984306573867798,
      "step": 3350,
      "unclipped_grad_norm": 0.9344873428344727
    },
    {
      "bits_per_byte": 1.771012673108479,
      "learning_rate": 0.00021093957854962178,
      "loss_nats": 1.2275724411010742,
      "step": 3360,
      "unclipped_grad_norm": 1.0185227394104004
    },
    {
      "bits_per_byte": 1.7121330367243894,
      "learning_rate": 0.00021041444601964157,
      "loss_nats": 1.1867601871490479,
      "step": 3370,
      "unclipped_grad_norm": 1.017809271812439
    },
    {
      "bits_per_byte": 1.7444269070075025,
      "learning_rate": 0.00020988853748208164,
      "loss_nats": 1.2091445922851562,
      "step": 3380,
      "unclipped_grad_norm": 0.9594104886054993
    },
    {
      "bits_per_byte": 1.782493202984175,
      "learning_rate": 0.00020936186192326782,
      "loss_nats": 1.235530138015747,
      "step": 3390,
      "unclipped_grad_norm": 1.0056157112121582
    },
    {
      "bits_per_byte": 1.8303659497130447,
      "learning_rate": 0.00020883442834263225,
      "loss_nats": 1.2687129974365234,
      "step": 3400,
      "unclipped_grad_norm": 0.9542431235313416
    },
    {
      "bits_per_byte": 1.8696024156461875,
      "learning_rate": 0.0002083062457525596,
      "loss_nats": 1.2959096431732178,
      "step": 3410,
      "unclipped_grad_norm": 1.0261633396148682
    },
    {
      "bits_per_byte": 1.7306273630675726,
      "learning_rate": 0.00020777732317823306,
      "loss_nats": 1.1995794773101807,
      "step": 3420,
      "unclipped_grad_norm": 1.0138797760009766
    },
    {
      "bits_per_byte": 1.663148250175041,
      "learning_rate": 0.00020724766965748005,
      "loss_nats": 1.1528065204620361,
      "step": 3430,
      "unclipped_grad_norm": 0.9656930565834045
    },
    {
      "bits_per_byte": 1.7509053214828536,
      "learning_rate": 0.00020671729424061788,
      "loss_nats": 1.2136350870132446,
      "step": 3440,
      "unclipped_grad_norm": 1.1262567043304443
    },
    {
      "bits_per_byte": 1.8269866626062412,
      "learning_rate": 0.00020618620599029906,
      "loss_nats": 1.2663706541061401,
      "step": 3450,
      "unclipped_grad_norm": 1.0454469919204712
    },
    {
      "bits_per_byte": 1.7413009503454202,
      "learning_rate": 0.00020565441398135649,
      "loss_nats": 1.2069778442382812,
      "step": 3460,
      "unclipped_grad_norm": 1.1041505336761475
    },
    {
      "bits_per_byte": 1.7427603951206418,
      "learning_rate": 0.00020512192730064827,
      "loss_nats": 1.2079894542694092,
      "step": 3470,
      "unclipped_grad_norm": 1.0516799688339233
    },
    {
      "bits_per_byte": 1.6734345325231454,
      "learning_rate": 0.00020458875504690273,
      "loss_nats": 1.1599364280700684,
      "step": 3480,
      "unclipped_grad_norm": 0.9458639621734619
    },
    {
      "bits_per_byte": 1.8872512752777288,
      "learning_rate": 0.00020405490633056245,
      "loss_nats": 1.308142900466919,
      "step": 3490,
      "unclipped_grad_norm": 1.0662349462509155
    },
    {
      "bits_per_byte": 1.7517492403506374,
      "learning_rate": 0.00020352039027362917,
      "loss_nats": 1.2142200469970703,
      "step": 3500,
      "unclipped_grad_norm": 1.0241658687591553
    },
    {
      "bits_per_byte": 1.7728004327641977,
      "learning_rate": 0.0002029852160095075,
      "loss_nats": 1.2288116216659546,
      "step": 3510,
      "unclipped_grad_norm": 1.0039124488830566
    },
    {
      "bits_per_byte": 1.7731452579791778,
      "learning_rate": 0.000202449392682849,
      "loss_nats": 1.229050636291504,
      "step": 3520,
      "unclipped_grad_norm": 0.9908767342567444
    },
    {
      "bits_per_byte": 1.743414617124524,
      "learning_rate": 0.0002019129294493959,
      "loss_nats": 1.2084429264068604,
      "step": 3530,
      "unclipped_grad_norm": 0.9787898063659668
    },
    {
      "bits_per_byte": 1.7481145590073028,
      "learning_rate": 0.00020137583547582478,
      "loss_nats": 1.211700677871704,
      "step": 3540,
      "unclipped_grad_norm": 1.0219446420669556
    },
    {
      "bits_per_byte": 1.7531405799961175,
      "learning_rate": 0.00020083811993958973,
      "loss_nats": 1.2151844501495361,
      "step": 3550,
      "unclipped_grad_norm": 1.2019073963165283
    },
    {
      "bits_per_byte": 1.6952811446969462,
      "learning_rate": 0.00020029979202876563,
      "loss_nats": 1.175079345703125,
      "step": 3560,
      "unclipped_grad_norm": 0.9977114796638489
    },
    {
      "bits_per_byte": 1.7182931113129936,
      "learning_rate": 0.00019976086094189118,
      "loss_nats": 1.1910300254821777,
      "step": 3570,
      "unclipped_grad_norm": 1.0032203197479248
    },
    {
      "bits_per_byte": 1.7239327664000867,
      "learning_rate": 0.00019922133588781175,
      "loss_nats": 1.194939136505127,
      "step": 3580,
      "unclipped_grad_norm": 1.0535842180252075
    },
    {
      "bits_per_byte": 1.7655023489748345,
      "learning_rate": 0.0001986812260855219,
      "loss_nats": 1.2237529754638672,
      "step": 3590,
      "unclipped_grad_norm": 1.055069923400879
    },
    {
      "bits_per_byte": 1.7734857836278861,
      "learning_rate": 0.00019814054076400787,
      "loss_nats": 1.2292866706848145,
      "step": 3600,
      "unclipped_grad_norm": 1.0577540397644043
    },
    {
      "bits_per_byte": 1.7140950148054324,
      "learning_rate": 0.00019759928916209004,
      "loss_nats": 1.1881201267242432,
      "step": 3610,
      "unclipped_grad_norm": 1.0167864561080933
    },
    {
      "bits_per_byte": 1.6696412831757155,
      "learning_rate": 0.00019705748052826493,
      "loss_nats": 1.1573071479797363,
      "step": 3620,
      "unclipped_grad_norm": 0.9244848489761353
    },
    {
      "bits_per_byte": 1.7374531824976664,
      "learning_rate": 0.00019651512412054723,
      "loss_nats": 1.2043107748031616,
      "step": 3630,
      "unclipped_grad_norm": 0.9926364421844482
    },
    {
      "bits_per_byte": 1.7074227758825589,
      "learning_rate": 0.00019597222920631144,
      "loss_nats": 1.183495283126831,
      "step": 3640,
      "unclipped_grad_norm": 0.9468497633934021
    },
    {
      "bits_per_byte": 1.7634653864580154,
      "learning_rate": 0.0001954288050621338,
      "loss_nats": 1.2223410606384277,
      "step": 3650,
      "unclipped_grad_norm": 0.9865193367004395
    },
    {
      "bits_per_byte": 1.6666551484087844,
      "learning_rate": 0.00019488486097363363,
      "loss_nats": 1.1552373170852661,
      "step": 3660,
      "unclipped_grad_norm": 1.0027393102645874
    },
    {
      "bits_per_byte": 1.749947206134897,
      "learning_rate": 0.00019434040623531452,
      "loss_nats": 1.2129709720611572,
      "step": 3670,
      "unclipped_grad_norm": 1.0161325931549072
    },
    {
      "bits_per_byte": 1.700321268263132,
      "learning_rate": 0.0001937954501504058,
      "loss_nats": 1.1785728931427002,
      "step": 3680,
      "unclipped_grad_norm": 1.0996551513671875
    },
    {
      "bits_per_byte": 1.6689437215438159,
      "learning_rate": 0.00019325000203070333,
      "loss_nats": 1.1568236351013184,
      "step": 3690,
      "unclipped_grad_norm": 0.9738581776618958
    },
    {
      "bits_per_byte": 1.6837737855277157,
      "learning_rate": 0.0001927040711964106,
      "loss_nats": 1.1671030521392822,
      "step": 3700,
      "unclipped_grad_norm": 1.0792776346206665
    },
    {
      "bits_per_byte": 1.7280950905162684,
      "learning_rate": 0.00019215766697597923,
      "loss_nats": 1.197824239730835,
      "step": 3710,
      "unclipped_grad_norm": 0.9678228497505188
    },
    {
      "bits_per_byte": 1.705079684247204,
      "learning_rate": 0.00019161079870594978,
      "loss_nats": 1.1818711757659912,
      "step": 3720,
      "unclipped_grad_norm": 0.9917497634887695
    },
    {
      "bits_per_byte": 1.649159353336511,
      "learning_rate": 0.0001910634757307922,
      "loss_nats": 1.1431101560592651,
      "step": 3730,
      "unclipped_grad_norm": 0.9426941275596619
    },
    {
      "bits_per_byte": 1.6954601786364945,
      "learning_rate": 0.0001905157074027459,
      "loss_nats": 1.1752034425735474,
      "step": 3740,
      "unclipped_grad_norm": 1.0086418390274048
    },
    {
      "bits_per_byte": 1.721657263946521,
      "learning_rate": 0.00018996750308166037,
      "loss_nats": 1.1933618783950806,
      "step": 3750,
      "unclipped_grad_norm": 0.9619976878166199
    },
    {
      "bits_per_byte": 1.808596729732132,
      "learning_rate": 0.00018941887213483482,
      "loss_nats": 1.2536237239837646,
      "step": 3760,
      "unclipped_grad_norm": 1.012074589729309
    },
    {
      "bits_per_byte": 1.7638779728374354,
      "learning_rate": 0.00018886982393685845,
      "loss_nats": 1.22262704372406,
      "step": 3770,
      "unclipped_grad_norm": 1.050723910331726
    },
    {
      "bits_per_byte": 1.589899463311364,
      "learning_rate": 0.00018832036786945008,
      "loss_nats": 1.102034330368042,
      "step": 3780,
      "unclipped_grad_norm": 0.9615285992622375
    },
    {
      "bits_per_byte": 1.6942863970443558,
      "learning_rate": 0.0001877705133212979,
      "loss_nats": 1.1743898391723633,
      "step": 3790,
      "unclipped_grad_norm": 0.9870805144309998
    },
    {
      "bits_per_byte": 1.6994689222454558,
      "learning_rate": 0.00018722026968789906,
      "loss_nats": 1.1779820919036865,
      "step": 3800,
      "unclipped_grad_norm": 0.9850614070892334
    },
    {
      "bits_per_byte": 1.7199389852817506,
      "learning_rate": 0.00018666964637139907,
      "loss_nats": 1.1921708583831787,
      "step": 3810,
      "unclipped_grad_norm": 0.9774236083030701
    },
    {
      "bits_per_byte": 1.718822817877651,
      "learning_rate": 0.00018611865278043115,
      "loss_nats": 1.1913971900939941,
      "step": 3820,
      "unclipped_grad_norm": 1.0395604372024536
    },
    {
      "bits_per_byte": 1.762180848038721,
      "learning_rate": 0.00018556729832995555,
      "loss_nats": 1.221450686454773,
      "step": 3830,
      "unclipped_grad_norm": 1.0331096649169922
    },
    {
      "bits_per_byte": 1.68565630762406,
      "learning_rate": 0.00018501559244109864,
      "loss_nats": 1.168407917022705,
      "step": 3840,
      "unclipped_grad_norm": 1.0491844415664673
    },
    {
      "bits_per_byte": 1.6311636046981792,
      "learning_rate": 0.00018446354454099178,
      "loss_nats": 1.13063645362854,
      "step": 3850,
      "unclipped_grad_norm": 0.9598016738891602
    },
    {
      "bits_per_byte": 1.7078809376644575,
      "learning_rate": 0.00018391116406261048,
      "loss_nats": 1.1838128566741943,
      "step": 3860,
      "unclipped_grad_norm": 1.0586371421813965
    },
    {
      "bits_per_byte": 1.778100422115837,
      "learning_rate": 0.00018335846044461305,
      "loss_nats": 1.232485294342041,
      "step": 3870,
      "unclipped_grad_norm": 0.9732621908187866
    },
    {
      "bits_per_byte": 1.6634643542873266,
      "learning_rate": 0.0001828054431311793,
      "loss_nats": 1.1530256271362305,
      "step": 3880,
      "unclipped_grad_norm": 0.9620565176010132
    },
    {
      "bits_per_byte": 1.6362388127251406,
      "learning_rate": 0.00018225212157184934,
      "loss_nats": 1.1341543197631836,
      "step": 3890,
      "unclipped_grad_norm": 0.9925653338432312
    },
    {
      "bits_per_byte": 1.721069255263221,
      "learning_rate": 0.00018169850522136192,
      "loss_nats": 1.1929543018341064,
      "step": 3900,
      "unclipped_grad_norm": 1.121357798576355
    },
    {
      "bits_per_byte": 1.7172674067832479,
      "learning_rate": 0.000181144603539493,
      "loss_nats": 1.1903190612792969,
      "step": 3910,
      "unclipped_grad_norm": 1.0240309238433838
    },
    {
      "bits_per_byte": 1.671385531220766,
      "learning_rate": 0.00018059042599089412,
      "loss_nats": 1.1585161685943604,
      "step": 3920,
      "unclipped_grad_norm": 0.9621630311012268
    },
    {
      "bits_per_byte": 1.7072636919305109,
      "learning_rate": 0.00018003598204493048,
      "loss_nats": 1.1833850145339966,
      "step": 3930,
      "unclipped_grad_norm": 0.9956491589546204
    },
    {
      "bits_per_byte": 1.7254322831329596,
      "learning_rate": 0.00017948128117551936,
      "loss_nats": 1.1959785223007202,
      "step": 3940,
      "unclipped_grad_norm": 0.9730517268180847
    },
    {
      "bits_per_byte": 1.7623370082857046,
      "learning_rate": 0.00017892633286096816,
      "loss_nats": 1.221558928489685,
      "step": 3950,
      "unclipped_grad_norm": 0.9878451228141785
    },
    {
      "bits_per_byte": 1.7644881672826966,
      "learning_rate": 0.00017837114658381247,
      "loss_nats": 1.2230499982833862,
      "step": 3960,
      "unclipped_grad_norm": 1.003400444984436
    },
    {
      "bits_per_byte": 1.682019734471566,
      "learning_rate": 0.00017781573183065386,
      "loss_nats": 1.1658872365951538,
      "step": 3970,
      "unclipped_grad_norm": 1.0887466669082642
    },
    {
      "bits_per_byte": 1.7137744391442242,
      "learning_rate": 0.0001772600980919981,
      "loss_nats": 1.1878979206085205,
      "step": 3980,
      "unclipped_grad_norm": 1.0592182874679565
    },
    {
      "bits_per_byte": 1.7370323409510051,
      "learning_rate": 0.0001767042548620927,
      "loss_nats": 1.2040190696716309,
      "step": 3990,
      "unclipped_grad_norm": 1.0668442249298096
    },
    {
      "bits_per_byte": 1.620528197568823,
      "learning_rate": 0.00017614821163876486,
      "loss_nats": 1.1232645511627197,
      "step": 4000,
      "unclipped_grad_norm": 0.979034423828125
    },
    {
      "bits_per_byte": 1.7224021208074078,
      "learning_rate": 0.00017559197792325906,
      "loss_nats": 1.193878173828125,
      "step": 4010,
      "unclipped_grad_norm": 1.0592741966247559
    },
    {
      "bits_per_byte": 1.6779786581242426,
      "learning_rate": 0.0001750355632200748,
      "loss_nats": 1.163086175918579,
      "step": 4020,
      "unclipped_grad_norm": 1.0186548233032227
    },
    {
      "bits_per_byte": 1.7290010951210137,
      "learning_rate": 0.00017447897703680418,
      "loss_nats": 1.1984522342681885,
      "step": 4030,
      "unclipped_grad_norm": 1.0059210062026978
    },
    {
      "bits_per_byte": 1.6595417739864478,
      "learning_rate": 0.0001739222288839694,
      "loss_nats": 1.1503067016601562,
      "step": 4040,
      "unclipped_grad_norm": 1.009366750717163
    },
    {
      "bits_per_byte": 1.6963183720643,
      "learning_rate": 0.00017336532827486026,
      "loss_nats": 1.1757982969284058,
      "step": 4050,
      "unclipped_grad_norm": 1.0469119548797607
    },
    {
      "bits_per_byte": 1.706078903448717,
      "learning_rate": 0.00017280828472537163,
      "loss_nats": 1.1825637817382812,
      "step": 4060,
      "unclipped_grad_norm": 1.0410232543945312
    },
    {
      "bits_per_byte": 1.7226642223673228,
      "learning_rate": 0.0001722511077538408,
      "loss_nats": 1.1940598487854004,
      "step": 4070,
      "unclipped_grad_norm": 1.0424585342407227
    },
    {
      "bits_per_byte": 1.7834826191745887,
      "learning_rate": 0.00017169380688088497,
      "loss_nats": 1.2362159490585327,
      "step": 4080,
      "unclipped_grad_norm": 1.0965845584869385
    },
    {
      "bits_per_byte": 1.6886807945221334,
      "learning_rate": 0.0001711363916292384,
      "loss_nats": 1.1705043315887451,
      "step": 4090,
      "unclipped_grad_norm": 1.0564258098602295
    },
    {
      "bits_per_byte": 1.6871728506393684,
      "learning_rate": 0.00017057887152358974,
      "loss_nats": 1.1694591045379639,
      "step": 4100,
      "unclipped_grad_norm": 1.032038688659668
    },
    {
      "bits_per_byte": 1.6672990514536148,
      "learning_rate": 0.0001700212560904194,
      "loss_nats": 1.1556836366653442,
      "step": 4110,
      "unclipped_grad_norm": 1.0212913751602173
    },
    {
      "bits_per_byte": 1.7325631997856843,
      "learning_rate": 0.00016946355485783656,
      "loss_nats": 1.2009212970733643,
      "step": 4120,
      "unclipped_grad_norm": 1.1421489715576172
    },
    {
      "bits_per_byte": 1.6623953101295634,
      "learning_rate": 0.0001689057773554166,
      "loss_nats": 1.1522846221923828,
      "step": 4130,
      "unclipped_grad_norm": 1.0443917512893677
    },
    {
      "bits_per_byte": 1.7315172013031366,
      "learning_rate": 0.00016834793311403806,
      "loss_nats": 1.2001962661743164,
      "step": 4140,
      "unclipped_grad_norm": 1.0389643907546997
    },
    {
      "bits_per_byte": 1.6632394009799982,
      "learning_rate": 0.00016779003166571981,
      "loss_nats": 1.152869701385498,
      "step": 4150,
      "unclipped_grad_norm": 0.9869729280471802
    },
    {
      "bits_per_byte": 1.697900440469587,
      "learning_rate": 0.00016723208254345838,
      "loss_nats": 1.1768949031829834,
      "step": 4160,
      "unclipped_grad_norm": 1.018755316734314
    },
    {
      "bits_per_byte": 1.74134876152236,
      "learning_rate": 0.00016667409528106481,
      "loss_nats": 1.2070109844207764,
      "step": 4170,
      "unclipped_grad_norm": 1.0534698963165283
    },
    {
      "bits_per_byte": 1.7750522016119445,
      "learning_rate": 0.00016611607941300184,
      "loss_nats": 1.230372428894043,
      "step": 4180,
      "unclipped_grad_norm": 1.1320860385894775
    },
    {
      "bits_per_byte": 1.6878769476120008,
      "learning_rate": 0.0001655580444742211,
      "loss_nats": 1.1699471473693848,
      "step": 4190,
      "unclipped_grad_norm": 1.0554869174957275
    },
    {
      "bits_per_byte": 1.725129765650092,
      "learning_rate": 0.00016499999999999997,
      "loss_nats": 1.1957688331604004,
      "step": 4200,
      "unclipped_grad_norm": 1.0452762842178345
    },
    {
      "bits_per_byte": 1.7145272072070508,
      "learning_rate": 0.0001644419555257789,
      "loss_nats": 1.1884196996688843,
      "step": 4210,
      "unclipped_grad_norm": 0.9850034117698669
    },
    {
      "bits_per_byte": 1.7283826455085112,
      "learning_rate": 0.0001638839205869981,
      "loss_nats": 1.1980235576629639,
      "step": 4220,
      "unclipped_grad_norm": 1.0109167098999023
    },
    {
      "bits_per_byte": 1.7140664656853892,
      "learning_rate": 0.0001633259047189352,
      "loss_nats": 1.1881003379821777,
      "step": 4230,
      "unclipped_grad_norm": 1.0496208667755127
    },
    {
      "bits_per_byte": 1.659261958213494,
      "learning_rate": 0.0001627679174565416,
      "loss_nats": 1.1501127481460571,
      "step": 4240,
      "unclipped_grad_norm": 0.9920905232429504
    },
    {
      "bits_per_byte": 1.6850357942197471,
      "learning_rate": 0.00016220996833428016,
      "loss_nats": 1.1679778099060059,
      "step": 4250,
      "unclipped_grad_norm": 1.0322012901306152
    },
    {
      "bits_per_byte": 1.667805196395104,
      "learning_rate": 0.00016165206688596191,
      "loss_nats": 1.1560344696044922,
      "step": 4260,
      "unclipped_grad_norm": 1.0430845022201538
    },
    {
      "bits_per_byte": 1.6625848350108143,
      "learning_rate": 0.00016109422264458335,
      "loss_nats": 1.1524159908294678,
      "step": 4270,
      "unclipped_grad_norm": 1.0265511274337769
    },
    {
      "bits_per_byte": 1.6648945620119018,
      "learning_rate": 0.00016053644514216336,
      "loss_nats": 1.1540169715881348,
      "step": 4280,
      "unclipped_grad_norm": 0.9929917454719543
    },
    {
      "bits_per_byte": 1.72171487813456,
      "learning_rate": 0.00015997874390958058,
      "loss_nats": 1.19340181350708,
      "step": 4290,
      "unclipped_grad_norm": 1.0577914714813232
    },
    {
      "bits_per_byte": 1.6424783432984433,
      "learning_rate": 0.00015942112847641018,
      "loss_nats": 1.138479232788086,
      "step": 4300,
      "unclipped_grad_norm": 1.0359195470809937
    },
    {
      "bits_per_byte": 1.7237528725472844,
      "learning_rate": 0.00015886360837076156,
      "loss_nats": 1.1948144435882568,
      "step": 4310,
      "unclipped_grad_norm": 1.124787449836731
    },
    {
      "bits_per_byte": 1.7081155220002342,
      "learning_rate": 0.00015830619311911495,
      "loss_nats": 1.1839754581451416,
      "step": 4320,
      "unclipped_grad_norm": 1.0496878623962402
    },
    {
      "bits_per_byte": 1.6609448084521865,
      "learning_rate": 0.00015774889224615915,
      "loss_nats": 1.1512792110443115,
      "step": 4330,
      "unclipped_grad_norm": 1.0365654230117798
    },
    {
      "bits_per_byte": 1.6981388084236828,
      "learning_rate": 0.00015719171527462834,
      "loss_nats": 1.1770601272583008,
      "step": 4340,
      "unclipped_grad_norm": 1.0712857246398926
    },
    {
      "bits_per_byte": 1.679871155214337,
      "learning_rate": 0.0001566346717251397,
      "loss_nats": 1.164397954940796,
      "step": 4350,
      "unclipped_grad_norm": 1.0368938446044922
    },
    {
      "bits_per_byte": 1.658068570599157,
      "learning_rate": 0.00015607777111603056,
      "loss_nats": 1.1492855548858643,
      "step": 4360,
      "unclipped_grad_norm": 1.054489254951477
    },
    {
      "bits_per_byte": 1.6791636185886873,
      "learning_rate": 0.00015552102296319576,
      "loss_nats": 1.163907527923584,
      "step": 4370,
      "unclipped_grad_norm": 1.05704927444458
    },
    {
      "bits_per_byte": 1.733114060216398,
      "learning_rate": 0.00015496443677992516,
      "loss_nats": 1.2013031244277954,
      "step": 4380,
      "unclipped_grad_norm": 1.066345453262329
    },
    {
      "bits_per_byte": 1.6866214742607022,
      "learning_rate": 0.0001544080220767409,
      "loss_nats": 1.169076919555664,
      "step": 4390,
      "unclipped_grad_norm": 1.039746880531311
    },
    {
      "bits_per_byte": 1.656244522604106,
      "learning_rate": 0.00015385178836123511,
      "loss_nats": 1.1480212211608887,
      "step": 4400,
      "unclipped_grad_norm": 1.0477714538574219
    },
    {
      "bits_per_byte": 1.7190543065257124,
      "learning_rate": 0.00015329574513790726,
      "loss_nats": 1.1915576457977295,
      "step": 4410,
      "unclipped_grad_norm": 1.0500718355178833
    },
    {
      "bits_per_byte": 1.6606173534849438,
      "learning_rate": 0.00015273990190800186,
      "loss_nats": 1.1510522365570068,
      "step": 4420,
      "unclipped_grad_norm": 1.0457862615585327
    },
    {
      "bits_per_byte": 1.7152531459763425,
      "learning_rate": 0.0001521842681693461,
      "loss_nats": 1.1889228820800781,
      "step": 4430,
      "unclipped_grad_norm": 1.0248607397079468
    },
    {
      "bits_per_byte": 1.6480127450032085,
      "learning_rate": 0.00015162885341618752,
      "loss_nats": 1.14231538772583,
      "step": 4440,
      "unclipped_grad_norm": 1.026747703552246
    },
    {
      "bits_per_byte": 1.64553137931656,
      "learning_rate": 0.00015107366713903178,
      "loss_nats": 1.1405954360961914,
      "step": 4450,
      "unclipped_grad_norm": 1.0325950384140015
    },
    {
      "bits_per_byte": 1.6368648295742814,
      "learning_rate": 0.0001505187188244806,
      "loss_nats": 1.1345882415771484,
      "step": 4460,
      "unclipped_grad_norm": 1.0030871629714966
    },
    {
      "bits_per_byte": 1.686227977955528,
      "learning_rate": 0.00014996401795506952,
      "loss_nats": 1.1688041687011719,
      "step": 4470,
      "unclipped_grad_norm": 0.9844985604286194
    },
    {
      "bits_per_byte": 1.7063241507088474,
      "learning_rate": 0.00014940957400910585,
      "loss_nats": 1.1827337741851807,
      "step": 4480,
      "unclipped_grad_norm": 1.1298258304595947
    },
    {
      "bits_per_byte": 1.6590796566035795,
      "learning_rate": 0.00014885539646050699,
      "loss_nats": 1.1499863862991333,
      "step": 4490,
      "unclipped_grad_norm": 0.9567995071411133
    },
    {
      "bits_per_byte": 1.699403568838128,
      "learning_rate": 0.00014830149477863805,
      "loss_nats": 1.1779367923736572,
      "step": 4500,
      "unclipped_grad_norm": 1.0041329860687256
    },
    {
      "bits_per_byte": 1.630450564627702,
      "learning_rate": 0.00014774787842815066,
      "loss_nats": 1.1301422119140625,
      "step": 4510,
      "unclipped_grad_norm": 1.0097919702529907
    },
    {
      "bits_per_byte": 1.677602188101504,
      "learning_rate": 0.00014719455686882066,
      "loss_nats": 1.1628252267837524,
      "step": 4520,
      "unclipped_grad_norm": 1.0421855449676514
    },
    {
      "bits_per_byte": 1.6955814264053528,
      "learning_rate": 0.00014664153955538695,
      "loss_nats": 1.1752874851226807,
      "step": 4530,
      "unclipped_grad_norm": 1.0619962215423584
    },
    {
      "bits_per_byte": 1.738972993183341,
      "learning_rate": 0.00014608883593738946,
      "loss_nats": 1.2053642272949219,
      "step": 4540,
      "unclipped_grad_norm": 1.0259038209915161
    },
    {
      "bits_per_byte": 1.6138776284599583,
      "learning_rate": 0.00014553645545900817,
      "loss_nats": 1.118654727935791,
      "step": 4550,
      "unclipped_grad_norm": 1.1216750144958496
    },
    {
      "bits_per_byte": 1.6847919228208237,
      "learning_rate": 0.0001449844075589013,
      "loss_nats": 1.1678087711334229,
      "step": 4560,
      "unclipped_grad_norm": 1.1666667461395264
    },
    {
      "bits_per_byte": 1.6566538413131595,
      "learning_rate": 0.0001444327016700444,
      "loss_nats": 1.1483049392700195,
      "step": 4570,
      "unclipped_grad_norm": 0.9492276310920715
    },
    {
      "bits_per_byte": 1.6127442627907724,
      "learning_rate": 0.0001438813472195688,
      "loss_nats": 1.1178691387176514,
      "step": 4580,
      "unclipped_grad_norm": 1.0188902616500854
    },
    {
      "bits_per_byte": 1.6539726317862082,
      "learning_rate": 0.00014333035362860093,
      "loss_nats": 1.1464464664459229,
      "step": 4590,
      "unclipped_grad_norm": 1.0299646854400635
    },
    {
      "bits_per_byte": 1.6561746976478557,
      "learning_rate": 0.00014277973031210088,
      "loss_nats": 1.147972822189331,
      "step": 4600,
      "unclipped_grad_norm": 1.0103821754455566
    },
    {
      "bits_per_byte": 1.719434732149421,
      "learning_rate": 0.00014222948667870207,
      "loss_nats": 1.1918213367462158,
      "step": 4610,
      "unclipped_grad_norm": 1.0808284282684326
    },
    {
      "bits_per_byte": 1.7013596995090414,
      "learning_rate": 0.00014167963213054991,
      "loss_nats": 1.1792926788330078,
      "step": 4620,
      "unclipped_grad_norm": 1.0917437076568604
    },
    {
      "bits_per_byte": 1.683035292024911,
      "learning_rate": 0.00014113017606314152,
      "loss_nats": 1.1665911674499512,
      "step": 4630,
      "unclipped_grad_norm": 1.0124032497406006
    },
    {
      "bits_per_byte": 1.7275268598378177,
      "learning_rate": 0.00014058112786516518,
      "loss_nats": 1.1974303722381592,
      "step": 4640,
      "unclipped_grad_norm": 1.0515177249908447
    },
    {
      "bits_per_byte": 1.6549154406782383,
      "learning_rate": 0.0001400324969183396,
      "loss_nats": 1.1470999717712402,
      "step": 4650,
      "unclipped_grad_norm": 1.1102732419967651
    },
    {
      "bits_per_byte": 1.6181718632693518,
      "learning_rate": 0.00013948429259725406,
      "loss_nats": 1.1216312646865845,
      "step": 4660,
      "unclipped_grad_norm": 1.066238522529602
    },
    {
      "bits_per_byte": 1.7593243881905407,
      "learning_rate": 0.00013893652426920776,
      "loss_nats": 1.219470739364624,
      "step": 4670,
      "unclipped_grad_norm": 1.1023285388946533
    },
    {
      "bits_per_byte": 1.7532637195741354,
      "learning_rate": 0.00013838920129405016,
      "loss_nats": 1.2152698040008545,
      "step": 4680,
      "unclipped_grad_norm": 1.0341681241989136
    },
    {
      "bits_per_byte": 1.7266088164475122,
      "learning_rate": 0.0001378423330240207,
      "loss_nats": 1.196794033050537,
      "step": 4690,
      "unclipped_grad_norm": 1.0434808731079102
    },
    {
      "bits_per_byte": 1.6483247215318737,
      "learning_rate": 0.00013729592880358936,
      "loss_nats": 1.1425316333770752,
      "step": 4700,
      "unclipped_grad_norm": 1.045780897140503
    },
    {
      "bits_per_byte": 1.7162458298371228,
      "learning_rate": 0.00013674999796929659,
      "loss_nats": 1.1896109580993652,
      "step": 4710,
      "unclipped_grad_norm": 1.1512850522994995
    },
    {
      "bits_per_byte": 1.6600910865733036,
      "learning_rate": 0.00013620454984959417,
      "loss_nats": 1.1506874561309814,
      "step": 4720,
      "unclipped_grad_norm": 1.0870006084442139
    },
    {
      "bits_per_byte": 1.579696248583626,
      "learning_rate": 0.00013565959376468542,
      "loss_nats": 1.0949620008468628,
      "step": 4730,
      "unclipped_grad_norm": 1.066159725189209
    },
    {
      "bits_per_byte": 1.6295564268258662,
      "learning_rate": 0.00013511513902636634,
      "loss_nats": 1.129522442817688,
      "step": 4740,
      "unclipped_grad_norm": 1.0375505685806274
    },
    {
      "bits_per_byte": 1.6749376608916864,
      "learning_rate": 0.0001345711949378662,
      "loss_nats": 1.1609783172607422,
      "step": 4750,
      "unclipped_grad_norm": 1.0134141445159912
    },
    {
      "bits_per_byte": 1.6434221840863785,
      "learning_rate": 0.00013402777079368853,
      "loss_nats": 1.1391334533691406,
      "step": 4760,
      "unclipped_grad_norm": 1.0448397397994995
    },
    {
      "bits_per_byte": 1.6362174868764336,
      "learning_rate": 0.0001334848758794528,
      "loss_nats": 1.1341395378112793,
      "step": 4770,
      "unclipped_grad_norm": 1.055105447769165
    },
    {
      "bits_per_byte": 1.666649472981306,
      "learning_rate": 0.00013294251947173504,
      "loss_nats": 1.155233383178711,
      "step": 4780,
      "unclipped_grad_norm": 1.0775461196899414
    },
    {
      "bits_per_byte": 1.6314788488972107,
      "learning_rate": 0.00013240071083790996,
      "loss_nats": 1.1308549642562866,
      "step": 4790,
      "unclipped_grad_norm": 1.065148949623108
    },
    {
      "bits_per_byte": 1.6269109896902945,
      "learning_rate": 0.0001318594592359921,
      "loss_nats": 1.1276887655258179,
      "step": 4800,
      "unclipped_grad_norm": 1.1373785734176636
    },
    {
      "bits_per_byte": 1.6901620810940148,
      "learning_rate": 0.0001313187739144781,
      "loss_nats": 1.171531081199646,
      "step": 4810,
      "unclipped_grad_norm": 1.1410114765167236
    },
    {
      "bits_per_byte": 1.6112782826748178,
      "learning_rate": 0.0001307786641121882,
      "loss_nats": 1.1168529987335205,
      "step": 4820,
      "unclipped_grad_norm": 1.0628036260604858
    },
    {
      "bits_per_byte": 1.649350942009572,
      "learning_rate": 0.00013023913905810876,
      "loss_nats": 1.1432429552078247,
      "step": 4830,
      "unclipped_grad_norm": 1.1659979820251465
    },
    {
      "bits_per_byte": 1.6571300612733986,
      "learning_rate": 0.00012970020797123432,
      "loss_nats": 1.1486350297927856,
      "step": 4840,
      "unclipped_grad_norm": 1.1067216396331787
    },
    {
      "bits_per_byte": 1.668376866726572,
      "learning_rate": 0.00012916188006041027,
      "loss_nats": 1.156430721282959,
      "step": 4850,
      "unclipped_grad_norm": 0.9945695996284485
    },
    {
      "bits_per_byte": 1.600552412671108,
      "learning_rate": 0.00012862416452417516,
      "loss_nats": 1.1094183921813965,
      "step": 4860,
      "unclipped_grad_norm": 1.0181769132614136
    },
    {
      "bits_per_byte": 1.6771526254521485,
      "learning_rate": 0.00012808707055060405,
      "loss_nats": 1.1625136137008667,
      "step": 4870,
      "unclipped_grad_norm": 1.0462807416915894
    },
    {
      "bits_per_byte": 1.539689988305221,
      "learning_rate": 0.000127550607317151,
      "loss_nats": 1.0672317743301392,
      "step": 4880,
      "unclipped_grad_norm": 1.0612690448760986
    },
    {
      "bits_per_byte": 1.5967718900398415,
      "learning_rate": 0.00012701478399049248,
      "loss_nats": 1.1067979335784912,
      "step": 4890,
      "unclipped_grad_norm": 1.0629737377166748
    },
    {
      "bits_per_byte": 1.6387246499607118,
      "learning_rate": 0.0001264796097263708,
      "loss_nats": 1.1358773708343506,
      "step": 4900,
      "unclipped_grad_norm": 1.0522006750106812
    },
    {
      "bits_per_byte": 1.7368791044090865,
      "learning_rate": 0.0001259450936694375,
      "loss_nats": 1.2039128541946411,
      "step": 4910,
      "unclipped_grad_norm": 1.1116063594818115
    },
    {
      "bits_per_byte": 1.6773633041994556,
      "learning_rate": 0.00012541124495309727,
      "loss_nats": 1.1626596450805664,
      "step": 4920,
      "unclipped_grad_norm": 1.0861767530441284
    },
    {
      "bits_per_byte": 1.6375162998557495,
      "learning_rate": 0.00012487807269935167,
      "loss_nats": 1.1350398063659668,
      "step": 4930,
      "unclipped_grad_norm": 1.0190138816833496
    },
    {
      "bits_per_byte": 1.6159926711002692,
      "learning_rate": 0.00012434558601864349,
      "loss_nats": 1.1201207637786865,
      "step": 4940,
      "unclipped_grad_norm": 1.0380074977874756
    },
    {
      "bits_per_byte": 1.5895295286293578,
      "learning_rate": 0.0001238137940097009,
      "loss_nats": 1.1017779111862183,
      "step": 4950,
      "unclipped_grad_norm": 1.0411629676818848
    },
    {
      "bits_per_byte": 1.5717379233976003,
      "learning_rate": 0.0001232827057593821,
      "loss_nats": 1.08944571018219,
      "step": 4960,
      "unclipped_grad_norm": 1.0460548400878906
    },
    {
      "bits_per_byte": 1.6461665112461963,
      "learning_rate": 0.00012275233034251992,
      "loss_nats": 1.1410356760025024,
      "step": 4970,
      "unclipped_grad_norm": 1.112047553062439
    },
    {
      "bits_per_byte": 1.7016685803499911,
      "learning_rate": 0.0001222226768217669,
      "loss_nats": 1.179506778717041,
      "step": 4980,
      "unclipped_grad_norm": 1.062510371208191
    },
    {
      "bits_per_byte": 1.636959248049605,
      "learning_rate": 0.00012169375424744034,
      "loss_nats": 1.1346536874771118,
      "step": 4990,
      "unclipped_grad_norm": 1.1910533905029297
    },
    {
      "bits_per_byte": 1.6019551031715449,
      "learning_rate": 0.00012116557165736772,
      "loss_nats": 1.1103906631469727,
      "step": 5000,
      "unclipped_grad_norm": 1.0508692264556885
    },
    {
      "bits_per_byte": 1.5837816964548719,
      "learning_rate": 0.00012063813807673214,
      "loss_nats": 1.0977938175201416,
      "step": 5010,
      "unclipped_grad_norm": 1.0227967500686646
    },
    {
      "bits_per_byte": 1.639255904369227,
      "learning_rate": 0.00012011146251791832,
      "loss_nats": 1.136245608329773,
      "step": 5020,
      "unclipped_grad_norm": 1.0390269756317139
    },
    {
      "bits_per_byte": 1.6173587292960725,
      "learning_rate": 0.0001195855539803584,
      "loss_nats": 1.1210676431655884,
      "step": 5030,
      "unclipped_grad_norm": 1.083649754524231
    },
    {
      "bits_per_byte": 1.591486175248224,
      "learning_rate": 0.00011906042145037817,
      "loss_nats": 1.1031341552734375,
      "step": 5040,
      "unclipped_grad_norm": 1.0160164833068848
    },
    {
      "bits_per_byte": 1.5612364907532663,
      "learning_rate": 0.0001185360739010438,
      "loss_nats": 1.0821666717529297,
      "step": 5050,
      "unclipped_grad_norm": 0.9697359800338745
    },
    {
      "bits_per_byte": 1.5996653261579574,
      "learning_rate": 0.00011801252029200808,
      "loss_nats": 1.1088035106658936,
      "step": 5060,
      "unclipped_grad_norm": 1.0825191736221313
    },
    {
      "bits_per_byte": 1.6612003746713688,
      "learning_rate": 0.00011748976956935777,
      "loss_nats": 1.151456356048584,
      "step": 5070,
      "unclipped_grad_norm": 1.051671028137207
    },
    {
      "bits_per_byte": 1.6135078657606032,
      "learning_rate": 0.00011696783066546024,
      "loss_nats": 1.1183984279632568,
      "step": 5080,
      "unclipped_grad_norm": 1.0758769512176514
    },
    {
      "bits_per_byte": 1.6053440213067967,
      "learning_rate": 0.00011644671249881147,
      "loss_nats": 1.1127396821975708,
      "step": 5090,
      "unclipped_grad_norm": 1.0736517906188965
    },
    {
      "bits_per_byte": 1.620226196033908,
      "learning_rate": 0.00011592642397388293,
      "loss_nats": 1.1230552196502686,
      "step": 5100,
      "unclipped_grad_norm": 1.0084763765335083
    },
    {
      "bits_per_byte": 1.5469340695422136,
      "learning_rate": 0.00011540697398097015,
      "loss_nats": 1.0722529888153076,
      "step": 5110,
      "unclipped_grad_norm": 1.001043438911438
    },
    {
      "bits_per_byte": 1.627192525289757,
      "learning_rate": 0.00011488837139604016,
      "loss_nats": 1.1278839111328125,
      "step": 5120,
      "unclipped_grad_norm": 1.0516539812088013
    },
    {
      "bits_per_byte": 1.642070228467946,
      "learning_rate": 0.00011437062508058039,
      "loss_nats": 1.138196349143982,
      "step": 5130,
      "unclipped_grad_norm": 1.082858681678772
    },
    {
      "bits_per_byte": 1.6022958008029042,
      "learning_rate": 0.00011385374388144673,
      "loss_nats": 1.1106268167495728,
      "step": 5140,
      "unclipped_grad_norm": 1.0696102380752563
    },
    {
      "bits_per_byte": 1.6803482350878305,
      "learning_rate": 0.00011333773663071286,
      "loss_nats": 1.1647286415100098,
      "step": 5150,
      "unclipped_grad_norm": 1.045096755027771
    },
    {
      "bits_per_byte": 1.5608273440268638,
      "learning_rate": 0.00011282261214551897,
      "loss_nats": 1.0818830728530884,
      "step": 5160,
      "unclipped_grad_norm": 1.0852984189987183
    },
    {
      "bits_per_byte": 1.7284662290768305,
      "learning_rate": 0.00011230837922792112,
      "loss_nats": 1.1980814933776855,
      "step": 5170,
      "unclipped_grad_norm": 1.2249319553375244
    },
    {
      "bits_per_byte": 1.6274811121779047,
      "learning_rate": 0.0001117950466647412,
      "loss_nats": 1.1280839443206787,
      "step": 5180,
      "unclipped_grad_norm": 1.1194853782653809
    },
    {
      "bits_per_byte": 1.567900990456851,
      "learning_rate": 0.00011128262322741627,
      "loss_nats": 1.086786150932312,
      "step": 5190,
      "unclipped_grad_norm": 1.0312306880950928
    },
    {
      "bits_per_byte": 1.5985697966719614,
      "learning_rate": 0.00011077111767184913,
      "loss_nats": 1.108044147491455,
      "step": 5200,
      "unclipped_grad_norm": 1.137611985206604
    },
    {
      "bits_per_byte": 1.6023676895509649,
      "learning_rate": 0.00011026053873825837,
      "loss_nats": 1.110676646232605,
      "step": 5210,
      "unclipped_grad_norm": 1.031370759010315
    },
    {
      "bits_per_byte": 1.6946726700781938,
      "learning_rate": 0.0001097508951510293,
      "loss_nats": 1.1746575832366943,
      "step": 5220,
      "unclipped_grad_norm": 1.113584280014038
    },
    {
      "bits_per_byte": 1.5914005278880945,
      "learning_rate": 0.0001092421956185646,
      "loss_nats": 1.1030747890472412,
      "step": 5230,
      "unclipped_grad_norm": 1.1469851732254028
    },
    {
      "bits_per_byte": 1.6053940682581977,
      "learning_rate": 0.00010873444883313581,
      "loss_nats": 1.11277437210083,
      "step": 5240,
      "unclipped_grad_norm": 1.0413556098937988
    },
    {
      "bits_per_byte": 1.5800591319769466,
      "learning_rate": 0.00010822766347073447,
      "loss_nats": 1.095213532447815,
      "step": 5250,
      "unclipped_grad_norm": 1.0926955938339233
    },
    {
      "bits_per_byte": 1.5918442431273205,
      "learning_rate": 0.00010772184819092424,
      "loss_nats": 1.1033823490142822,
      "step": 5260,
      "unclipped_grad_norm": 1.0704180002212524
    },
    {
      "bits_per_byte": 1.589662299235824,
      "learning_rate": 0.0001072170116366926,
      "loss_nats": 1.1018699407577515,
      "step": 5270,
      "unclipped_grad_norm": 1.0476908683776855
    },
    {
      "bits_per_byte": 1.6108980290337602,
      "learning_rate": 0.00010671316243430345,
      "loss_nats": 1.1165894269943237,
      "step": 5280,
      "unclipped_grad_norm": 1.1081280708312988
    },
    {
      "bits_per_byte": 1.608212003992585,
      "learning_rate": 0.00010621030919314944,
      "loss_nats": 1.1147276163101196,
      "step": 5290,
      "unclipped_grad_norm": 1.1043791770935059
    },
    {
      "bits_per_byte": 1.614489714714379,
      "learning_rate": 0.000105708460505605,
      "loss_nats": 1.1190789937973022,
      "step": 5300,
      "unclipped_grad_norm": 1.0849814414978027
    },
    {
      "bits_per_byte": 1.6119760162893682,
      "learning_rate": 0.00010520762494687964,
      "loss_nats": 1.117336630821228,
      "step": 5310,
      "unclipped_grad_norm": 1.1325145959854126
    },
    {
      "bits_per_byte": 1.5916898027068456,
      "learning_rate": 0.00010470781107487111,
      "loss_nats": 1.1032752990722656,
      "step": 5320,
      "unclipped_grad_norm": 1.072629451751709
    },
    {
      "bits_per_byte": 1.6015805249579658,
      "learning_rate": 0.00010420902743001955,
      "loss_nats": 1.110131025314331,
      "step": 5330,
      "unclipped_grad_norm": 1.1142079830169678
    },
    {
      "bits_per_byte": 1.553969879789011,
      "learning_rate": 0.00010371128253516117,
      "loss_nats": 1.07712984085083,
      "step": 5340,
      "unclipped_grad_norm": 1.0742131471633911
    },
    {
      "bits_per_byte": 1.626766180298268,
      "learning_rate": 0.000103214584895383,
      "loss_nats": 1.1275883913040161,
      "step": 5350,
      "unclipped_grad_norm": 1.127328634262085
    },
    {
      "bits_per_byte": 1.5508274127924457,
      "learning_rate": 0.00010271894299787715,
      "loss_nats": 1.0749516487121582,
      "step": 5360,
      "unclipped_grad_norm": 1.1100437641143799
    },
    {
      "bits_per_byte": 1.5306884163590617,
      "learning_rate": 0.00010222436531179622,
      "loss_nats": 1.0609923601150513,
      "step": 5370,
      "unclipped_grad_norm": 1.0785573720932007
    },
    {
      "bits_per_byte": 1.633070548330946,
      "learning_rate": 0.0001017308602881082,
      "loss_nats": 1.131958246231079,
      "step": 5380,
      "unclipped_grad_norm": 1.0940150022506714
    },
    {
      "bits_per_byte": 1.5790724675089465,
      "learning_rate": 0.00010123843635945242,
      "loss_nats": 1.094529628753662,
      "step": 5390,
      "unclipped_grad_norm": 1.1135237216949463
    },
    {
      "bits_per_byte": 1.6121983898569339,
      "learning_rate": 0.00010074710193999505,
      "loss_nats": 1.1174907684326172,
      "step": 5400,
      "unclipped_grad_norm": 1.1351183652877808
    },
    {
      "bits_per_byte": 1.6233261833157102,
      "learning_rate": 0.0001002568654252858,
      "loss_nats": 1.1252039670944214,
      "step": 5410,
      "unclipped_grad_norm": 1.1545859575271606
    },
    {
      "bits_per_byte": 1.5857118577455054,
      "learning_rate": 9.976773519211404e-05,
      "loss_nats": 1.09913170337677,
      "step": 5420,
      "unclipped_grad_norm": 1.1009039878845215
    },
    {
      "bits_per_byte": 1.6290494219711227,
      "learning_rate": 9.92797195983659e-05,
      "loss_nats": 1.1291710138320923,
      "step": 5430,
      "unclipped_grad_norm": 1.1000012159347534
    },
    {
      "bits_per_byte": 1.5753132707263873,
      "learning_rate": 9.879282698288142e-05,
      "loss_nats": 1.0919239521026611,
      "step": 5440,
      "unclipped_grad_norm": 1.0844659805297852
    },
    {
      "bits_per_byte": 1.5681787424379945,
      "learning_rate": 9.830706566531197e-05,
      "loss_nats": 1.0869786739349365,
      "step": 5450,
      "unclipped_grad_norm": 1.1008447408676147
    },
    {
      "bits_per_byte": 1.5540281819076536,
      "learning_rate": 9.782244394597826e-05,
      "loss_nats": 1.0771702527999878,
      "step": 5460,
      "unclipped_grad_norm": 1.077485203742981
    },
    {
      "bits_per_byte": 1.5997471898997682,
      "learning_rate": 9.733897010572829e-05,
      "loss_nats": 1.1088602542877197,
      "step": 5470,
      "unclipped_grad_norm": 1.0731958150863647
    },
    {
      "bits_per_byte": 1.6110182449067132,
      "learning_rate": 9.685665240579612e-05,
      "loss_nats": 1.1166727542877197,
      "step": 5480,
      "unclipped_grad_norm": 1.098453164100647
    },
    {
      "bits_per_byte": 1.627262694211309,
      "learning_rate": 9.637549908766041e-05,
      "loss_nats": 1.1279325485229492,
      "step": 5490,
      "unclipped_grad_norm": 1.2330873012542725
    },
    {
      "bits_per_byte": 1.5907870657724668,
      "learning_rate": 9.589551837290389e-05,
      "loss_nats": 1.1026495695114136,
      "step": 5500,
      "unclipped_grad_norm": 1.1288633346557617
    },
    {
      "bits_per_byte": 1.677719824234694,
      "learning_rate": 9.541671846307257e-05,
      "loss_nats": 1.1629067659378052,
      "step": 5510,
      "unclipped_grad_norm": 1.1779037714004517
    },
    {
      "bits_per_byte": 1.6037174093949362,
      "learning_rate": 9.493910753953597e-05,
      "loss_nats": 1.1116122007369995,
      "step": 5520,
      "unclipped_grad_norm": 1.07362699508667
    },
    {
      "bits_per_byte": 1.5749782485225066,
      "learning_rate": 9.446269376334689e-05,
      "loss_nats": 1.0916917324066162,
      "step": 5530,
      "unclipped_grad_norm": 1.0827255249023438
    },
    {
      "bits_per_byte": 1.6475040203219564,
      "learning_rate": 9.39874852751024e-05,
      "loss_nats": 1.1419627666473389,
      "step": 5540,
      "unclipped_grad_norm": 1.2362996339797974
    },
    {
      "bits_per_byte": 1.5801301608117528,
      "learning_rate": 9.351349019480433e-05,
      "loss_nats": 1.0952627658843994,
      "step": 5550,
      "unclipped_grad_norm": 1.0816401243209839
    },
    {
      "bits_per_byte": 1.566390638816974,
      "learning_rate": 9.304071662172086e-05,
      "loss_nats": 1.085739254951477,
      "step": 5560,
      "unclipped_grad_norm": 1.1727906465530396
    },
    {
      "bits_per_byte": 1.545297310653952,
      "learning_rate": 9.256917263424787e-05,
      "loss_nats": 1.0711184740066528,
      "step": 5570,
      "unclipped_grad_norm": 1.096352458000183
    },
    {
      "bits_per_byte": 1.6197793851069664,
      "learning_rate": 9.209886628977114e-05,
      "loss_nats": 1.1227455139160156,
      "step": 5580,
      "unclipped_grad_norm": 1.098443865776062
    },
    {
      "bits_per_byte": 1.5488499562728253,
      "learning_rate": 9.162980562452843e-05,
      "loss_nats": 1.0735809803009033,
      "step": 5590,
      "unclipped_grad_norm": 1.106048822402954
    },
    {
      "bits_per_byte": 1.6060310919969936,
      "learning_rate": 9.116199865347238e-05,
      "loss_nats": 1.1132159233093262,
      "step": 5600,
      "unclipped_grad_norm": 1.1421784162521362
    },
    {
      "bits_per_byte": 1.6012358717256368,
      "learning_rate": 9.069545337013333e-05,
      "loss_nats": 1.1098921298980713,
      "step": 5610,
      "unclipped_grad_norm": 1.0477343797683716
    },
    {
      "bits_per_byte": 1.612595669780427,
      "learning_rate": 9.0230177746483e-05,
      "loss_nats": 1.1177661418914795,
      "step": 5620,
      "unclipped_grad_norm": 1.0961613655090332
    },
    {
      "bits_per_byte": 1.5887812321154537,
      "learning_rate": 8.976617973279797e-05,
      "loss_nats": 1.1012592315673828,
      "step": 5630,
      "unclipped_grad_norm": 1.0803630352020264
    },
    {
      "bits_per_byte": 1.5675833385007074,
      "learning_rate": 8.930346725752419e-05,
      "loss_nats": 1.0865659713745117,
      "step": 5640,
      "unclipped_grad_norm": 1.0926858186721802
    },
    {
      "bits_per_byte": 1.598304599424331,
      "learning_rate": 8.88420482271411e-05,
      "loss_nats": 1.1078603267669678,
      "step": 5650,
      "unclipped_grad_norm": 1.1493103504180908
    },
    {
      "bits_per_byte": 1.6562716958629424,
      "learning_rate": 8.838193052602691e-05,
      "loss_nats": 1.1480400562286377,
      "step": 5660,
      "unclipped_grad_norm": 1.1589452028274536
    },
    {
      "bits_per_byte": 1.5621947780838739,
      "learning_rate": 8.792312201632352e-05,
      "loss_nats": 1.0828309059143066,
      "step": 5670,
      "unclipped_grad_norm": 1.118128776550293
    },
    {
      "bits_per_byte": 1.5797031278896605,
      "learning_rate": 8.746563053780261e-05,
      "loss_nats": 1.0949667692184448,
      "step": 5680,
      "unclipped_grad_norm": 1.079695701599121
    },
    {
      "bits_per_byte": 1.650425317629512,
      "learning_rate": 8.700946390773114e-05,
      "loss_nats": 1.1439876556396484,
      "step": 5690,
      "unclipped_grad_norm": 1.1217325925827026
    },
    {
      "bits_per_byte": 1.5713879387030945,
      "learning_rate": 8.655462992073833e-05,
      "loss_nats": 1.089203119277954,
      "step": 5700,
      "unclipped_grad_norm": 1.0890491008758545
    },
    {
      "bits_per_byte": 1.5792038622542057,
      "learning_rate": 8.610113634868199e-05,
      "loss_nats": 1.094620704650879,
      "step": 5710,
      "unclipped_grad_norm": 1.0754131078720093
    },
    {
      "bits_per_byte": 1.6025209260928837,
      "learning_rate": 8.564899094051613e-05,
      "loss_nats": 1.1107828617095947,
      "step": 5720,
      "unclipped_grad_norm": 1.0796459913253784
    },
    {
      "bits_per_byte": 1.5944050647986676,
      "learning_rate": 8.519820142215816e-05,
      "loss_nats": 1.1051573753356934,
      "step": 5730,
      "unclipped_grad_norm": 1.0518229007720947
    },
    {
      "bits_per_byte": 1.599884088089855,
      "learning_rate": 8.474877549635728e-05,
      "loss_nats": 1.1089551448822021,
      "step": 5740,
      "unclipped_grad_norm": 1.2038764953613281
    },
    {
      "bits_per_byte": 1.5972181850188307,
      "learning_rate": 8.430072084256247e-05,
      "loss_nats": 1.1071072816848755,
      "step": 5750,
      "unclipped_grad_norm": 1.1661580801010132
    },
    {
      "bits_per_byte": 1.6206038699352028,
      "learning_rate": 8.385404511679161e-05,
      "loss_nats": 1.123317003250122,
      "step": 5760,
      "unclipped_grad_norm": 1.1536771059036255
    },
    {
      "bits_per_byte": 1.569340313261922,
      "learning_rate": 8.340875595150034e-05,
      "loss_nats": 1.0877838134765625,
      "step": 5770,
      "unclipped_grad_norm": 1.1234960556030273
    },
    {
      "bits_per_byte": 1.548520437513772,
      "learning_rate": 8.296486095545203e-05,
      "loss_nats": 1.073352575302124,
      "step": 5780,
      "unclipped_grad_norm": 1.0824602842330933
    },
    {
      "bits_per_byte": 1.575600653735979,
      "learning_rate": 8.252236771358729e-05,
      "loss_nats": 1.0921231508255005,
      "step": 5790,
      "unclipped_grad_norm": 1.0697625875473022
    },
    {
      "bits_per_byte": 1.5474534571478191,
      "learning_rate": 8.208128378689484e-05,
      "loss_nats": 1.072613000869751,
      "step": 5800,
      "unclipped_grad_norm": 1.121078372001648
    },
    {
      "bits_per_byte": 1.6281222635003212,
      "learning_rate": 8.16416167122819e-05,
      "loss_nats": 1.128528356552124,
      "step": 5810,
      "unclipped_grad_norm": 1.163888931274414
    },
    {
      "bits_per_byte": 1.5571622217543264,
      "learning_rate": 8.120337400244578e-05,
      "loss_nats": 1.0793426036834717,
      "step": 5820,
      "unclipped_grad_norm": 1.0646225214004517
    },
    {
      "bits_per_byte": 1.5715148618994312,
      "learning_rate": 8.076656314574521e-05,
      "loss_nats": 1.0892910957336426,
      "step": 5830,
      "unclipped_grad_norm": 1.1465011835098267
    },
    {
      "bits_per_byte": 1.5656843060698804,
      "learning_rate": 8.03311916060725e-05,
      "loss_nats": 1.085249662399292,
      "step": 5840,
      "unclipped_grad_norm": 1.0994694232940674
    },
    {
      "bits_per_byte": 1.529561070082656,
      "learning_rate": 7.989726682272606e-05,
      "loss_nats": 1.060210943222046,
      "step": 5850,
      "unclipped_grad_norm": 1.123828649520874
    },
    {
      "bits_per_byte": 1.5867750544931392,
      "learning_rate": 7.946479621028318e-05,
      "loss_nats": 1.099868655204773,
      "step": 5860,
      "unclipped_grad_norm": 1.1375004053115845
    },
    {
      "bits_per_byte": 1.5937161022993112,
      "learning_rate": 7.903378715847343e-05,
      "loss_nats": 1.104679822921753,
      "step": 5870,
      "unclipped_grad_norm": 1.0862858295440674
    },
    {
      "bits_per_byte": 1.6139739387444414,
      "learning_rate": 7.860424703205224e-05,
      "loss_nats": 1.1187214851379395,
      "step": 5880,
      "unclipped_grad_norm": 1.1279133558273315
    },
    {
      "bits_per_byte": 1.6231863614205586,
      "learning_rate": 7.817618317067534e-05,
      "loss_nats": 1.1251070499420166,
      "step": 5890,
      "unclipped_grad_norm": 1.1251599788665771
    },
    {
      "bits_per_byte": 1.6124319422968056,
      "learning_rate": 7.774960288877295e-05,
      "loss_nats": 1.1176526546478271,
      "step": 5900,
      "unclipped_grad_norm": 1.1988507509231567
    },
    {
      "bits_per_byte": 1.6244132856518139,
      "learning_rate": 7.732451347542522e-05,
      "loss_nats": 1.1259574890136719,
      "step": 5910,
      "unclipped_grad_norm": 1.1206265687942505
    },
    {
      "bits_per_byte": 1.5302899325570125,
      "learning_rate": 7.690092219423722e-05,
      "loss_nats": 1.060716152191162,
      "step": 5920,
      "unclipped_grad_norm": 1.1038360595703125
    },
    {
      "bits_per_byte": 1.6585005910181243,
      "learning_rate": 7.647883628321536e-05,
      "loss_nats": 1.1495850086212158,
      "step": 5930,
      "unclipped_grad_norm": 1.2028963565826416
    },
    {
      "bits_per_byte": 1.5036174872000014,
      "learning_rate": 7.605826295464316e-05,
      "loss_nats": 1.0422282218933105,
      "step": 5940,
      "unclipped_grad_norm": 1.0828943252563477
    },
    {
      "bits_per_byte": 1.564500033536039,
      "learning_rate": 7.563920939495851e-05,
      "loss_nats": 1.0844287872314453,
      "step": 5950,
      "unclipped_grad_norm": 1.1633635759353638
    },
    {
      "bits_per_byte": 1.6026038217305996,
      "learning_rate": 7.522168276463047e-05,
      "loss_nats": 1.1108403205871582,
      "step": 5960,
      "unclipped_grad_norm": 1.1734113693237305
    },
    {
      "bits_per_byte": 1.582973034030515,
      "learning_rate": 7.480569019803719e-05,
      "loss_nats": 1.0972332954406738,
      "step": 5970,
      "unclipped_grad_norm": 1.255017638206482
    },
    {
      "bits_per_byte": 1.6250308753510623,
      "learning_rate": 7.439123880334388e-05,
      "loss_nats": 1.1263855695724487,
      "step": 5980,
      "unclipped_grad_norm": 1.1516860723495483
    },
    {
      "bits_per_byte": 1.5794843659577629,
      "learning_rate": 7.39783356623813e-05,
      "loss_nats": 1.0948151350021362,
      "step": 5990,
      "unclipped_grad_norm": 1.1278661489486694
    },
    {
      "bits_per_byte": 1.5522483334538744,
      "learning_rate": 7.356698783052497e-05,
      "loss_nats": 1.0759365558624268,
      "step": 6000,
      "unclipped_grad_norm": 1.067203402519226
    },
    {
      "bits_per_byte": 1.577093635128119,
      "learning_rate": 7.315720233657431e-05,
      "loss_nats": 1.0931580066680908,
      "step": 6010,
      "unclipped_grad_norm": 1.1185256242752075
    },
    {
      "bits_per_byte": 1.534402725669746,
      "learning_rate": 7.274898618263286e-05,
      "loss_nats": 1.0635669231414795,
      "step": 6020,
      "unclipped_grad_norm": 1.1662800312042236
    },
    {
      "bits_per_byte": 1.5661140907143865,
      "learning_rate": 7.234234634398831e-05,
      "loss_nats": 1.0855475664138794,
      "step": 6030,
      "unclipped_grad_norm": 1.1327592134475708
    },
    {
      "bits_per_byte": 1.6326177180112242,
      "learning_rate": 7.193728976899362e-05,
      "loss_nats": 1.131644368171692,
      "step": 6040,
      "unclipped_grad_norm": 1.1918420791625977
    },
    {
      "bits_per_byte": 1.5457630396724886,
      "learning_rate": 7.153382337894797e-05,
      "loss_nats": 1.0714412927627563,
      "step": 6050,
      "unclipped_grad_norm": 1.147464394569397
    },
    {
      "bits_per_byte": 1.5419047808830324,
      "learning_rate": 7.113195406797887e-05,
      "loss_nats": 1.0687669515609741,
      "step": 6060,
      "unclipped_grad_norm": 1.1692534685134888
    },
    {
      "bits_per_byte": 1.6009338701907216,
      "learning_rate": 7.073168870292397e-05,
      "loss_nats": 1.1096827983856201,
      "step": 6070,
      "unclipped_grad_norm": 1.1858196258544922
    },
    {
      "bits_per_byte": 1.5324048032146727,
      "learning_rate": 7.033303412321404e-05,
      "loss_nats": 1.062182068824768,
      "step": 6080,
      "unclipped_grad_norm": 1.1669044494628906
    },
    {
      "bits_per_byte": 1.5495642002218584,
      "learning_rate": 6.993599714075586e-05,
      "loss_nats": 1.0740760564804077,
      "step": 6090,
      "unclipped_grad_norm": 1.0681768655776978
    },
    {
      "bits_per_byte": 1.5432428059067451,
      "learning_rate": 6.954058453981609e-05,
      "loss_nats": 1.0696943998336792,
      "step": 6100,
      "unclipped_grad_norm": 1.086612582206726
    },
    {
      "bits_per_byte": 1.6119146184830102,
      "learning_rate": 6.914680307690504e-05,
      "loss_nats": 1.1172940731048584,
      "step": 6110,
      "unclipped_grad_norm": 1.1307899951934814
    },
    {
      "bits_per_byte": 1.6174417969164392,
      "learning_rate": 6.875465948066139e-05,
      "loss_nats": 1.1211252212524414,
      "step": 6120,
      "unclipped_grad_norm": 1.2087455987930298
    },
    {
      "bits_per_byte": 1.5350297744147916,
      "learning_rate": 6.836416045173733e-05,
      "loss_nats": 1.0640015602111816,
      "step": 6130,
      "unclipped_grad_norm": 1.1170084476470947
    },
    {
      "bits_per_byte": 1.5441861307467275,
      "learning_rate": 6.797531266268377e-05,
      "loss_nats": 1.0703482627868652,
      "step": 6140,
      "unclipped_grad_norm": 1.191177487373352
    },
    {
      "bits_per_byte": 1.6015168913771465,
      "learning_rate": 6.758812275783663e-05,
      "loss_nats": 1.1100869178771973,
      "step": 6150,
      "unclipped_grad_norm": 1.1631611585617065
    },
    {
      "bits_per_byte": 1.4840940166740553,
      "learning_rate": 6.720259735320298e-05,
      "loss_nats": 1.0286955833435059,
      "step": 6160,
      "unclipped_grad_norm": 1.0670503377914429
    },
    {
      "bits_per_byte": 1.5765931656141083,
      "learning_rate": 6.681874303634842e-05,
      "loss_nats": 1.092811107635498,
      "step": 6170,
      "unclipped_grad_norm": 1.12460458278656
    },
    {
      "bits_per_byte": 1.531391309453138,
      "learning_rate": 6.643656636628402e-05,
      "loss_nats": 1.0614795684814453,
      "step": 6180,
      "unclipped_grad_norm": 1.1588267087936401
    },
    {
      "bits_per_byte": 1.5676526475090051,
      "learning_rate": 6.605607387335466e-05,
      "loss_nats": 1.0866140127182007,
      "step": 6190,
      "unclipped_grad_norm": 1.2090874910354614
    },
    {
      "bits_per_byte": 1.5723472579296074,
      "learning_rate": 6.567727205912724e-05,
      "loss_nats": 1.0898680686950684,
      "step": 6200,
      "unclipped_grad_norm": 1.1369835138320923
    },
    {
      "bits_per_byte": 1.5728810920778855,
      "learning_rate": 6.530016739627959e-05,
      "loss_nats": 1.090238094329834,
      "step": 6210,
      "unclipped_grad_norm": 1.2529284954071045
    },
    {
      "bits_per_byte": 1.5681563846933824,
      "learning_rate": 6.492476632848993e-05,
      "loss_nats": 1.086963176727295,
      "step": 6220,
      "unclipped_grad_norm": 1.0888316631317139
    },
    {
      "bits_per_byte": 1.5207944264375752,
      "learning_rate": 6.455107527032678e-05,
      "loss_nats": 1.0541343688964844,
      "step": 6230,
      "unclipped_grad_norm": 1.15282142162323
    },
    {
      "bits_per_byte": 1.5617125387308546,
      "learning_rate": 6.417910060713926e-05,
      "loss_nats": 1.0824966430664062,
      "step": 6240,
      "unclipped_grad_norm": 1.227396845817566
    },
    {
      "bits_per_byte": 1.595514008931431,
      "learning_rate": 6.380884869494799e-05,
      "loss_nats": 1.1059260368347168,
      "step": 6250,
      "unclipped_grad_norm": 1.1354495286941528
    },
    {
      "bits_per_byte": 1.4806825688115406,
      "learning_rate": 6.34403258603367e-05,
      "loss_nats": 1.0263309478759766,
      "step": 6260,
      "unclipped_grad_norm": 1.1806098222732544
    },
    {
      "bits_per_byte": 1.561176984756068,
      "learning_rate": 6.30735384003437e-05,
      "loss_nats": 1.0821254253387451,
      "step": 6270,
      "unclipped_grad_norm": 1.1072839498519897
    },
    {
      "bits_per_byte": 1.557884892853252,
      "learning_rate": 6.270849258235484e-05,
      "loss_nats": 1.079843521118164,
      "step": 6280,
      "unclipped_grad_norm": 1.16572904586792
    },
    {
      "bits_per_byte": 1.5084749651909706,
      "learning_rate": 6.234519464399581e-05,
      "loss_nats": 1.0455951690673828,
      "step": 6290,
      "unclipped_grad_norm": 1.1536295413970947
    },
    {
      "bits_per_byte": 1.5632667459467018,
      "learning_rate": 6.198365079302614e-05,
      "loss_nats": 1.0835739374160767,
      "step": 6300,
      "unclipped_grad_norm": 1.1973258256912231
    },
    {
      "bits_per_byte": 1.5420322200273218,
      "learning_rate": 6.16238672072326e-05,
      "loss_nats": 1.0688552856445312,
      "step": 6310,
      "unclipped_grad_norm": 1.146715760231018
    },
    {
      "bits_per_byte": 1.4920311879940247,
      "learning_rate": 6.126585003432406e-05,
      "loss_nats": 1.034197211265564,
      "step": 6320,
      "unclipped_grad_norm": 1.1094690561294556
    },
    {
      "bits_per_byte": 1.5741831727275677,
      "learning_rate": 6.090960539182614e-05,
      "loss_nats": 1.091140627861023,
      "step": 6330,
      "unclipped_grad_norm": 1.2368261814117432
    },
    {
      "bits_per_byte": 1.5350222071781536,
      "learning_rate": 6.0555139366976935e-05,
      "loss_nats": 1.0639963150024414,
      "step": 6340,
      "unclipped_grad_norm": 1.105275273323059
    },
    {
      "bits_per_byte": 1.6462146663884378,
      "learning_rate": 6.020245801662274e-05,
      "loss_nats": 1.1410690546035767,
      "step": 6350,
      "unclipped_grad_norm": 1.3086775541305542
    },
    {
      "bits_per_byte": 1.52795922367252,
      "learning_rate": 5.985156736711483e-05,
      "loss_nats": 1.05910062789917,
      "step": 6360,
      "unclipped_grad_norm": 1.1954522132873535
    },
    {
      "bits_per_byte": 1.5079418189732958,
      "learning_rate": 5.950247341420618e-05,
      "loss_nats": 1.0452256202697754,
      "step": 6370,
      "unclipped_grad_norm": 1.1107361316680908
    },
    {
      "bits_per_byte": 1.5318044117805105,
      "learning_rate": 5.915518212294935e-05,
      "loss_nats": 1.0617659091949463,
      "step": 6380,
      "unclipped_grad_norm": 1.0837970972061157
    },
    {
      "bits_per_byte": 1.5828646849604715,
      "learning_rate": 5.880969942759425e-05,
      "loss_nats": 1.0971581935882568,
      "step": 6390,
      "unclipped_grad_norm": 1.1240276098251343
    },
    {
      "bits_per_byte": 1.5799186221511916,
      "learning_rate": 5.846603123148688e-05,
      "loss_nats": 1.095116138458252,
      "step": 6400,
      "unclipped_grad_norm": 1.1799321174621582
    },
    {
      "bits_per_byte": 1.552409309215082,
      "learning_rate": 5.812418340696858e-05,
      "loss_nats": 1.0760481357574463,
      "step": 6410,
      "unclipped_grad_norm": 1.1518590450286865
    },
    {
      "bits_per_byte": 1.5442817531006072,
      "learning_rate": 5.778416179527541e-05,
      "loss_nats": 1.0704145431518555,
      "step": 6420,
      "unclipped_grad_norm": 1.1955360174179077
    },
    {
      "bits_per_byte": 1.5805626971786728,
      "learning_rate": 5.7445972206438635e-05,
      "loss_nats": 1.0955625772476196,
      "step": 6430,
      "unclipped_grad_norm": 1.156053066253662
    },
    {
      "bits_per_byte": 1.5364046037257888,
      "learning_rate": 5.710962041918516e-05,
      "loss_nats": 1.0649545192718506,
      "step": 6440,
      "unclipped_grad_norm": 1.1582752466201782
    },
    {
      "bits_per_byte": 1.5944274225432797,
      "learning_rate": 5.677511218083908e-05,
      "loss_nats": 1.105172872543335,
      "step": 6450,
      "unclipped_grad_norm": 1.2264126539230347
    },
    {
      "bits_per_byte": 1.645741886081216,
      "learning_rate": 5.644245320722316e-05,
      "loss_nats": 1.1407413482666016,
      "step": 6460,
      "unclipped_grad_norm": 1.2444238662719727
    },
    {
      "bits_per_byte": 1.6345733327341851,
      "learning_rate": 5.61116491825615e-05,
      "loss_nats": 1.1329998970031738,
      "step": 6470,
      "unclipped_grad_norm": 1.2292388677597046
    },
    {
      "bits_per_byte": 1.5111716531564994,
      "learning_rate": 5.5782705759382104e-05,
      "loss_nats": 1.047464370727539,
      "step": 6480,
      "unclipped_grad_norm": 1.2222721576690674
    },
    {
      "bits_per_byte": 1.6197842006211907,
      "learning_rate": 5.5455628558420546e-05,
      "loss_nats": 1.122748851776123,
      "step": 6490,
      "unclipped_grad_norm": 1.2203607559204102
    },
    {
      "bits_per_byte": 1.5564085937782457,
      "learning_rate": 5.513042316852368e-05,
      "loss_nats": 1.0788202285766602,
      "step": 6500,
      "unclipped_grad_norm": 1.1637120246887207
    },
    {
      "bits_per_byte": 1.5288421826020495,
      "learning_rate": 5.480709514655443e-05,
      "loss_nats": 1.0597126483917236,
      "step": 6510,
      "unclipped_grad_norm": 1.1861555576324463
    },
    {
      "bits_per_byte": 1.5068566084463515,
      "learning_rate": 5.448565001729653e-05,
      "loss_nats": 1.04447340965271,
      "step": 6520,
      "unclipped_grad_norm": 1.1399540901184082
    },
    {
      "bits_per_byte": 1.5271775625243484,
      "learning_rate": 5.41660932733604e-05,
      "loss_nats": 1.0585588216781616,
      "step": 6530,
      "unclipped_grad_norm": 1.200762152671814
    },
    {
      "bits_per_byte": 1.5654438743239742,
      "learning_rate": 5.384843037508902e-05,
      "loss_nats": 1.0850830078125,
      "step": 6540,
      "unclipped_grad_norm": 1.1505160331726074
    },
    {
      "bits_per_byte": 1.505103417303456,
      "learning_rate": 5.353266675046498e-05,
      "loss_nats": 1.0432581901550293,
      "step": 6550,
      "unclipped_grad_norm": 1.1259007453918457
    },
    {
      "bits_per_byte": 1.5822820077393482,
      "learning_rate": 5.321880779501729e-05,
      "loss_nats": 1.0967543125152588,
      "step": 6560,
      "unclipped_grad_norm": 1.1489042043685913
    },
    {
      "bits_per_byte": 1.5307443107205922,
      "learning_rate": 5.290685887172962e-05,
      "loss_nats": 1.0610311031341553,
      "step": 6570,
      "unclipped_grad_norm": 1.2110720872879028
    },
    {
      "bits_per_byte": 1.5820749406277095,
      "learning_rate": 5.2596825310948295e-05,
      "loss_nats": 1.0966107845306396,
      "step": 6580,
      "unclipped_grad_norm": 1.2832127809524536
    },
    {
      "bits_per_byte": 1.555234640203456,
      "learning_rate": 5.228871241029154e-05,
      "loss_nats": 1.0780065059661865,
      "step": 6590,
      "unclipped_grad_norm": 1.1875557899475098
    },
    {
      "bits_per_byte": 1.4893780116391642,
      "learning_rate": 5.198252543455865e-05,
      "loss_nats": 1.032358169555664,
      "step": 6600,
      "unclipped_grad_norm": 1.1434928178787231
    },
    {
      "bits_per_byte": 1.4980550523231455,
      "learning_rate": 5.167826961564027e-05,
      "loss_nats": 1.0383726358413696,
      "step": 6610,
      "unclipped_grad_norm": 1.1379001140594482
    },
    {
      "bits_per_byte": 1.4935792038344409,
      "learning_rate": 5.137595015242884e-05,
      "loss_nats": 1.0352702140808105,
      "step": 6620,
      "unclipped_grad_norm": 1.1951154470443726
    },
    {
      "bits_per_byte": 1.5329076804857955,
      "learning_rate": 5.1075572210729925e-05,
      "loss_nats": 1.0625306367874146,
      "step": 6630,
      "unclipped_grad_norm": 1.144928216934204
    },
    {
      "bits_per_byte": 1.5874349519244997,
      "learning_rate": 5.07771409231737e-05,
      "loss_nats": 1.1003260612487793,
      "step": 6640,
      "unclipped_grad_norm": 1.2067164182662964
    },
    {
      "bits_per_byte": 1.5760562357781147,
      "learning_rate": 5.048066138912756e-05,
      "loss_nats": 1.0924389362335205,
      "step": 6650,
      "unclipped_grad_norm": 1.1507362127304077
    },
    {
      "bits_per_byte": 1.5807757836830918,
      "learning_rate": 5.018613867460863e-05,
      "loss_nats": 1.095710277557373,
      "step": 6660,
      "unclipped_grad_norm": 1.134096622467041
    },
    {
      "bits_per_byte": 1.5557022610311522,
      "learning_rate": 4.989357781219755e-05,
      "loss_nats": 1.078330636024475,
      "step": 6670,
      "unclipped_grad_norm": 1.2659887075424194
    },
    {
      "bits_per_byte": 1.5265064862206816,
      "learning_rate": 4.960298380095222e-05,
      "loss_nats": 1.0580936670303345,
      "step": 6680,
      "unclipped_grad_norm": 1.196817398071289
    },
    {
      "bits_per_byte": 1.558342366704547,
      "learning_rate": 4.931436160632256e-05,
      "loss_nats": 1.0801606178283691,
      "step": 6690,
      "unclipped_grad_norm": 1.191389799118042
    },
    {
      "bits_per_byte": 1.6202516494662358,
      "learning_rate": 4.9027716160065485e-05,
      "loss_nats": 1.123072862625122,
      "step": 6700,
      "unclipped_grad_norm": 1.2432118654251099
    },
    {
      "bits_per_byte": 1.5747254340257382,
      "learning_rate": 4.874305236016084e-05,
      "loss_nats": 1.0915164947509766,
      "step": 6710,
      "unclipped_grad_norm": 1.1514227390289307
    },
    {
      "bits_per_byte": 1.5896156919374402,
      "learning_rate": 4.846037507072752e-05,
      "loss_nats": 1.1018376350402832,
      "step": 6720,
      "unclipped_grad_norm": 1.1839321851730347
    },
    {
      "bits_per_byte": 1.5521088555240246,
      "learning_rate": 4.817968912194049e-05,
      "loss_nats": 1.075839877128601,
      "step": 6730,
      "unclipped_grad_norm": 1.2353639602661133
    },
    {
      "bits_per_byte": 1.4976266435398462,
      "learning_rate": 4.790099930994813e-05,
      "loss_nats": 1.0380756855010986,
      "step": 6740,
      "unclipped_grad_norm": 1.1758193969726562
    },
    {
      "bits_per_byte": 1.5596673210467942,
      "learning_rate": 4.762431039679047e-05,
      "loss_nats": 1.0810790061950684,
      "step": 6750,
      "unclipped_grad_norm": 1.1156134605407715
    },
    {
      "bits_per_byte": 1.536689063030316,
      "learning_rate": 4.734962711031755e-05,
      "loss_nats": 1.0651516914367676,
      "step": 6760,
      "unclipped_grad_norm": 1.2122220993041992
    },
    {
      "bits_per_byte": 1.443431470599958,
      "learning_rate": 4.707695414410892e-05,
      "loss_nats": 1.0005104541778564,
      "step": 6770,
      "unclipped_grad_norm": 1.1533452272415161
    },
    {
      "bits_per_byte": 1.5402759331967106,
      "learning_rate": 4.680629615739322e-05,
      "loss_nats": 1.0676379203796387,
      "step": 6780,
      "unclipped_grad_norm": 1.1691333055496216
    },
    {
      "bits_per_byte": 1.523325667092974,
      "learning_rate": 4.653765777496863e-05,
      "loss_nats": 1.0558888912200928,
      "step": 6790,
      "unclipped_grad_norm": 1.2405608892440796
    },
    {
      "bits_per_byte": 1.57861671348416,
      "learning_rate": 4.6271043587123986e-05,
      "loss_nats": 1.0942137241363525,
      "step": 6800,
      "unclipped_grad_norm": 1.2281029224395752
    },
    {
      "bits_per_byte": 1.5738859867068768,
      "learning_rate": 4.600645814956006e-05,
      "loss_nats": 1.0909346342086792,
      "step": 6810,
      "unclipped_grad_norm": 1.2371455430984497
    },
    {
      "bits_per_byte": 1.582858837550342,
      "learning_rate": 4.574390598331204e-05,
      "loss_nats": 1.097154140472412,
      "step": 6820,
      "unclipped_grad_norm": 1.3370100259780884
    },
    {
      "bits_per_byte": 1.5273318309621724,
      "learning_rate": 4.548339157467199e-05,
      "loss_nats": 1.0586657524108887,
      "step": 6830,
      "unclipped_grad_norm": 1.196358561515808
    },
    {
      "bits_per_byte": 1.4954078953610652,
      "learning_rate": 4.52249193751124e-05,
      "loss_nats": 1.036537766456604,
      "step": 6840,
      "unclipped_grad_norm": 1.1750952005386353
    },
    {
      "bits_per_byte": 1.5476113372213112,
      "learning_rate": 4.4968493801209985e-05,
      "loss_nats": 1.0727224349975586,
      "step": 6850,
      "unclipped_grad_norm": 1.1924985647201538
    },
    {
      "bits_per_byte": 1.526412067745358,
      "learning_rate": 4.471411923457034e-05,
      "loss_nats": 1.058028221130371,
      "step": 6860,
      "unclipped_grad_norm": 1.1877104043960571
    },
    {
      "bits_per_byte": 1.4832949852781467,
      "learning_rate": 4.446180002175291e-05,
      "loss_nats": 1.028141736984253,
      "step": 6870,
      "unclipped_grad_norm": 1.175843596458435
    },
    {
      "bits_per_byte": 1.4908706490660026,
      "learning_rate": 4.421154047419693e-05,
      "loss_nats": 1.0333927869796753,
      "step": 6880,
      "unclipped_grad_norm": 1.1246671676635742
    },
    {
      "bits_per_byte": 1.5010700801754213,
      "learning_rate": 4.396334486814753e-05,
      "loss_nats": 1.0404624938964844,
      "step": 6890,
      "unclipped_grad_norm": 1.1924952268600464
    },
    {
      "bits_per_byte": 1.6045640799851337,
      "learning_rate": 4.371721744458285e-05,
      "loss_nats": 1.112199068069458,
      "step": 6900,
      "unclipped_grad_norm": 1.2239227294921875
    },
    {
      "bits_per_byte": 1.5736668808096776,
      "learning_rate": 4.347316240914145e-05,
      "loss_nats": 1.0907827615737915,
      "step": 6910,
      "unclipped_grad_norm": 1.2664575576782227
    },
    {
      "bits_per_byte": 1.5080825007817016,
      "learning_rate": 4.3231183932050546e-05,
      "loss_nats": 1.045323133468628,
      "step": 6920,
      "unclipped_grad_norm": 1.1531251668930054
    },
    {
      "bits_per_byte": 1.5085007625886,
      "learning_rate": 4.299128614805464e-05,
      "loss_nats": 1.0456130504608154,
      "step": 6930,
      "unclipped_grad_norm": 1.2472591400146484
    },
    {
      "bits_per_byte": 1.484484417291514,
      "learning_rate": 4.2753473156344965e-05,
      "loss_nats": 1.0289661884307861,
      "step": 6940,
      "unclipped_grad_norm": 1.187600016593933
    },
    {
      "bits_per_byte": 1.4929882714460763,
      "learning_rate": 4.251774902048942e-05,
      "loss_nats": 1.034860610961914,
      "step": 6950,
      "unclipped_grad_norm": 1.228901743888855
    },
    {
      "bits_per_byte": 1.5665772399931601,
      "learning_rate": 4.228411776836306e-05,
      "loss_nats": 1.0858685970306396,
      "step": 6960,
      "unclipped_grad_norm": 1.205407738685608
    },
    {
      "bits_per_byte": 1.5512240047853354,
      "learning_rate": 4.2052583392079427e-05,
      "loss_nats": 1.0752265453338623,
      "step": 6970,
      "unclipped_grad_norm": 1.2078452110290527
    },
    {
      "bits_per_byte": 1.5256931802647515,
      "learning_rate": 4.182314984792214e-05,
      "loss_nats": 1.0575299263000488,
      "step": 6980,
      "unclipped_grad_norm": 1.216407060623169
    },
    {
      "bits_per_byte": 1.5495781308165784,
      "learning_rate": 4.159582105627748e-05,
      "loss_nats": 1.0740857124328613,
      "step": 6990,
      "unclipped_grad_norm": 1.2360340356826782
    },
    {
      "bits_per_byte": 1.579722561929208,
      "learning_rate": 4.137060090156724e-05,
      "loss_nats": 1.094980239868164,
      "step": 7000,
      "unclipped_grad_norm": 1.2281304597854614
    },
    {
      "bits_per_byte": 1.5775139607268278,
      "learning_rate": 4.114749323218249e-05,
      "loss_nats": 1.093449354171753,
      "step": 7010,
      "unclipped_grad_norm": 1.238048791885376
    },
    {
      "bits_per_byte": 1.5315163408403152,
      "learning_rate": 4.0926501860417706e-05,
      "loss_nats": 1.0615662336349487,
      "step": 7020,
      "unclipped_grad_norm": 1.2856208086013794
    },
    {
      "bits_per_byte": 1.5716388613907033,
      "learning_rate": 4.070763056240571e-05,
      "loss_nats": 1.0893770456314087,
      "step": 7030,
      "unclipped_grad_norm": 1.228158950805664
    },
    {
      "bits_per_byte": 1.5737668027298288,
      "learning_rate": 4.049088307805306e-05,
      "loss_nats": 1.0908520221710205,
      "step": 7040,
      "unclipped_grad_norm": 1.227662444114685
    },
    {
      "bits_per_byte": 1.5448513596402649,
      "learning_rate": 4.027626311097629e-05,
      "loss_nats": 1.0708093643188477,
      "step": 7050,
      "unclipped_grad_norm": 1.2028948068618774
    },
    {
      "bits_per_byte": 1.5624737339435735,
      "learning_rate": 4.006377432843846e-05,
      "loss_nats": 1.083024263381958,
      "step": 7060,
      "unclipped_grad_norm": 1.259883999824524
    },
    {
      "bits_per_byte": 1.539107139101447,
      "learning_rate": 3.985342036128656e-05,
      "loss_nats": 1.0668277740478516,
      "step": 7070,
      "unclipped_grad_norm": 1.2382662296295166
    },
    {
      "bits_per_byte": 1.5001651074665814,
      "learning_rate": 3.964520480388956e-05,
      "loss_nats": 1.0398352146148682,
      "step": 7080,
      "unclipped_grad_norm": 1.1878225803375244
    },
    {
      "bits_per_byte": 1.5461551601164558,
      "learning_rate": 3.943913121407678e-05,
      "loss_nats": 1.0717130899429321,
      "step": 7090,
      "unclipped_grad_norm": 1.3277226686477661
    },
    {
      "bits_per_byte": 1.525824231044709,
      "learning_rate": 3.923520311307735e-05,
      "loss_nats": 1.0576207637786865,
      "step": 7100,
      "unclipped_grad_norm": 1.2164647579193115
    },
    {
      "bits_per_byte": 1.5404795606553323,
      "learning_rate": 3.9033423985459816e-05,
      "loss_nats": 1.0677790641784668,
      "step": 7110,
      "unclipped_grad_norm": 1.1725497245788574
    },
    {
      "bits_per_byte": 1.5736082347257334,
      "learning_rate": 3.88337972790728e-05,
      "loss_nats": 1.0907421112060547,
      "step": 7120,
      "unclipped_grad_norm": 1.218058705329895
    },
    {
      "bits_per_byte": 1.5565896915096042,
      "learning_rate": 3.8636326404985845e-05,
      "loss_nats": 1.0789457559585571,
      "step": 7130,
      "unclipped_grad_norm": 1.3638907670974731
    },
    {
      "bits_per_byte": 1.5273908210114182,
      "learning_rate": 3.844101473743143e-05,
      "loss_nats": 1.0587066411972046,
      "step": 7140,
      "unclipped_grad_norm": 1.2031904458999634
    },
    {
      "bits_per_byte": 1.5130964829849558,
      "learning_rate": 3.824786561374701e-05,
      "loss_nats": 1.0487985610961914,
      "step": 7150,
      "unclipped_grad_norm": 1.1664750576019287
    },
    {
      "bits_per_byte": 1.5860308855628558,
      "learning_rate": 3.805688233431824e-05,
      "loss_nats": 1.0993528366088867,
      "step": 7160,
      "unclipped_grad_norm": 1.2491129636764526
    },
    {
      "bits_per_byte": 1.5206325907631133,
      "learning_rate": 3.786806816252236e-05,
      "loss_nats": 1.054022192955017,
      "step": 7170,
      "unclipped_grad_norm": 1.20036780834198
    },
    {
      "bits_per_byte": 1.5424085180674096,
      "learning_rate": 3.768142632467269e-05,
      "loss_nats": 1.0691161155700684,
      "step": 7180,
      "unclipped_grad_norm": 1.2460989952087402
    },
    {
      "bits_per_byte": 1.5104255924170564,
      "learning_rate": 3.7496960009963236e-05,
      "loss_nats": 1.0469472408294678,
      "step": 7190,
      "unclipped_grad_norm": 1.2477872371673584
    },
    {
      "bits_per_byte": 1.550646659026389,
      "learning_rate": 3.7314672370414325e-05,
      "loss_nats": 1.0748263597488403,
      "step": 7200,
      "unclipped_grad_norm": 1.26192045211792
    },
    {
      "bits_per_byte": 1.5958559104413463,
      "learning_rate": 3.713456652081882e-05,
      "loss_nats": 1.1061630249023438,
      "step": 7210,
      "unclipped_grad_norm": 1.2875406742095947
    },
    {
      "bits_per_byte": 1.5889428958072649,
      "learning_rate": 3.695664553868873e-05,
      "loss_nats": 1.1013712882995605,
      "step": 7220,
      "unclipped_grad_norm": 1.2469474077224731
    },
    {
      "bits_per_byte": 1.512987102019007,
      "learning_rate": 3.678091246420277e-05,
      "loss_nats": 1.048722743988037,
      "step": 7230,
      "unclipped_grad_norm": 1.2010514736175537
    },
    {
      "bits_per_byte": 1.5378745394427134,
      "learning_rate": 3.660737030015427e-05,
      "loss_nats": 1.0659734010696411,
      "step": 7240,
      "unclipped_grad_norm": 1.201562762260437
    },
    {
      "bits_per_byte": 1.5587365509403246,
      "learning_rate": 3.6436022011900056e-05,
      "loss_nats": 1.0804338455200195,
      "step": 7250,
      "unclipped_grad_norm": 1.305864691734314
    },
    {
      "bits_per_byte": 1.457526824699372,
      "learning_rate": 3.626687052730954e-05,
      "loss_nats": 1.0102806091308594,
      "step": 7260,
      "unclipped_grad_norm": 1.1730221509933472
    },
    {
      "bits_per_byte": 1.4923226985872373,
      "learning_rate": 3.6099918736714955e-05,
      "loss_nats": 1.0343992710113525,
      "step": 7270,
      "unclipped_grad_norm": 1.222469687461853
    },
    {
      "bits_per_byte": 1.56038208094378,
      "learning_rate": 3.5935169492861716e-05,
      "loss_nats": 1.0815744400024414,
      "step": 7280,
      "unclipped_grad_norm": 1.2304381132125854
    },
    {
      "bits_per_byte": 1.4970130094415677,
      "learning_rate": 3.5772625610859886e-05,
      "loss_nats": 1.0376503467559814,
      "step": 7290,
      "unclipped_grad_norm": 1.1906161308288574
    },
    {
      "bits_per_byte": 1.5193676583660174,
      "learning_rate": 3.5612289868135887e-05,
      "loss_nats": 1.053145408630371,
      "step": 7300,
      "unclipped_grad_norm": 1.2887614965438843
    },
    {
      "bits_per_byte": 1.5227811700203424,
      "learning_rate": 3.545416500438522e-05,
      "loss_nats": 1.055511474609375,
      "step": 7310,
      "unclipped_grad_norm": 1.1584672927856445
    },
    {
      "bits_per_byte": 1.4581731355013146,
      "learning_rate": 3.52982537215255e-05,
      "loss_nats": 1.0107285976409912,
      "step": 7320,
      "unclipped_grad_norm": 1.2041478157043457
    },
    {
      "bits_per_byte": 1.4976147767369368,
      "learning_rate": 3.5144558683650396e-05,
      "loss_nats": 1.0380674600601196,
      "step": 7330,
      "unclipped_grad_norm": 1.2121145725250244
    },
    {
      "bits_per_byte": 1.456979231939025,
      "learning_rate": 3.499308251698404e-05,
      "loss_nats": 1.0099010467529297,
      "step": 7340,
      "unclipped_grad_norm": 1.236539363861084
    },
    {
      "bits_per_byte": 1.5469846324415673,
      "learning_rate": 3.484382780983616e-05,
      "loss_nats": 1.0722880363464355,
      "step": 7350,
      "unclipped_grad_norm": 1.240252137184143
    },
    {
      "bits_per_byte": 1.4978240796230369,
      "learning_rate": 3.469679711255795e-05,
      "loss_nats": 1.038212537765503,
      "step": 7360,
      "unclipped_grad_norm": 1.2412164211273193
    },
    {
      "bits_per_byte": 1.5397654886889498,
      "learning_rate": 3.4551992937498284e-05,
      "loss_nats": 1.067284107208252,
      "step": 7370,
      "unclipped_grad_norm": 1.2414754629135132
    },
    {
      "bits_per_byte": 1.4664532402271044,
      "learning_rate": 3.440941775896104e-05,
      "loss_nats": 1.0164679288864136,
      "step": 7380,
      "unclipped_grad_norm": 1.196345567703247
    },
    {
      "bits_per_byte": 1.5561741814251195,
      "learning_rate": 3.4269074013162596e-05,
      "loss_nats": 1.0786577463150024,
      "step": 7390,
      "unclipped_grad_norm": 1.2357971668243408
    },
    {
      "bits_per_byte": 1.4878109057245024,
      "learning_rate": 3.41309640981904e-05,
      "loss_nats": 1.0312719345092773,
      "step": 7400,
      "unclipped_grad_norm": 1.2395251989364624
    },
    {
      "bits_per_byte": 1.4828184213526059,
      "learning_rate": 3.3995090373961795e-05,
      "loss_nats": 1.0278114080429077,
      "step": 7410,
      "unclipped_grad_norm": 1.1855907440185547
    },
    {
      "bits_per_byte": 1.5432890692398271,
      "learning_rate": 3.3861455162183876e-05,
      "loss_nats": 1.0697264671325684,
      "step": 7420,
      "unclipped_grad_norm": 1.2337415218353271
    },
    {
      "bits_per_byte": 1.5564147851536767,
      "learning_rate": 3.3730060746313664e-05,
      "loss_nats": 1.078824520111084,
      "step": 7430,
      "unclipped_grad_norm": 1.271388053894043
    },
    {
      "bits_per_byte": 1.5798298791033463,
      "learning_rate": 3.3600909371519225e-05,
      "loss_nats": 1.0950546264648438,
      "step": 7440,
      "unclipped_grad_norm": 1.3756663799285889
    },
    {
      "bits_per_byte": 1.5198117175705452,
      "learning_rate": 3.3474003244641194e-05,
      "loss_nats": 1.0534532070159912,
      "step": 7450,
      "unclipped_grad_norm": 1.2638351917266846
    },
    {
      "bits_per_byte": 1.5153886677556554,
      "learning_rate": 3.334934453415514e-05,
      "loss_nats": 1.0503873825073242,
      "step": 7460,
      "unclipped_grad_norm": 1.2390333414077759
    },
    {
      "bits_per_byte": 1.500396252149341,
      "learning_rate": 3.3226935370134486e-05,
      "loss_nats": 1.0399954319000244,
      "step": 7470,
      "unclipped_grad_norm": 1.1914620399475098
    },
    {
      "bits_per_byte": 1.5462138062004,
      "learning_rate": 3.31067778442141e-05,
      "loss_nats": 1.071753740310669,
      "step": 7480,
      "unclipped_grad_norm": 1.2698674201965332
    },
    {
      "bits_per_byte": 1.5915580639962847,
      "learning_rate": 3.298887400955457e-05,
      "loss_nats": 1.1031839847564697,
      "step": 7490,
      "unclipped_grad_norm": 1.3026750087738037
    },
    {
      "bits_per_byte": 1.5533068866699349,
      "learning_rate": 3.287322588080714e-05,
      "loss_nats": 1.0766702890396118,
      "step": 7500,
      "unclipped_grad_norm": 1.1910096406936646
    },
    {
      "bits_per_byte": 1.5004994417398587,
      "learning_rate": 3.275983543407924e-05,
      "loss_nats": 1.0400669574737549,
      "step": 7510,
      "unclipped_grad_norm": 1.2149840593338013
    },
    {
      "bits_per_byte": 1.5185469571561,
      "learning_rate": 3.2648704606900735e-05,
      "loss_nats": 1.0525765419006348,
      "step": 7520,
      "unclipped_grad_norm": 1.25714111328125
    },
    {
      "bits_per_byte": 1.5776331447038756,
      "learning_rate": 3.253983529819086e-05,
      "loss_nats": 1.0935319662094116,
      "step": 7530,
      "unclipped_grad_norm": 1.2656311988830566
    },
    {
      "bits_per_byte": 1.5129728274589855,
      "learning_rate": 3.243322936822571e-05,
      "loss_nats": 1.0487128496170044,
      "step": 7540,
      "unclipped_grad_norm": 1.2008068561553955
    },
    {
      "bits_per_byte": 1.542592195538531,
      "learning_rate": 3.232888863860648e-05,
      "loss_nats": 1.0692434310913086,
      "step": 7550,
      "unclipped_grad_norm": 1.3036167621612549
    },
    {
      "bits_per_byte": 1.5036082001368547,
      "learning_rate": 3.222681489222838e-05,
      "loss_nats": 1.0422217845916748,
      "step": 7560,
      "unclipped_grad_norm": 1.2302430868148804
    },
    {
      "bits_per_byte": 1.4793622580008667,
      "learning_rate": 3.212700987325009e-05,
      "loss_nats": 1.0254157781600952,
      "step": 7570,
      "unclipped_grad_norm": 1.2421420812606812
    },
    {
      "bits_per_byte": 1.6626436530774094,
      "learning_rate": 3.202947528706403e-05,
      "loss_nats": 1.1524567604064941,
      "step": 7580,
      "unclipped_grad_norm": 1.2984310388565063
    },
    {
      "bits_per_byte": 1.510733441362101,
      "learning_rate": 3.1934212800267116e-05,
      "loss_nats": 1.0471606254577637,
      "step": 7590,
      "unclipped_grad_norm": 1.2202924489974976
    },
    {
      "bits_per_byte": 1.5062574208907455,
      "learning_rate": 3.1841224040632484e-05,
      "loss_nats": 1.044058084487915,
      "step": 7600,
      "unclipped_grad_norm": 1.2830675840377808
    },
    {
      "bits_per_byte": 1.5167008953817387,
      "learning_rate": 3.175051059708143e-05,
      "loss_nats": 1.0512969493865967,
      "step": 7610,
      "unclipped_grad_norm": 1.2556214332580566
    },
    {
      "bits_per_byte": 1.4693692060724834,
      "learning_rate": 3.1662074019656414e-05,
      "loss_nats": 1.018489122390747,
      "step": 7620,
      "unclipped_grad_norm": 1.2434303760528564
    },
    {
      "bits_per_byte": 1.482038652013594,
      "learning_rate": 3.157591581949455e-05,
      "loss_nats": 1.0272709131240845,
      "step": 7630,
      "unclipped_grad_norm": 1.2516939640045166
    },
    {
      "bits_per_byte": 1.4886369383965965,
      "learning_rate": 3.149203746880175e-05,
      "loss_nats": 1.0318444967269897,
      "step": 7640,
      "unclipped_grad_norm": 1.2500675916671753
    },
    {
      "bits_per_byte": 1.61461749782397,
      "learning_rate": 3.141044040082763e-05,
      "loss_nats": 1.1191675662994385,
      "step": 7650,
      "unclipped_grad_norm": 1.2970722913742065
    },
    {
      "bits_per_byte": 1.5327549598918293,
      "learning_rate": 3.133112600984091e-05,
      "loss_nats": 1.0624247789382935,
      "step": 7660,
      "unclipped_grad_norm": 1.313043236732483
    },
    {
      "bits_per_byte": 1.560077327686451,
      "learning_rate": 3.125409565110571e-05,
      "loss_nats": 1.0813632011413574,
      "step": 7670,
      "unclipped_grad_norm": 1.2718157768249512
    },
    {
      "bits_per_byte": 1.506822211916179,
      "learning_rate": 3.117935064085835e-05,
      "loss_nats": 1.0444495677947998,
      "step": 7680,
      "unclipped_grad_norm": 1.2939503192901611
    },
    {
      "bits_per_byte": 1.5160353225228995,
      "learning_rate": 3.110689225628479e-05,
      "loss_nats": 1.0508356094360352,
      "step": 7690,
      "unclipped_grad_norm": 1.2968342304229736
    },
    {
      "bits_per_byte": 1.4732078588397413,
      "learning_rate": 3.1036721735498926e-05,
      "loss_nats": 1.0211498737335205,
      "step": 7700,
      "unclipped_grad_norm": 1.2705638408660889
    },
    {
      "bits_per_byte": 1.539139815805111,
      "learning_rate": 3.096884027752132e-05,
      "loss_nats": 1.0668504238128662,
      "step": 7710,
      "unclipped_grad_norm": 1.2731654644012451
    },
    {
      "bits_per_byte": 1.5645254869683667,
      "learning_rate": 3.090324904225885e-05,
      "loss_nats": 1.0844464302062988,
      "step": 7720,
      "unclipped_grad_norm": 1.281749963760376
    },
    {
      "bits_per_byte": 1.5594863952980864,
      "learning_rate": 3.083994915048472e-05,
      "loss_nats": 1.080953598022461,
      "step": 7730,
      "unclipped_grad_norm": 1.2457833290100098
    },
    {
      "bits_per_byte": 1.5044648457208023,
      "learning_rate": 3.077894168381942e-05,
      "loss_nats": 1.0428155660629272,
      "step": 7740,
      "unclipped_grad_norm": 1.313395619392395
    },
    {
      "bits_per_byte": 1.5058419108062608,
      "learning_rate": 3.0720227684712254e-05,
      "loss_nats": 1.0437700748443604,
      "step": 7750,
      "unclipped_grad_norm": 1.252753496170044
    },
    {
      "bits_per_byte": 1.5365542286320395,
      "learning_rate": 3.0663808156423456e-05,
      "loss_nats": 1.0650582313537598,
      "step": 7760,
      "unclipped_grad_norm": 1.208566665649414
    },
    {
      "bits_per_byte": 1.4627389309164203,
      "learning_rate": 3.060968406300711e-05,
      "loss_nats": 1.0138933658599854,
      "step": 7770,
      "unclipped_grad_norm": 1.2469086647033691
    },
    {
      "bits_per_byte": 1.539320397588517,
      "learning_rate": 3.055785632929463e-05,
      "loss_nats": 1.0669755935668945,
      "step": 7780,
      "unclipped_grad_norm": 1.2808659076690674
    },
    {
      "bits_per_byte": 1.4840185162903266,
      "learning_rate": 3.0508325840878984e-05,
      "loss_nats": 1.028643250465393,
      "step": 7790,
      "unclipped_grad_norm": 1.1823458671569824
    },
    {
      "bits_per_byte": 1.5017442521668036,
      "learning_rate": 3.046109344409957e-05,
      "loss_nats": 1.0409297943115234,
      "step": 7800,
      "unclipped_grad_norm": 1.2036527395248413
    },
    {
      "bits_per_byte": 1.4928728710873473,
      "learning_rate": 3.0416159946027727e-05,
      "loss_nats": 1.0347806215286255,
      "step": 7810,
      "unclipped_grad_norm": 1.2650182247161865
    },
    {
      "bits_per_byte": 1.4653472197994057,
      "learning_rate": 3.0373526114452946e-05,
      "loss_nats": 1.0157012939453125,
      "step": 7820,
      "unclipped_grad_norm": 1.2730145454406738
    },
    {
      "bits_per_byte": 1.5153749091435864,
      "learning_rate": 3.0333192677869803e-05,
      "loss_nats": 1.0503778457641602,
      "step": 7830,
      "unclipped_grad_norm": 1.3102401494979858
    },
    {
      "bits_per_byte": 1.5122207473267624,
      "learning_rate": 3.029516032546544e-05,
      "loss_nats": 1.0481915473937988,
      "step": 7840,
      "unclipped_grad_norm": 1.2572261095046997
    },
    {
      "bits_per_byte": 1.458128591994741,
      "learning_rate": 3.02594297071078e-05,
      "loss_nats": 1.0106977224349976,
      "step": 7850,
      "unclipped_grad_norm": 1.253402829170227
    },
    {
      "bits_per_byte": 1.5241797329371587,
      "learning_rate": 3.0226001433334603e-05,
      "loss_nats": 1.056480884552002,
      "step": 7860,
      "unclipped_grad_norm": 1.2006174325942993
    },
    {
      "bits_per_byte": 1.539945898489705,
      "learning_rate": 3.019487607534278e-05,
      "loss_nats": 1.0674091577529907,
      "step": 7870,
      "unclipped_grad_norm": 1.2778089046478271
    },
    {
      "bits_per_byte": 1.453407496245906,
      "learning_rate": 3.016605416497886e-05,
      "loss_nats": 1.007425308227539,
      "step": 7880,
      "unclipped_grad_norm": 1.2744390964508057
    },
    {
      "bits_per_byte": 1.5524588402185304,
      "learning_rate": 3.0139536194729734e-05,
      "loss_nats": 1.076082468032837,
      "step": 7890,
      "unclipped_grad_norm": 1.2452044486999512
    },
    {
      "bits_per_byte": 1.5187442212566398,
      "learning_rate": 3.0115322617714362e-05,
      "loss_nats": 1.0527132749557495,
      "step": 7900,
      "unclipped_grad_norm": 1.270271897315979
    },
    {
      "bits_per_byte": 1.5143311464355,
      "learning_rate": 3.0093413847675965e-05,
      "loss_nats": 1.0496543645858765,
      "step": 7910,
      "unclipped_grad_norm": 1.2356430292129517
    },
    {
      "bits_per_byte": 1.5350039770171622,
      "learning_rate": 3.0073810258974985e-05,
      "loss_nats": 1.063983678817749,
      "step": 7920,
      "unclipped_grad_norm": 1.2489451169967651
    },
    {
      "bits_per_byte": 1.4882054339255817,
      "learning_rate": 3.0056512186582658e-05,
      "loss_nats": 1.0315454006195068,
      "step": 7930,
      "unclipped_grad_norm": 1.2472636699676514
    },
    {
      "bits_per_byte": 1.4964919020094534,
      "learning_rate": 3.0041519926075316e-05,
      "loss_nats": 1.0372891426086426,
      "step": 7940,
      "unclipped_grad_norm": 1.3003731966018677
    },
    {
      "bits_per_byte": 1.5605986071012163,
      "learning_rate": 3.0028833733629313e-05,
      "loss_nats": 1.0817245244979858,
      "step": 7950,
      "unclipped_grad_norm": 1.2693594694137573
    },
    {
      "bits_per_byte": 1.4811777068633747,
      "learning_rate": 3.0018453826016686e-05,
      "loss_nats": 1.0266741514205933,
      "step": 7960,
      "unclipped_grad_norm": 1.3084068298339844
    },
    {
      "bits_per_byte": 1.5064964767754447,
      "learning_rate": 3.001038038060143e-05,
      "loss_nats": 1.0442237854003906,
      "step": 7970,
      "unclipped_grad_norm": 1.2679471969604492
    },
    {
      "bits_per_byte": 1.5642893547887322,
      "learning_rate": 3.0004613535336433e-05,
      "loss_nats": 1.0842827558517456,
      "step": 7980,
      "unclipped_grad_norm": 1.3388662338256836
    },
    {
      "bits_per_byte": 1.598760697414419,
      "learning_rate": 3.000115338876117e-05,
      "loss_nats": 1.1081764698028564,
      "step": 7990,
      "unclipped_grad_norm": 1.3723206520080566
    },
    {
      "bits_per_byte": 1.4810289418703784,
      "learning_rate": 3e-05,
      "loss_nats": 1.0265710353851318,
      "step": 8000,
      "unclipped_grad_norm": 1.1791528463363647
    }
  ]
}