llm-atlas/experiments/k3/attnres_gradient/results/raw/formal-depth-16-baseline-seed-2026073001.json

{
  "architecture": "baseline",
  "batch_size": 32,
  "canonical_sha256_without_self": "09d1bf3e3857ce9c1fd57a3596a5acf1e980ac95e31a3ef6ad060395c15d99ae",
  "depth": 16,
  "diagnostics": [
    {
      "activation_grad_rms_by_block": [
        0.0002865509013645351,
        0.00024785567075014114,
        0.00021551651298068464,
        0.00019683866412378848,
        0.0001823508064262569,
        0.0001679604029050097,
        0.00015584587526973337,
        0.00014737028686795384,
        0.00014130656199995428,
        0.00013612695329356939,
        0.00013129493163432926,
        0.00012807638267986476,
        0.00012550120300147682,
        0.00012324050476308912,
        0.00012108208466088399,
        0.00011888353037647903
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00023669043730478734,
        "first_to_last_ratio": 1.9372775995887173,
        "imbalance_abs_log_ratio": 0.6612836883477485,
        "last_quartile_mean": 0.00012217683070048224,
        "mean": 0.00016411257956860936,
        "normalized": [
          1.7460629899168627,
          1.5102783187106137,
          1.313223602646409,
          1.1994124072706904,
          1.1111324123086055,
          1.0234462424910682,
          0.9496278449793059,
          0.8979828801383491,
          0.8610343117596252,
          0.8294729974472175,
          0.8000296624393728,
          0.7804178266926869,
          0.7647262832098098,
          0.7509509940495869,
          0.7377989242455608,
          0.7244023016942358
        ],
        "population_cv": 0.29361872380036635
      },
      "activation_output_rms_by_block": [
        0.02866268903017044,
        0.029099803417921066,
        0.02953805774450302,
        0.030088091269135475,
        0.030644793063402176,
        0.03131929785013199,
        0.03213750571012497,
        0.03286394104361534,
        0.033869802951812744,
        0.03499744459986687,
        0.03588006645441055,
        0.037365153431892395,
        0.03789033368229866,
        0.039445169270038605,
        0.04050002992153168,
        0.04121527820825577
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.0293471603654325,
        "first_to_last_ratio": 0.7380574840395957,
        "imbalance_abs_log_ratio": 0.3037335657624931,
        "last_quartile_mean": 0.03976270277053118,
        "mean": 0.034094841103069484,
        "normalized": [
          0.8406752488894867,
          0.8534957922212247,
          0.8663497699023965,
          0.8824822259232266,
          0.8988102619619861,
          0.9185934539320196,
          0.942591449919727,
          0.9638977622528551,
          0.9933996421752943,
          1.0264733158329964,
          1.0523605710888722,
          1.0959180985456622,
          1.1113216092650298,
          1.1569248600043318,
          1.1878638706395246,
          1.2088420674453664
        ],
        "population_cv": 0.11952987076282337
      },
      "bits_per_byte": 8.096463027059821,
      "branch_output_rms_by_sublayer": [
        0.0033055038657039404,
        0.0037518907338380814,
        0.0033751516602933407,
        0.003911525942385197,
        0.004229962360113859,
        0.0038461871445178986,
        0.004243654198944569,
        0.0038366704247891903,
        0.004372371360659599,
        0.003851204412057996,
        0.005078981164842844,
        0.003818925702944398,
        0.0054668826051056385,
        0.0038749484810978174,
        0.005823117680847645,
        0.0037566579412668943,
        0.0065947300754487514,
        0.003876061411574483,
        0.006763988174498081,
        0.003795720636844635,
        0.006863070651888847,
        0.0038706199266016483,
        0.007856340147554874,
        0.003839300014078617,
        0.00830968376249075,
        0.003927029203623533,
        0.009655521251261234,
        0.0038513424806296825,
        0.009004893712699413,
        0.0039484030567109585,
        0.008608299307525158,
        0.003791053779423237
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00809059897248305,
        0.007313812062277777,
        0.007090710224412361,
        0.0067033738367094624,
        0.006535407867454384,
        0.006785021795736036,
        0.006295993569919972,
        0.005509300326230647,
        0.006231736048910802,
        0.006047390980174007,
        0.005629405359532731,
        0.005784042737956458,
        0.0059591736113607996,
        0.0061983173180850636,
        0.006571690039252182,
        0.005060205437758869
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.007299623773970663,
        "first_to_last_ratio": 1.227374872012571,
        "imbalance_abs_log_ratio": 0.2048776382299483,
        "last_quartile_mean": 0.005947346601614228,
        "mean": 0.006362886261765913,
        "normalized": [
          1.271529717747562,
          1.1494488132258316,
          1.1143858200043408,
          1.05351149791715,
          1.0271137340180259,
          1.066343404015675,
          0.9894870520870547,
          0.8658492545019393,
          0.9793882512652816,
          0.9504163254515969,
          0.8847251275509024,
          0.909028151691524,
          0.9365519618304368,
          0.9741361173356609,
          1.0328158902888074,
          0.7952688810682109
        ],
        "population_cv": 0.11384977604868386
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.02823694422841072,
        0.028404271230101585,
        0.02866268903017044,
        0.02886480651795864,
        0.029099803417921066,
        0.029365191236138344,
        0.02953805774450302,
        0.029881296679377556,
        0.030088091269135475,
        0.030441828072071075,
        0.030644793063402176,
        0.03106885403394699,
        0.03131929785013199,
        0.031892918050289154,
        0.03213750571012497,
        0.03266792371869087,
        0.03286394104361534,
        0.03364725783467293,
        0.033869802951812744,
        0.03485054895281792,
        0.03499744459986687,
        0.03562505170702934,
        0.03588006645441055,
        0.03719272464513779,
        0.037365153431892395,
        0.03768136352300644,
        0.03789033368229866,
        0.03927159309387207,
        0.039445169270038605,
        0.04041972756385803,
        0.04050002992153168,
        0.04110744968056679
      ],
      "loss_nats": 5.6120405197143555,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 0,
      "stream_state_rms_by_sublayer": [
        0.028404271230101585,
        0.02866268903017044,
        0.02886480651795864,
        0.029099803417921066,
        0.029365191236138344,
        0.02953805774450302,
        0.029881296679377556,
        0.030088091269135475,
        0.030441828072071075,
        0.030644793063402176,
        0.03106885403394699,
        0.03131929785013199,
        0.031892918050289154,
        0.03213750571012497,
        0.03266792371869087,
        0.03286394104361534,
        0.03364725783467293,
        0.033869802951812744,
        0.03485054895281792,
        0.03499744459986687,
        0.03562505170702934,
        0.03588006645441055,
        0.03719272464513779,
        0.037365153431892395,
        0.03768136352300644,
        0.03789033368229866,
        0.03927159309387207,
        0.039445169270038605,
        0.04041972756385803,
        0.04050002992153168,
        0.04110744968056679,
        0.04121527820825577
      ]
    },
    {
      "activation_grad_rms_by_block": [
        8.509291365044191e-05,
        7.616560469614342e-05,
        6.756903894711286e-05,
        5.824641266372055e-05,
        5.0209364417241886e-05,
        4.2265295633114874e-05,
        3.587255923775956e-05,
        3.076014036196284e-05,
        2.70738746621646e-05,
        2.470596882631071e-05,
        2.293860052304808e-05,
        2.172161293856334e-05,
        2.083007166220341e-05,
        2.0244739062036388e-05,
        1.9782984963967465e-05,
        1.9472558051347733e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 7.176849248935468e-05,
        "first_to_last_ratio": 3.5736674444153773,
        "imbalance_abs_log_ratio": 1.2735923638392248,
        "last_quartile_mean": 2.008258843488875e-05,
        "mean": 3.8934483768571226e-05,
        "normalized": [
          2.185541078603713,
          1.9562505348440236,
          1.7354548566444863,
          1.496010914384804,
          1.28958598027623,
          1.0855491467240759,
          0.921357001957136,
          0.7900487533057547,
          0.6953700689366589,
          0.6345523668212576,
          0.5891589743271394,
          0.5579016551928063,
          0.5350031551983206,
          0.519969371685323,
          0.5081095997460412,
          0.5001365413522295
        ],
        "population_cv": 0.5464287537448471
      },
      "activation_output_rms_by_block": [
        0.030502017587423325,
        0.03408988192677498,
        0.038947492837905884,
        0.04481571167707443,
        0.0512830950319767,
        0.05888458341360092,
        0.06758055835962296,
        0.07802816480398178,
        0.08916938304901123,
        0.10193294286727905,
        0.11700897663831711,
        0.13432858884334564,
        0.15294352173805237,
        0.17351281642913818,
        0.19949108362197876,
        0.2260059416294098
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.037088776007294655,
        "first_to_last_ratio": 0.19729295890734105,
        "imbalance_abs_log_ratio": 1.623065553831421,
        "last_quartile_mean": 0.18798834085464478,
        "mean": 0.09990779752843082,
        "normalized": [
          0.3053016715611547,
          0.3412134264802905,
          0.38983436529889043,
          0.4485707100521478,
          0.5133042295060407,
          0.5893892656060614,
          0.676429268099835,
          0.7810017509571989,
          0.8925167529955432,
          1.0202701429612833,
          1.1711696137132819,
          1.344525574243789,
          1.530846695870051,
          1.7367294717888415,
          1.9967518908142226,
          2.262145170051368
        ],
        "population_cv": 0.5980263271119333
      },
      "bits_per_byte": 5.097173307164379,
      "branch_output_rms_by_sublayer": [
        0.004186291713267565,
        0.008577318862080574,
        0.004227571655064821,
        0.009763575159013271,
        0.005347618833184242,
        0.009802278131246567,
        0.006270815618336201,
        0.009791693650186062,
        0.005826776847243309,
        0.010311179794371128,
        0.006844967603683472,
        0.010455452837049961,
        0.00757947750389576,
        0.01094198040664196,
        0.009920897893607616,
        0.011405051685869694,
        0.010475707240402699,
        0.012199299409985542,
        0.012102399952709675,
        0.012131479568779469,
        0.013124288059771061,
        0.013151587918400764,
        0.01721765287220478,
        0.012752228416502476,
        0.016962353140115738,
        0.012886312790215015,
        0.01880411058664322,
        0.012915017083287239,
        0.024245616048574448,
        0.014021328650414944,
        0.02384282648563385,
        0.013684137724339962
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0005223696653307753,
        0.0004106838611462913,
        0.0004143731432945227,
        0.00042601318321813184,
        0.0003557105812490162,
        0.00034956609676611394,
        0.00031105173709675404,
        0.00028609388990673116,
        0.0002585596876336936,
        0.00022586454081316873,
        0.00020762781370074637,
        0.0002072362198298983,
        0.00019288470561650055,
        0.00019339001384203696,
        0.00020597582961175812,
        0.0001909336964292271
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.0004433599632474303,
        "first_to_last_ratio": 2.2643967408442993,
        "imbalance_abs_log_ratio": 0.8173083838772207,
        "last_quartile_mean": 0.0001957960613748807,
        "mean": 0.00029739591659283537,
        "normalized": [
          1.7564789433405417,
          1.3809330869480578,
          1.3933383754620974,
          1.4324782535645448,
          1.1960842816010129,
          1.175423323800053,
          1.0459179825344234,
          0.9619966984900542,
          0.8694123665043039,
          0.7594742503556288,
          0.6981528733799311,
          0.696836130785297,
          0.6485788635779385,
          0.6502779730725327,
          0.6925980422715741,
          0.6420185543120094
        ],
        "population_cv": 0.3407962002760142
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.028665566816926003,
        0.0289339367300272,
        0.030502017587423325,
        0.030956681817770004,
        0.03408988192677498,
        0.03459245339035988,
        0.038947492837905884,
        0.03989224135875702,
        0.04481571167707443,
        0.04556844383478165,
        0.0512830950319767,
        0.052534833550453186,
        0.05888458341360092,
        0.060703132301568985,
        0.06758055835962296,
        0.07060512155294418,
        0.07802816480398178,
        0.08117377012968063,
        0.08916938304901123,
        0.09386278688907623,
        0.10193294286727905,
        0.10801172256469727,
        0.11700897663831711,
        0.12567132711410522,
        0.13432858884334564,
        0.14358282089233398,
        0.15294352173805237,
        0.16399860382080078,
        0.17351281642913818,
        0.18877196311950684,
        0.19949108362197876,
        0.21516621112823486
      ],
      "loss_nats": 3.5330913066864014,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 100,
      "stream_state_rms_by_sublayer": [
        0.0289339367300272,
        0.030502017587423325,
        0.030956681817770004,
        0.03408988192677498,
        0.03459245339035988,
        0.038947492837905884,
        0.03989224135875702,
        0.04481571167707443,
        0.04556844383478165,
        0.0512830950319767,
        0.052534833550453186,
        0.05888458341360092,
        0.060703132301568985,
        0.06758055835962296,
        0.07060512155294418,
        0.07802816480398178,
        0.08117377012968063,
        0.08916938304901123,
        0.09386278688907623,
        0.10193294286727905,
        0.10801172256469727,
        0.11700897663831711,
        0.12567132711410522,
        0.13432858884334564,
        0.14358282089233398,
        0.15294352173805237,
        0.16399860382080078,
        0.17351281642913818,
        0.18877196311950684,
        0.19949108362197876,
        0.21516621112823486,
        0.2260059416294098
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.00014349650882650167,
        0.00014052700134925544,
        0.00012983661144971848,
        0.0001149297459051013,
        0.0001033403241308406,
        9.356468945043162e-05,
        8.457523654215038e-05,
        7.733132952125743e-05,
        7.047204417176545e-05,
        6.370909250108525e-05,
        5.659156886395067e-05,
        4.938157871947624e-05,
        4.27361810579896e-05,
        3.8237969420151785e-05,
        3.594611553126015e-05,
        3.5643912269733846e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00013219746688264422,
        "first_to_last_ratio": 3.4660158989817993,
        "imbalance_abs_log_ratio": 1.243005778409783,
        "last_quartile_mean": 3.8141044569783844e-05,
        "mean": 8.001999435691687e-05,
        "normalized": [
          1.7932581722820122,
          1.7561486035909524,
          1.622552119544051,
          1.4362628594107976,
          1.2914312849099563,
          1.1692663840127346,
          1.0569262997559778,
          0.9664000871623777,
          0.8806804441579406,
          0.7961646712560445,
          0.7072178562214424,
          0.6171154986492164,
          0.5340687837013764,
          0.47785518766218865,
          0.44921417228459215,
          0.44543757539833934
        ],
        "population_cv": 0.4531561582313658
      },
      "activation_output_rms_by_block": [
        0.05229421332478523,
        0.06407254934310913,
        0.07844720035791397,
        0.08364550769329071,
        0.0890495702624321,
        0.09326893091201782,
        0.09662611037492752,
        0.09771094471216202,
        0.09944888204336166,
        0.10062506794929504,
        0.10494095832109451,
        0.11238908767700195,
        0.12405165284872055,
        0.14480136334896088,
        0.17238570749759674,
        0.2138574868440628
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.06961486767977476,
        "first_to_last_ratio": 0.4250665264722616,
        "imbalance_abs_log_ratio": 0.8555095894318028,
        "last_quartile_mean": 0.16377405263483524,
        "mean": 0.10797595209442079,
        "normalized": [
          0.48431351898667185,
          0.5933964748658124,
          0.7265247384835739,
          0.7746679336538376,
          0.8247166941816979,
          0.8637935494234656,
          0.8948854675569957,
          0.9049324670619027,
          0.9210280633265217,
          0.9319210990730817,
          0.9718919471006628,
          1.040871467183008,
          1.1488822320385021,
          1.3410519707419453,
          1.596519448579181,
          1.9806029277431396
        ],
        "population_cv": 0.361888909970992
      },
      "bits_per_byte": 3.1304997557876018,
      "branch_output_rms_by_sublayer": [
        0.014489351771771908,
        0.02032402716577053,
        0.0192408449947834,
        0.013354538939893246,
        0.02438616193830967,
        0.010533644817769527,
        0.026007508859038353,
        0.009820356033742428,
        0.019688289612531662,
        0.009847180917859077,
        0.015725189819931984,
        0.00970252975821495,
        0.017999378964304924,
        0.012247176840901375,
        0.019808221608400345,
        0.012050271965563297,
        0.019276488572359085,
        0.012983737513422966,
        0.01952224411070347,
        0.015011198818683624,
        0.01836247928440571,
        0.01764002814888954,
        0.020674988627433777,
        0.02150883898139,
        0.01928989216685295,
        0.025364212691783905,
        0.025591954588890076,
        0.031167680397629738,
        0.011463184840977192,
        0.039293259382247925,
        0.019522786140441895,
        0.04468192160129547
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0006452065099731355,
        0.00042477980025779674,
        0.0003406169302843733,
        0.00031739507901072504,
        0.0002899614562735046,
        0.0003022750042151422,
        0.0002946052417003999,
        0.0002490739832611916,
        0.00022953214745651343,
        0.00020477988838642542,
        0.00019411347501299613,
        0.00019138854173966402,
        0.00017481673923870362,
        0.00016715970609089626,
        0.0001418296786517743,
        0.00014407196058940572
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00043199957988150763,
        "first_to_last_ratio": 2.752123958438998,
        "imbalance_abs_log_ratio": 1.0123729620941166,
        "last_quartile_mean": 0.00015696952114269496,
        "mean": 0.00026947538388391546,
        "normalized": [
          2.394305931301975,
          1.576321347558719,
          1.2640001671956211,
          1.177825872018991,
          1.0760220547581225,
          1.121716573360022,
          1.0932547435475957,
          0.9242921548948887,
          0.8517740809876394,
          0.7599205739498657,
          0.720338430231595,
          0.7102264369427908,
          0.6487299014815064,
          0.6203153092562447,
          0.526317753434843,
          0.5346386690795809
        ],
        "population_cv": 0.4584519745606109
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.043568700551986694,
        0.04671650007367134,
        0.05229421332478523,
        0.059547748416662216,
        0.06407254934310913,
        0.07493934780359268,
        0.07844720035791397,
        0.08121047168970108,
        0.08364550769329071,
        0.0867149755358696,
        0.0890495702624321,
        0.09094811230897903,
        0.09326893091201782,
        0.09377976506948471,
        0.09662611037492752,
        0.09500574320554733,
        0.09771094471216202,
        0.0959710031747818,
        0.09944888204336166,
        0.09544596821069717,
        0.10062506794929504,
        0.097707599401474,
        0.10494095832109451,
        0.10151325911283493,
        0.11238908767700195,
        0.10858181118965149,
        0.12405165284872055,
        0.1238408088684082,
        0.14480136334896088,
        0.1446269154548645,
        0.17238570749759674,
        0.18076260387897491
      ],
      "loss_nats": 2.1698970794677734,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 500,
      "stream_state_rms_by_sublayer": [
        0.04671650007367134,
        0.05229421332478523,
        0.059547748416662216,
        0.06407254934310913,
        0.07493934780359268,
        0.07844720035791397,
        0.08121047168970108,
        0.08364550769329071,
        0.0867149755358696,
        0.0890495702624321,
        0.09094811230897903,
        0.09326893091201782,
        0.09377976506948471,
        0.09662611037492752,
        0.09500574320554733,
        0.09771094471216202,
        0.0959710031747818,
        0.09944888204336166,
        0.09544596821069717,
        0.10062506794929504,
        0.097707599401474,
        0.10494095832109451,
        0.10151325911283493,
        0.11238908767700195,
        0.10858181118965149,
        0.12405165284872055,
        0.1238408088684082,
        0.14480136334896088,
        0.1446269154548645,
        0.17238570749759674,
        0.18076260387897491,
        0.2138574868440628
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.00016704612062312663,
        0.00018574431305751204,
        0.0002023741544689983,
        0.00018453448137734085,
        0.00017654163821134716,
        0.00017114127695094794,
        0.00016412502736784518,
        0.00015690893633291125,
        0.0001434983714716509,
        0.00012688931019511074,
        0.00010475149611011147,
        9.039731958182529e-05,
        7.724175520706922e-05,
        6.240545917535201e-05,
        4.393732160679065e-05,
        2.913598837039899e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00018492476738174446,
        "first_to_last_ratio": 3.4773281598182453,
        "imbalance_abs_log_ratio": 1.246264228732751,
        "last_quartile_mean": 5.3180131089902716e-05,
        "mean": 0.00013041706063177116,
        "normalized": [
          1.2808609534206308,
          1.4242332418605552,
          1.551746017650221,
          1.4149566052433022,
          1.3536698140269194,
          1.312261418267664,
          1.2584628619352762,
          1.2031319796106978,
          1.100303677881642,
          0.9729502381086403,
          0.8032039336162798,
          0.6931402927187534,
          0.5922672603790627,
          0.47850686768314793,
          0.33689857288569336,
          0.22340626471151362
        ],
        "population_cv": 0.41009229518876916
      },
      "activation_output_rms_by_block": [
        0.08545628190040588,
        0.1388079822063446,
        0.16856427490711212,
        0.17068639397621155,
        0.17051894962787628,
        0.1691380739212036,
        0.16795755922794342,
        0.15964734554290771,
        0.14745007455348969,
        0.13513104617595673,
        0.13549363613128662,
        0.14635984599590302,
        0.16386321187019348,
        0.19344064593315125,
        0.2505952715873718,
        0.40341031551361084
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.14087873324751854,
        "first_to_last_ratio": 0.5572131614407935,
        "imbalance_abs_log_ratio": 0.5848074166792235,
        "last_quartile_mean": 0.25282736122608185,
        "mean": 0.17540755681693554,
        "normalized": [
          0.4871870029498929,
          0.7913455082850954,
          0.9609863905865502,
          0.9730846097716802,
          0.9721300080921755,
          0.9642576237335369,
          0.9575274992469833,
          0.9101508990831222,
          0.8406141515748735,
          0.77038326414287,
          0.7724503926173194,
          0.8343987491294447,
          0.9341855895137383,
          1.1028067971725755,
          1.4286458128420665,
          2.2998457012580755
        ],
        "population_cv": 0.3845661956400748
      },
      "bits_per_byte": 2.092252187349085,
      "branch_output_rms_by_sublayer": [
        0.036218997091054916,
        0.037682514637708664,
        0.07264158874750137,
        0.009124485775828362,
        0.041704896837472916,
        0.003249978879466653,
        0.02913317084312439,
        0.0056674908846616745,
        0.02289137803018093,
        0.008465219289064407,
        0.019867170602083206,
        0.010350172407925129,
        0.024333713576197624,
        0.014746740460395813,
        0.03023042529821396,
        0.01600552536547184,
        0.0432618223130703,
        0.020560188218951225,
        0.04687124863266945,
        0.025769658386707306,
        0.04757922887802124,
        0.04326478764414787,
        0.029664678499102592,
        0.051030874252319336,
        0.02047067880630493,
        0.05886564403772354,
        0.0222918838262558,
        0.07155895233154297,
        0.011497496627271175,
        0.10460206121206284,
        0.015142823569476604,
        0.212371364235878
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00040658806478613896,
        0.0003235049151816101,
        0.00024397059654808256,
        0.0002149925859972515,
        0.00020566351114887415,
        0.00024487046739718384,
        0.00031061648932541417,
        0.0003911240721720923,
        0.000499392430316199,
        0.0005175204124242609,
        0.0005752549257371158,
        0.00046811732551685614,
        0.00041118767134459974,
        0.0003618888130405475,
        0.0003117946917671641,
        0.00025214502653225866
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00029726404062827076,
        "first_to_last_ratio": 0.8893356416516115,
        "imbalance_abs_log_ratio": 0.11728056505781807,
        "last_quartile_mean": 0.0003342540506711425,
        "mean": 0.0003586644999522281,
        "normalized": [
          1.1336166942652368,
          0.9019708257290558,
          0.6802195271084202,
          0.5994253293144072,
          0.5734147404503854,
          0.6827284758591918,
          0.8660363358146301,
          1.0905012127606375,
          1.3923664884110778,
          1.4429094947874448,
          1.6038802998728234,
          1.3051677140592575,
          1.1464409535913576,
          1.0089897748139245,
          0.8693213067049939,
          0.7030108264571566
        ],
        "population_cv": 0.30960458504189897
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.05539870634675026,
        0.07199405878782272,
        0.08545628190040588,
        0.1365138292312622,
        0.1388079822063446,
        0.16841135919094086,
        0.16856427490711212,
        0.17091451585292816,
        0.17068639397621155,
        0.17101584374904633,
        0.17051894962787628,
        0.16970102488994598,
        0.1691380739212036,
        0.16952496767044067,
        0.16795755922794342,
        0.16297046840190887,
        0.15964734554290771,
        0.14957723021507263,
        0.14745007455348969,
        0.1322014182806015,
        0.13513104617595673,
        0.12475068122148514,
        0.13549363613128662,
        0.1319173276424408,
        0.14635984599590302,
        0.14501839876174927,
        0.16386321187019348,
        0.16489192843437195,
        0.19344064593315125,
        0.19336768984794617,
        0.2505952715873718,
        0.2517329454421997
      ],
      "loss_nats": 1.4502387046813965,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 2000,
      "stream_state_rms_by_sublayer": [
        0.07199405878782272,
        0.08545628190040588,
        0.1365138292312622,
        0.1388079822063446,
        0.16841135919094086,
        0.16856427490711212,
        0.17091451585292816,
        0.17068639397621155,
        0.17101584374904633,
        0.17051894962787628,
        0.16970102488994598,
        0.1691380739212036,
        0.16952496767044067,
        0.16795755922794342,
        0.16297046840190887,
        0.15964734554290771,
        0.14957723021507263,
        0.14745007455348969,
        0.1322014182806015,
        0.13513104617595673,
        0.12475068122148514,
        0.13549363613128662,
        0.1319173276424408,
        0.14635984599590302,
        0.14501839876174927,
        0.16386321187019348,
        0.16489192843437195,
        0.19344064593315125,
        0.19336768984794617,
        0.2505952715873718,
        0.2517329454421997,
        0.40341031551361084
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.00017181786824949086,
        0.0001916729233926162,
        0.00020346585370134562,
        0.00018768427253235132,
        0.00018128244846593589,
        0.00017597789701540023,
        0.0001705518807284534,
        0.00016394301201216877,
        0.0001481794024584815,
        0.0001306164776906371,
        0.00010858976020244882,
        9.323586709797382e-05,
        7.920334610389546e-05,
        6.374791701091453e-05,
        4.397464726935141e-05,
        2.655091702763457e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.000188660229468951,
        "first_to_last_ratio": 3.535001559771659,
        "imbalance_abs_log_ratio": 1.2627137405851174,
        "last_quartile_mean": 5.336920685294899e-05,
        "mean": 0.00013378090568494372,
        "normalized": [
          1.2843228065305883,
          1.4327375226776322,
          1.5208885951221702,
          1.4029227233245896,
          1.3550696755847886,
          1.315418640010041,
          1.2748596659235207,
          1.2254589784154797,
          1.107627442794015,
          0.9763461909746755,
          0.8116984979768304,
          0.6969295552352281,
          0.5920377478264399,
          0.476509832883341,
          0.3287064551118561,
          0.19846566960880369
        ],
        "population_cv": 0.41282075642614385
      },
      "activation_output_rms_by_block": [
        0.10104851424694061,
        0.1665971875190735,
        0.18799307942390442,
        0.18892255425453186,
        0.18902969360351562,
        0.1888171136379242,
        0.18645423650741577,
        0.1778823733329773,
        0.16393223404884338,
        0.15616974234580994,
        0.16218091547489166,
        0.17675508558750153,
        0.19963829219341278,
        0.23325374722480774,
        0.2989467680454254,
        0.5114414095878601
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.1611403338611126,
        "first_to_last_ratio": 0.5184360907576058,
        "imbalance_abs_log_ratio": 0.6569385168504118,
        "last_quartile_mean": 0.3108200542628765,
        "mean": 0.20556643418967724,
        "normalized": [
          0.49156135166358245,
          0.8104299136957027,
          0.9145125281029207,
          0.9190340582558922,
          0.9195552491273791,
          0.9185211310505179,
          0.907026661441106,
          0.8653279122837938,
          0.797465961284235,
          0.7597044865880744,
          0.7889464839636536,
          0.8598440999585013,
          0.9711619164887857,
          1.1346879082875139,
          1.4542586644743063,
          2.4879616733340346
        ],
        "population_cv": 0.4294257518805603
      },
      "bits_per_byte": 1.9625077876075785,
      "branch_output_rms_by_sublayer": [
        0.053652405738830566,
        0.041928116232156754,
        0.08492356538772583,
        0.010198036208748817,
        0.03331374749541283,
        0.00243964372202754,
        0.028503542765975,
        0.005333597306162119,
        0.022150175645947456,
        0.008892077021300793,
        0.02107803337275982,
        0.011626621708273888,
        0.028634268790483475,
        0.017805112525820732,
        0.03554495424032211,
        0.0204806849360466,
        0.0531805157661438,
        0.028403647243976593,
        0.05312826856970787,
        0.03512146323919296,
        0.05336543545126915,
        0.05582646653056145,
        0.03752146288752556,
        0.06727498769760132,
        0.032218869775533676,
        0.07892175018787384,
        0.028152117505669594,
        0.09350594878196716,
        0.016922952607274055,
        0.13253848254680634,
        0.02406182885169983,
        0.29775944352149963
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00044260282991257096,
        0.0003095538604202773,
        0.00019879384700301542,
        0.00020176150294426366,
        0.00022494965081173505,
        0.00026863158839826584,
        0.00037546716364749414,
        0.0004673111939829261,
        0.0006383031641084419,
        0.0006422551721113508,
        0.0007081700332195422,
        0.0005856936655792496,
        0.0005168621280586129,
        0.00045203145589075384,
        0.0003829553066507503,
        0.0003084257693651058
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.0002881780100700318,
        "first_to_last_ratio": 0.6942899678444003,
        "imbalance_abs_log_ratio": 0.36486558463682156,
        "last_quartile_mean": 0.0004150686649913057,
        "mean": 0.00042023552075652225,
        "normalized": [
          1.0532256509772957,
          0.7366199312780793,
          0.47305341215597385,
          0.48011529958497023,
          0.5352942331166415,
          0.6392405570920472,
          0.8934684125976915,
          1.1120221183151215,
          1.5189176844435874,
          1.5283219537347563,
          1.6851741422159423,
          1.3937271759532939,
          1.2299344118463382,
          1.075662178858635,
          0.9112873323067531,
          0.7339355055228726
        ],
        "population_cv": 0.3791677394455083
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.058993492275476456,
        0.08599413186311722,
        0.10104851424694061,
        0.16386346518993378,
        0.1665971875190735,
        0.18777909874916077,
        0.18799307942390442,
        0.18934063613414764,
        0.18892255425453186,
        0.18942749500274658,
        0.18902969360351562,
        0.18932336568832397,
        0.1888171136379242,
        0.18826909363269806,
        0.18645423650741577,
        0.18273916840553284,
        0.1778823733329773,
        0.16554005444049835,
        0.16393223404884338,
        0.15082815289497375,
        0.15616974234580994,
        0.1484535187482834,
        0.16218091547489166,
        0.15933844447135925,
        0.17675508558750153,
        0.177049919962883,
        0.19963829219341278,
        0.20120804011821747,
        0.23325374722480774,
        0.23333875834941864,
        0.2989467680454254,
        0.3003007471561432
      ],
      "loss_nats": 1.360306739807129,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 4000,
      "stream_state_rms_by_sublayer": [
        0.08599413186311722,
        0.10104851424694061,
        0.16386346518993378,
        0.1665971875190735,
        0.18777909874916077,
        0.18799307942390442,
        0.18934063613414764,
        0.18892255425453186,
        0.18942749500274658,
        0.18902969360351562,
        0.18932336568832397,
        0.1888171136379242,
        0.18826909363269806,
        0.18645423650741577,
        0.18273916840553284,
        0.1778823733329773,
        0.16554005444049835,
        0.16393223404884338,
        0.15082815289497375,
        0.15616974234580994,
        0.1484535187482834,
        0.16218091547489166,
        0.15933844447135925,
        0.17675508558750153,
        0.177049919962883,
        0.19963829219341278,
        0.20120804011821747,
        0.23325374722480774,
        0.23333875834941864,
        0.2989467680454254,
        0.3003007471561432,
        0.5114414095878601
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.0002529843186493963,
        0.0002798262285068631,
        0.0002946711319964379,
        0.00027859603869728744,
        0.0002704691141843796,
        0.0002655670396052301,
        0.0002578931744210422,
        0.00024818466044962406,
        0.00022750988136976957,
        0.00020225343178026378,
        0.00016641257388982922,
        0.0001406865194439888,
        0.000117213734483812,
        9.274082549382001e-05,
        6.181560456752777e-05,
        3.130940967821516e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.0002765194294624962,
        "first_to_last_ratio": 3.6494630846841103,
        "imbalance_abs_log_ratio": 1.2945800566875763,
        "last_quartile_mean": 7.576989355584374e-05,
        "mean": 0.00019925835545109294,
        "normalized": [
          1.269629662839861,
          1.4043387434036372,
          1.478839526349316,
          1.3981649003706025,
          1.3573790347314445,
          1.3327774336188996,
          1.2942652961137227,
          1.2455420496057434,
          1.1417833939998108,
          1.0150311203883542,
          0.8351598281190995,
          0.7060507908212645,
          0.5882500345767512,
          0.4654300457507431,
          0.31022841891666686,
          0.15712972039408363
        ],
        "population_cv": 0.41719159337744666
      },
      "activation_output_rms_by_block": [
        0.10425227880477905,
        0.15637855231761932,
        0.1697961539030075,
        0.17037443816661835,
        0.16992402076721191,
        0.16921496391296387,
        0.1664763242006302,
        0.1570158749818802,
        0.1424521654844284,
        0.13391907513141632,
        0.13948261737823486,
        0.15248820185661316,
        0.17248840630054474,
        0.20097237825393677,
        0.2573379576206207,
        0.4688045382499695
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.15020035579800606,
        "first_to_last_ratio": 0.5463801662721245,
        "imbalance_abs_log_ratio": 0.6044402703006951,
        "last_quartile_mean": 0.27490082010626793,
        "mean": 0.18321112170815468,
        "normalized": [
          0.5690281126647281,
          0.853542900996599,
          0.9267786383267906,
          0.9299350202004414,
          0.927476559189958,
          0.9236063964638243,
          0.9086583972004597,
          0.8570215253198324,
          0.7775301202038757,
          0.7309549435800197,
          0.7613217804562272,
          0.8323086526347376,
          0.9414734470940544,
          1.0969442050252538,
          1.4045979044358783,
          2.5588213962073194
        ],
        "population_cv": 0.43808900348348184
      },
      "bits_per_byte": 1.8689156889212923,
      "branch_output_rms_by_sublayer": [
        0.06537462025880814,
        0.033620789647102356,
        0.06649091839790344,
        0.009441502392292023,
        0.024681083858013153,
        0.00226230570115149,
        0.0236189141869545,
        0.004977936390787363,
        0.01721731387078762,
        0.008134713396430016,
        0.01816825568675995,
        0.010567971505224705,
        0.02614794671535492,
        0.015832560136914253,
        0.03134627267718315,
        0.01896953023970127,
        0.04721185564994812,
        0.02692008763551712,
        0.04645893722772598,
        0.03406013175845146,
        0.04632203280925751,
        0.053266141563653946,
        0.032039228826761246,
        0.06378299742937088,
        0.028799353167414665,
        0.07541537284851074,
        0.025642387568950653,
        0.08898185938596725,
        0.017618922516703606,
        0.12298691272735596,
        0.026708398014307022,
        0.2926487922668457
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0005010958566464473,
        0.000367257410216879,
        0.0002469638962710481,
        0.00025453790022962814,
        0.00027094925891000524,
        0.0003539210655471446,
        0.0005067460397887177,
        0.0006525825825832493,
        0.0009467684204760593,
        0.0009621097966915796,
        0.0010586988551551607,
        0.0008532929973946807,
        0.0007592856290288629,
        0.0006522457278840908,
        0.0005350698764559804,
        0.0004007556744895193
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00034246376584100063,
        "first_to_last_ratio": 0.5835734049551724,
        "imbalance_abs_log_ratio": 0.5385850340449921,
        "last_quartile_mean": 0.0005868392269646134,
        "mean": 0.0005826425617355658,
        "normalized": [
          0.8600399104963966,
          0.6303305565643863,
          0.4238686160094385,
          0.43686801642402323,
          0.4650351290899624,
          0.6074411462370523,
          0.8697374223387168,
          1.1200393267518036,
          1.6249558179475279,
          1.6512865002955899,
          1.8170640538197664,
          1.4645222533226991,
          1.3031757013547307,
          1.1194611769198464,
          0.9183501371100032,
          0.6878242353180568
        ],
        "population_cv": 0.44619098566843096
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.060730163007974625,
        0.09550311416387558,
        0.10425227880477905,
        0.15394315123558044,
        0.15637855231761932,
        0.1696780025959015,
        0.1697961539030075,
        0.17087572813034058,
        0.17037443816661835,
        0.1706146001815796,
        0.16992402076721191,
        0.17012864351272583,
        0.16921496391296387,
        0.16865167021751404,
        0.1664763242006302,
        0.16138805449008942,
        0.1570158749818802,
        0.14425092935562134,
        0.1424521654844284,
        0.1296483427286148,
        0.13391907513141632,
        0.1270018070936203,
        0.13948261737823486,
        0.13754454255104065,
        0.15248820185661316,
        0.1527755856513977,
        0.17248840630054474,
        0.17395897209644318,
        0.20097237825393677,
        0.2011977583169937,
        0.2573379576206207,
        0.25867754220962524
      ],
      "loss_nats": 1.2954336404800415,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 8000,
      "stream_state_rms_by_sublayer": [
        0.09550311416387558,
        0.10425227880477905,
        0.15394315123558044,
        0.15637855231761932,
        0.1696780025959015,
        0.1697961539030075,
        0.17087572813034058,
        0.17037443816661835,
        0.1706146001815796,
        0.16992402076721191,
        0.17012864351272583,
        0.16921496391296387,
        0.16865167021751404,
        0.1664763242006302,
        0.16138805449008942,
        0.1570158749818802,
        0.14425092935562134,
        0.1424521654844284,
        0.1296483427286148,
        0.13391907513141632,
        0.1270018070936203,
        0.13948261737823486,
        0.13754454255104065,
        0.15248820185661316,
        0.1527755856513977,
        0.17248840630054474,
        0.17395897209644318,
        0.20097237825393677,
        0.2011977583169937,
        0.2573379576206207,
        0.25867754220962524,
        0.4688045382499695
      ]
    }
  ],
  "environment": {
    "autocast": "cuda-bfloat16-forward-fp32-cross-entropy",
    "compile": false,
    "compute_capability": [
      12,
      0
    ],
    "cublas_workspace_config": ":4096:8",
    "cuda": "12.8",
    "deterministic_algorithms": true,
    "gpu": "NVIDIA GeForce RTX 5090",
    "python": "3.10.14",
    "torch": "2.11.0+cu128"
  },
  "evaluations": [
    {
      "bits_per_byte": 8.076786148009306,
      "cross_entropy_nats": 5.5984015464782715,
      "step": 0
    },
    {
      "bits_per_byte": 5.114718375266125,
      "cross_entropy_nats": 3.5452526211738586,
      "step": 100
    },
    {
      "bits_per_byte": 3.1628078576350735,
      "cross_entropy_nats": 2.192291349172592,
      "step": 500
    },
    {
      "bits_per_byte": 1.9765894690862489,
      "cross_entropy_nats": 1.3700674176216125,
      "step": 2000
    },
    {
      "bits_per_byte": 1.840425322451167,
      "cross_entropy_nats": 1.2756856232881546,
      "step": 4000
    },
    {
      "bits_per_byte": 1.7488000388579794,
      "cross_entropy_nats": 1.2121758162975311,
      "step": 8000
    }
  ],
  "gradient_gate": null,
  "hashes": {
    "final_mixer_parameters": null,
    "final_model_state": "8d20d6d3e99824ff7e0090ba966238e9ab3b4820635e5965124d6da2dc740323",
    "final_optimizer_state": "e9f9db84e99e09e299da28a6a358f3b99d039dde4de0eceec7d22aff7014508e",
    "final_public_parameters": "8d20d6d3e99824ff7e0090ba966238e9ab3b4820635e5965124d6da2dc740323",
    "initial_mixer_parameters": null,
    "initial_public_parameter_elements": 9541824,
    "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe",
    "initial_public_parameter_tensors": 115,
    "initial_public_parameters": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1"
  },
  "manifest": {
    "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716",
    "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371",
    "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
    "input_gate_tensor_hashes": {
      "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434",
      "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94",
      "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc"
    },
    "path": "experiments/k3/attnres_gradient/manifest.json",
    "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338"
  },
  "model": {
    "attnres_aggregation_groups": 8,
    "context": 256,
    "d_ff": 768,
    "d_head": 32,
    "d_model": 192,
    "heads": 6,
    "layers": 16,
    "parameters": {
      "core": 9541824,
      "embedding": 98304,
      "mixer": 0,
      "total": 9541824
    },
    "sublayers": 32,
    "sublayers_per_attnres_group": 4,
    "transformer_blocks_per_attnres_group": 2,
    "vocabulary": 256
  },
  "optimizer": {
    "betas": [
      0.9,
      0.95
    ],
    "epsilon": 1e-08,
    "grad_clip": 1.0,
    "min_lr": 3e-05,
    "name": "AdamW",
    "peak_lr": 0.0003,
    "warmup_steps": 400,
    "weight_decay_ndim_ge_2": 0.1
  },
  "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
  "run_kind": "formal",
  "schema_version": 1,
  "seed": 2026073001,
  "steps": 8000,
  "target_bytes_seen": 65536000,
  "timing": {
    "mean_ms": 21.257796359896474,
    "measured_steps": 7980,
    "median_ms": 21.201465002377518,
    "p95_ms": 21.46604766312521,
    "peak_allocated_bytes": 3036005376,
    "peak_reserved_bytes": 3282042880,
    "warmup_steps_excluded": 20
  },
  "training_history": [
    {
      "bits_per_byte": 8.088097790921855,
      "learning_rate": 7.499999999999999e-07,
      "loss_nats": 5.6062421798706055,
      "step": 1,
      "unclipped_grad_norm": 19.475919723510742
    },
    {
      "bits_per_byte": 7.474302716882146,
      "learning_rate": 7.499999999999999e-06,
      "loss_nats": 5.180791854858398,
      "step": 10,
      "unclipped_grad_norm": 12.938376426696777
    },
    {
      "bits_per_byte": 6.789615018295581,
      "learning_rate": 1.4999999999999999e-05,
      "loss_nats": 4.706202507019043,
      "step": 20,
      "unclipped_grad_norm": 4.482712745666504
    },
    {
      "bits_per_byte": 6.558995914517002,
      "learning_rate": 2.2499999999999998e-05,
      "loss_nats": 4.54634952545166,
      "step": 30,
      "unclipped_grad_norm": 2.908937454223633
    },
    {
      "bits_per_byte": 6.417572516920704,
      "learning_rate": 2.9999999999999997e-05,
      "loss_nats": 4.448322296142578,
      "step": 40,
      "unclipped_grad_norm": 2.8319714069366455
    },
    {
      "bits_per_byte": 6.245190178377291,
      "learning_rate": 3.75e-05,
      "loss_nats": 4.328835964202881,
      "step": 50,
      "unclipped_grad_norm": 2.816138982772827
    },
    {
      "bits_per_byte": 5.943838049951856,
      "learning_rate": 4.4999999999999996e-05,
      "loss_nats": 4.119954586029053,
      "step": 60,
      "unclipped_grad_norm": 2.6310009956359863
    },
    {
      "bits_per_byte": 5.699140446373649,
      "learning_rate": 5.2499999999999995e-05,
      "loss_nats": 3.950343132019043,
      "step": 70,
      "unclipped_grad_norm": 2.5801262855529785
    },
    {
      "bits_per_byte": 5.518637431021795,
      "learning_rate": 5.9999999999999995e-05,
      "loss_nats": 3.825227975845337,
      "step": 80,
      "unclipped_grad_norm": 2.382185459136963
    },
    {
      "bits_per_byte": 5.304690325417858,
      "learning_rate": 6.749999999999999e-05,
      "loss_nats": 3.676931142807007,
      "step": 90,
      "unclipped_grad_norm": 2.2607486248016357
    },
    {
      "bits_per_byte": 5.142740110510481,
      "learning_rate": 7.5e-05,
      "loss_nats": 3.564675807952881,
      "step": 100,
      "unclipped_grad_norm": 2.145101547241211
    },
    {
      "bits_per_byte": 4.946587017895412,
      "learning_rate": 8.249999999999999e-05,
      "loss_nats": 3.428712844848633,
      "step": 110,
      "unclipped_grad_norm": 1.954435110092163
    },
    {
      "bits_per_byte": 4.732811894942338,
      "learning_rate": 8.999999999999999e-05,
      "loss_nats": 3.2805352210998535,
      "step": 120,
      "unclipped_grad_norm": 1.7519731521606445
    },
    {
      "bits_per_byte": 4.552989930887901,
      "learning_rate": 9.75e-05,
      "loss_nats": 3.1558921337127686,
      "step": 130,
      "unclipped_grad_norm": 1.5486056804656982
    },
    {
      "bits_per_byte": 4.412413719829753,
      "learning_rate": 0.00010499999999999999,
      "loss_nats": 3.0584521293640137,
      "step": 140,
      "unclipped_grad_norm": 1.3235344886779785
    },
    {
      "bits_per_byte": 4.26477418130067,
      "learning_rate": 0.0001125,
      "loss_nats": 2.956116199493408,
      "step": 150,
      "unclipped_grad_norm": 1.2421116828918457
    },
    {
      "bits_per_byte": 4.169433535003061,
      "learning_rate": 0.00011999999999999999,
      "loss_nats": 2.890031099319458,
      "step": 160,
      "unclipped_grad_norm": 0.9132174849510193
    },
    {
      "bits_per_byte": 4.040492638206386,
      "learning_rate": 0.00012749999999999998,
      "loss_nats": 2.8006560802459717,
      "step": 170,
      "unclipped_grad_norm": 0.901505708694458
    },
    {
      "bits_per_byte": 3.967424777091417,
      "learning_rate": 0.00013499999999999997,
      "loss_nats": 2.750009298324585,
      "step": 180,
      "unclipped_grad_norm": 0.677867591381073
    },
    {
      "bits_per_byte": 3.8797366708266,
      "learning_rate": 0.0001425,
      "loss_nats": 2.6892285346984863,
      "step": 190,
      "unclipped_grad_norm": 0.7871851325035095
    },
    {
      "bits_per_byte": 3.825796032209989,
      "learning_rate": 0.00015,
      "loss_nats": 2.6518397331237793,
      "step": 200,
      "unclipped_grad_norm": 0.6514210104942322
    },
    {
      "bits_per_byte": 3.7775081193619364,
      "learning_rate": 0.0001575,
      "loss_nats": 2.6183691024780273,
      "step": 210,
      "unclipped_grad_norm": 0.6251815557479858
    },
    {
      "bits_per_byte": 3.6875825188093945,
      "learning_rate": 0.00016499999999999997,
      "loss_nats": 2.556037425994873,
      "step": 220,
      "unclipped_grad_norm": 0.7170947790145874
    },
    {
      "bits_per_byte": 3.6778121844138782,
      "learning_rate": 0.0001725,
      "loss_nats": 2.549265146255493,
      "step": 230,
      "unclipped_grad_norm": 0.6953222155570984
    },
    {
      "bits_per_byte": 3.5779222530356467,
      "learning_rate": 0.00017999999999999998,
      "loss_nats": 2.4800267219543457,
      "step": 240,
      "unclipped_grad_norm": 0.6264667510986328
    },
    {
      "bits_per_byte": 3.617201026700902,
      "learning_rate": 0.0001875,
      "loss_nats": 2.5072526931762695,
      "step": 250,
      "unclipped_grad_norm": 0.8498449921607971
    },
    {
      "bits_per_byte": 3.6087914190390125,
      "learning_rate": 0.000195,
      "loss_nats": 2.5014235973358154,
      "step": 260,
      "unclipped_grad_norm": 0.6882923245429993
    },
    {
      "bits_per_byte": 3.6099880743237156,
      "learning_rate": 0.00020249999999999996,
      "loss_nats": 2.5022530555725098,
      "step": 270,
      "unclipped_grad_norm": 0.7451158165931702
    },
    {
      "bits_per_byte": 3.535934064688699,
      "learning_rate": 0.00020999999999999998,
      "loss_nats": 2.450922727584839,
      "step": 280,
      "unclipped_grad_norm": 0.7390120029449463
    },
    {
      "bits_per_byte": 3.452133454263985,
      "learning_rate": 0.00021749999999999997,
      "loss_nats": 2.392836570739746,
      "step": 290,
      "unclipped_grad_norm": 0.601737380027771
    },
    {
      "bits_per_byte": 3.4977253670771122,
      "learning_rate": 0.000225,
      "loss_nats": 2.4244384765625,
      "step": 300,
      "unclipped_grad_norm": 0.8893438577651978
    },
    {
      "bits_per_byte": 3.4549257645833937,
      "learning_rate": 0.00023249999999999996,
      "loss_nats": 2.3947720527648926,
      "step": 310,
      "unclipped_grad_norm": 1.0071029663085938
    },
    {
      "bits_per_byte": 3.428612419001384,
      "learning_rate": 0.00023999999999999998,
      "loss_nats": 2.376533031463623,
      "step": 320,
      "unclipped_grad_norm": 0.7961780428886414
    },
    {
      "bits_per_byte": 3.4202605975101843,
      "learning_rate": 0.0002475,
      "loss_nats": 2.370743989944458,
      "step": 330,
      "unclipped_grad_norm": 1.010308027267456
    },
    {
      "bits_per_byte": 3.507607834160991,
      "learning_rate": 0.00025499999999999996,
      "loss_nats": 2.431288480758667,
      "step": 340,
      "unclipped_grad_norm": 0.7371622920036316
    },
    {
      "bits_per_byte": 3.4475772898773274,
      "learning_rate": 0.0002625,
      "loss_nats": 2.389678478240967,
      "step": 350,
      "unclipped_grad_norm": 0.8701758980751038
    },
    {
      "bits_per_byte": 3.4091412312013003,
      "learning_rate": 0.00026999999999999995,
      "loss_nats": 2.363036632537842,
      "step": 360,
      "unclipped_grad_norm": 0.7921563386917114
    },
    {
      "bits_per_byte": 3.4048547356111953,
      "learning_rate": 0.00027749999999999997,
      "loss_nats": 2.360065460205078,
      "step": 370,
      "unclipped_grad_norm": 0.8983097076416016
    },
    {
      "bits_per_byte": 3.4312740225061367,
      "learning_rate": 0.000285,
      "loss_nats": 2.378377914428711,
      "step": 380,
      "unclipped_grad_norm": 0.6835634708404541
    },
    {
      "bits_per_byte": 3.36810135519121,
      "learning_rate": 0.0002925,
      "loss_nats": 2.334589958190918,
      "step": 390,
      "unclipped_grad_norm": 0.7815158367156982
    },
    {
      "bits_per_byte": 3.32671476215698,
      "learning_rate": 0.0003,
      "loss_nats": 2.3059029579162598,
      "step": 400,
      "unclipped_grad_norm": 0.9281390309333801
    },
    {
      "bits_per_byte": 3.3045228088202467,
      "learning_rate": 0.0002999988466112388,
      "loss_nats": 2.290520668029785,
      "step": 410,
      "unclipped_grad_norm": 0.8773083090782166
    },
    {
      "bits_per_byte": 3.346598020388531,
      "learning_rate": 0.00029999538646466355,
      "loss_nats": 2.3196849822998047,
      "step": 420,
      "unclipped_grad_norm": 0.9256104230880737
    },
    {
      "bits_per_byte": 3.314825601502834,
      "learning_rate": 0.0002999896196193985,
      "loss_nats": 2.2976620197296143,
      "step": 430,
      "unclipped_grad_norm": 0.7776153087615967
    },
    {
      "bits_per_byte": 3.382362156600754,
      "learning_rate": 0.00029998154617398326,
      "loss_nats": 2.3444747924804688,
      "step": 440,
      "unclipped_grad_norm": 1.105894923210144
    },
    {
      "bits_per_byte": 3.28705625079862,
      "learning_rate": 0.0002999711662663707,
      "loss_nats": 2.278413772583008,
      "step": 450,
      "unclipped_grad_norm": 1.029218077659607
    },
    {
      "bits_per_byte": 3.3030692114551545,
      "learning_rate": 0.0002999584800739247,
      "loss_nats": 2.289513111114502,
      "step": 460,
      "unclipped_grad_norm": 0.986072301864624
    },
    {
      "bits_per_byte": 3.2452720338755956,
      "learning_rate": 0.0002999434878134173,
      "loss_nats": 2.249451160430908,
      "step": 470,
      "unclipped_grad_norm": 0.9585361480712891
    },
    {
      "bits_per_byte": 3.221687708997477,
      "learning_rate": 0.000299926189741025,
      "loss_nats": 2.2331037521362305,
      "step": 480,
      "unclipped_grad_norm": 1.0470387935638428
    },
    {
      "bits_per_byte": 3.155786020978659,
      "learning_rate": 0.000299906586152324,
      "loss_nats": 2.1874241828918457,
      "step": 490,
      "unclipped_grad_norm": 1.4241349697113037
    },
    {
      "bits_per_byte": 3.107888164782763,
      "learning_rate": 0.0002998846773822856,
      "loss_nats": 2.154223918914795,
      "step": 500,
      "unclipped_grad_norm": 1.0626088380813599
    },
    {
      "bits_per_byte": 3.1122186879314886,
      "learning_rate": 0.0002998604638052702,
      "loss_nats": 2.1572256088256836,
      "step": 510,
      "unclipped_grad_norm": 0.9446885585784912
    },
    {
      "bits_per_byte": 3.0380233085574626,
      "learning_rate": 0.0002998339458350211,
      "loss_nats": 2.105797290802002,
      "step": 520,
      "unclipped_grad_norm": 1.1227535009384155
    },
    {
      "bits_per_byte": 3.076978410873794,
      "learning_rate": 0.0002998051239246572,
      "loss_nats": 2.132798910140991,
      "step": 530,
      "unclipped_grad_norm": 0.9776996374130249
    },
    {
      "bits_per_byte": 3.01776100056341,
      "learning_rate": 0.00029977399856666537,
      "loss_nats": 2.091752529144287,
      "step": 540,
      "unclipped_grad_norm": 1.1720482110977173
    },
    {
      "bits_per_byte": 2.99280700585382,
      "learning_rate": 0.0002997405702928922,
      "loss_nats": 2.074455738067627,
      "step": 550,
      "unclipped_grad_norm": 1.1986867189407349
    },
    {
      "bits_per_byte": 2.977761275625738,
      "learning_rate": 0.0002997048396745345,
      "loss_nats": 2.0640268325805664,
      "step": 560,
      "unclipped_grad_norm": 1.1189862489700317
    },
    {
      "bits_per_byte": 2.9082912915668184,
      "learning_rate": 0.0002996668073221302,
      "loss_nats": 2.015873908996582,
      "step": 570,
      "unclipped_grad_norm": 0.9214995503425598
    },
    {
      "bits_per_byte": 2.96765970264466,
      "learning_rate": 0.000299626473885547,
      "loss_nats": 2.0570249557495117,
      "step": 580,
      "unclipped_grad_norm": 0.9309345483779907
    },
    {
      "bits_per_byte": 2.863993376218783,
      "learning_rate": 0.00029958384005397226,
      "loss_nats": 1.9851689338684082,
      "step": 590,
      "unclipped_grad_norm": 0.9733412861824036
    },
    {
      "bits_per_byte": 2.9319891249945065,
      "learning_rate": 0.0002995389065559004,
      "loss_nats": 2.0322999954223633,
      "step": 600,
      "unclipped_grad_norm": 1.4851497411727905
    },
    {
      "bits_per_byte": 2.842428815592397,
      "learning_rate": 0.000299491674159121,
      "loss_nats": 1.9702215194702148,
      "step": 610,
      "unclipped_grad_norm": 1.0783556699752808
    },
    {
      "bits_per_byte": 2.8677622040297908,
      "learning_rate": 0.00029944214367070535,
      "loss_nats": 1.987781286239624,
      "step": 620,
      "unclipped_grad_norm": 0.9511895179748535
    },
    {
      "bits_per_byte": 2.802268974806772,
      "learning_rate": 0.00029939031593699284,
      "loss_nats": 1.9423848390579224,
      "step": 630,
      "unclipped_grad_norm": 1.0907046794891357
    },
    {
      "bits_per_byte": 2.773638163004386,
      "learning_rate": 0.00029933619184357654,
      "loss_nats": 1.922539472579956,
      "step": 640,
      "unclipped_grad_norm": 1.5626765489578247
    },
    {
      "bits_per_byte": 2.718988439918267,
      "learning_rate": 0.00029927977231528774,
      "loss_nats": 1.8846591711044312,
      "step": 650,
      "unclipped_grad_norm": 1.6085642576217651
    },
    {
      "bits_per_byte": 2.628252113149558,
      "learning_rate": 0.00029922105831618057,
      "loss_nats": 1.8217655420303345,
      "step": 660,
      "unclipped_grad_norm": 1.223048448562622
    },
    {
      "bits_per_byte": 2.676602111734572,
      "learning_rate": 0.0002991600508495153,
      "loss_nats": 1.8552792072296143,
      "step": 670,
      "unclipped_grad_norm": 1.3098845481872559
    },
    {
      "bits_per_byte": 2.620622790774633,
      "learning_rate": 0.00029909675095774114,
      "loss_nats": 1.8164772987365723,
      "step": 680,
      "unclipped_grad_norm": 1.1348049640655518
    },
    {
      "bits_per_byte": 2.6956237368853,
      "learning_rate": 0.0002990311597224787,
      "loss_nats": 1.8684639930725098,
      "step": 690,
      "unclipped_grad_norm": 0.956748902797699
    },
    {
      "bits_per_byte": 2.6070459643675705,
      "learning_rate": 0.00029896327826450103,
      "loss_nats": 1.807066559791565,
      "step": 700,
      "unclipped_grad_norm": 1.1346901655197144
    },
    {
      "bits_per_byte": 2.6253973731278863,
      "learning_rate": 0.0002988931077437152,
      "loss_nats": 1.819786787033081,
      "step": 710,
      "unclipped_grad_norm": 1.3684734106063843
    },
    {
      "bits_per_byte": 2.6636225810739558,
      "learning_rate": 0.00029882064935914164,
      "loss_nats": 1.8462824821472168,
      "step": 720,
      "unclipped_grad_norm": 1.6031205654144287
    },
    {
      "bits_per_byte": 2.6068867084328713,
      "learning_rate": 0.0002987459043488943,
      "loss_nats": 1.806956171989441,
      "step": 730,
      "unclipped_grad_norm": 1.1380469799041748
    },
    {
      "bits_per_byte": 2.5424343182128744,
      "learning_rate": 0.0002986688739901591,
      "loss_nats": 1.7622811794281006,
      "step": 740,
      "unclipped_grad_norm": 1.2064875364303589
    },
    {
      "bits_per_byte": 2.5150987077541362,
      "learning_rate": 0.0002985895595991723,
      "loss_nats": 1.7433335781097412,
      "step": 750,
      "unclipped_grad_norm": 1.2007335424423218
    },
    {
      "bits_per_byte": 2.45541642833847,
      "learning_rate": 0.00029850796253119824,
      "loss_nats": 1.7019649744033813,
      "step": 760,
      "unclipped_grad_norm": 1.2129080295562744
    },
    {
      "bits_per_byte": 2.6149697210407723,
      "learning_rate": 0.00029842408418050544,
      "loss_nats": 1.812558889389038,
      "step": 770,
      "unclipped_grad_norm": 1.221384048461914
    },
    {
      "bits_per_byte": 2.484800524151635,
      "learning_rate": 0.0002983379259803436,
      "loss_nats": 1.72233247756958,
      "step": 780,
      "unclipped_grad_norm": 1.1399351358413696
    },
    {
      "bits_per_byte": 2.4467844471090148,
      "learning_rate": 0.0002982494894029186,
      "loss_nats": 1.695981740951538,
      "step": 790,
      "unclipped_grad_norm": 1.4305745363235474
    },
    {
      "bits_per_byte": 2.3889862376335507,
      "learning_rate": 0.0002981587759593675,
      "loss_nats": 1.655919075012207,
      "step": 800,
      "unclipped_grad_norm": 0.9679463505744934
    },
    {
      "bits_per_byte": 2.474951249719373,
      "learning_rate": 0.00029806578719973283,
      "loss_nats": 1.7155054807662964,
      "step": 810,
      "unclipped_grad_norm": 1.4494884014129639
    },
    {
      "bits_per_byte": 2.4123399337768623,
      "learning_rate": 0.00029797052471293596,
      "loss_nats": 1.6721066236495972,
      "step": 820,
      "unclipped_grad_norm": 1.4326691627502441
    },
    {
      "bits_per_byte": 2.4471743317785206,
      "learning_rate": 0.0002978729901267499,
      "loss_nats": 1.6962519884109497,
      "step": 830,
      "unclipped_grad_norm": 1.0609163045883179
    },
    {
      "bits_per_byte": 2.528181256022619,
      "learning_rate": 0.0002977731851077716,
      "loss_nats": 1.7524017095565796,
      "step": 840,
      "unclipped_grad_norm": 1.2206887006759644
    },
    {
      "bits_per_byte": 2.4185156587866947,
      "learning_rate": 0.0002976711113613935,
      "loss_nats": 1.6763873100280762,
      "step": 850,
      "unclipped_grad_norm": 1.1635538339614868
    },
    {
      "bits_per_byte": 2.438352137737211,
      "learning_rate": 0.0002975667706317743,
      "loss_nats": 1.6901369094848633,
      "step": 860,
      "unclipped_grad_norm": 1.1800516843795776
    },
    {
      "bits_per_byte": 2.364430554743359,
      "learning_rate": 0.00029746016470180915,
      "loss_nats": 1.6388983726501465,
      "step": 870,
      "unclipped_grad_norm": 1.172108769416809
    },
    {
      "bits_per_byte": 2.3856504621374155,
      "learning_rate": 0.00029735129539309926,
      "loss_nats": 1.65360689163208,
      "step": 880,
      "unclipped_grad_norm": 1.2126058340072632
    },
    {
      "bits_per_byte": 2.2982358082874708,
      "learning_rate": 0.0002972401645659207,
      "loss_nats": 1.5930156707763672,
      "step": 890,
      "unclipped_grad_norm": 0.9879708886146545
    },
    {
      "bits_per_byte": 2.330757727565628,
      "learning_rate": 0.00029712677411919283,
      "loss_nats": 1.61555814743042,
      "step": 900,
      "unclipped_grad_norm": 1.5637768507003784
    },
    {
      "bits_per_byte": 2.3422774694857202,
      "learning_rate": 0.0002970111259904454,
      "loss_nats": 1.6235430240631104,
      "step": 910,
      "unclipped_grad_norm": 1.2122199535369873
    },
    {
      "bits_per_byte": 2.3285455147275793,
      "learning_rate": 0.0002968932221557859,
      "loss_nats": 1.6140247583389282,
      "step": 920,
      "unclipped_grad_norm": 1.228180170059204
    },
    {
      "bits_per_byte": 2.3433181365060913,
      "learning_rate": 0.0002967730646298655,
      "loss_nats": 1.6242643594741821,
      "step": 930,
      "unclipped_grad_norm": 1.113643765449524
    },
    {
      "bits_per_byte": 2.3376019491393643,
      "learning_rate": 0.0002966506554658449,
      "loss_nats": 1.6203022003173828,
      "step": 940,
      "unclipped_grad_norm": 1.0455681085586548
    },
    {
      "bits_per_byte": 2.3211645353005004,
      "learning_rate": 0.00029652599675535876,
      "loss_nats": 1.6089086532592773,
      "step": 950,
      "unclipped_grad_norm": 1.2410560846328735
    },
    {
      "bits_per_byte": 2.362830772125033,
      "learning_rate": 0.0002963990906284808,
      "loss_nats": 1.6377894878387451,
      "step": 960,
      "unclipped_grad_norm": 1.1456577777862549
    },
    {
      "bits_per_byte": 2.268770708645749,
      "learning_rate": 0.0002962699392536863,
      "loss_nats": 1.57259202003479,
      "step": 970,
      "unclipped_grad_norm": 1.1831417083740234
    },
    {
      "bits_per_byte": 2.35130105521119,
      "learning_rate": 0.0002961385448378161,
      "loss_nats": 1.6297976970672607,
      "step": 980,
      "unclipped_grad_norm": 1.1095668077468872
    },
    {
      "bits_per_byte": 2.3073481370607856,
      "learning_rate": 0.00029600490962603817,
      "loss_nats": 1.5993318557739258,
      "step": 990,
      "unclipped_grad_norm": 1.1233346462249756
    },
    {
      "bits_per_byte": 2.2862505093314924,
      "learning_rate": 0.00029586903590180956,
      "loss_nats": 1.5847080945968628,
      "step": 1000,
      "unclipped_grad_norm": 1.2164878845214844
    },
    {
      "bits_per_byte": 2.265617922690132,
      "learning_rate": 0.00029573092598683735,
      "loss_nats": 1.5704066753387451,
      "step": 1010,
      "unclipped_grad_norm": 1.195701241493225
    },
    {
      "bits_per_byte": 2.236535656429231,
      "learning_rate": 0.00029559058224103895,
      "loss_nats": 1.550248384475708,
      "step": 1020,
      "unclipped_grad_norm": 1.3137857913970947
    },
    {
      "bits_per_byte": 2.2352426908600447,
      "learning_rate": 0.0002954480070625017,
      "loss_nats": 1.5493521690368652,
      "step": 1030,
      "unclipped_grad_norm": 1.0306543111801147
    },
    {
      "bits_per_byte": 2.288523947993248,
      "learning_rate": 0.00029530320288744205,
      "loss_nats": 1.5862839221954346,
      "step": 1040,
      "unclipped_grad_norm": 1.111948847770691
    },
    {
      "bits_per_byte": 2.267572505517188,
      "learning_rate": 0.0002951561721901638,
      "loss_nats": 1.5717614889144897,
      "step": 1050,
      "unclipped_grad_norm": 1.4793521165847778
    },
    {
      "bits_per_byte": 2.1784870402141108,
      "learning_rate": 0.00029500691748301594,
      "loss_nats": 1.510012149810791,
      "step": 1060,
      "unclipped_grad_norm": 1.2200031280517578
    },
    {
      "bits_per_byte": 2.2691612812458586,
      "learning_rate": 0.00029485544131634963,
      "loss_nats": 1.5728627443313599,
      "step": 1070,
      "unclipped_grad_norm": 1.243715524673462
    },
    {
      "bits_per_byte": 2.2409759045092073,
      "learning_rate": 0.0002947017462784745,
      "loss_nats": 1.55332612991333,
      "step": 1080,
      "unclipped_grad_norm": 1.1007108688354492
    },
    {
      "bits_per_byte": 2.2116298168618824,
      "learning_rate": 0.00029454583499561476,
      "loss_nats": 1.532984972000122,
      "step": 1090,
      "unclipped_grad_norm": 1.183898687362671
    },
    {
      "bits_per_byte": 2.265987857372138,
      "learning_rate": 0.00029438771013186413,
      "loss_nats": 1.5706630945205688,
      "step": 1100,
      "unclipped_grad_norm": 0.9869716167449951
    },
    {
      "bits_per_byte": 2.1294642454989217,
      "learning_rate": 0.0002942273743891401,
      "loss_nats": 1.4760321378707886,
      "step": 1110,
      "unclipped_grad_norm": 1.0796256065368652
    },
    {
      "bits_per_byte": 2.1685739602183784,
      "learning_rate": 0.00029406483050713824,
      "loss_nats": 1.503140926361084,
      "step": 1120,
      "unclipped_grad_norm": 1.0218125581741333
    },
    {
      "bits_per_byte": 2.111904300897876,
      "learning_rate": 0.00029390008126328504,
      "loss_nats": 1.4638605117797852,
      "step": 1130,
      "unclipped_grad_norm": 1.0956017971038818
    },
    {
      "bits_per_byte": 2.24492118652,
      "learning_rate": 0.00029373312947269045,
      "loss_nats": 1.556060791015625,
      "step": 1140,
      "unclipped_grad_norm": 1.0603337287902832
    },
    {
      "bits_per_byte": 2.107949387858635,
      "learning_rate": 0.00029356397798809995,
      "loss_nats": 1.4611191749572754,
      "step": 1150,
      "unclipped_grad_norm": 1.118997573852539
    },
    {
      "bits_per_byte": 2.133223098316179,
      "learning_rate": 0.0002933926296998457,
      "loss_nats": 1.4786375761032104,
      "step": 1160,
      "unclipped_grad_norm": 1.0214279890060425
    },
    {
      "bits_per_byte": 2.1914696665624422,
      "learning_rate": 0.0002932190875357972,
      "loss_nats": 1.5190110206604004,
      "step": 1170,
      "unclipped_grad_norm": 1.0747911930084229
    },
    {
      "bits_per_byte": 2.168808888519457,
      "learning_rate": 0.00029304335446131126,
      "loss_nats": 1.5033037662506104,
      "step": 1180,
      "unclipped_grad_norm": 1.0325632095336914
    },
    {
      "bits_per_byte": 2.099094345131011,
      "learning_rate": 0.00029286543347918116,
      "loss_nats": 1.4549813270568848,
      "step": 1190,
      "unclipped_grad_norm": 1.1188828945159912
    },
    {
      "bits_per_byte": 2.0751268429067697,
      "learning_rate": 0.0002926853276295856,
      "loss_nats": 1.438368320465088,
      "step": 1200,
      "unclipped_grad_norm": 1.219267725944519
    },
    {
      "bits_per_byte": 2.1310088216863203,
      "learning_rate": 0.0002925030399900368,
      "loss_nats": 1.4771027565002441,
      "step": 1210,
      "unclipped_grad_norm": 1.0049350261688232
    },
    {
      "bits_per_byte": 2.151060622915717,
      "learning_rate": 0.00029231857367532726,
      "loss_nats": 1.4910016059875488,
      "step": 1220,
      "unclipped_grad_norm": 1.0629826784133911
    },
    {
      "bits_per_byte": 2.098358087402667,
      "learning_rate": 0.0002921319318374776,
      "loss_nats": 1.4544709920883179,
      "step": 1230,
      "unclipped_grad_norm": 1.041517734527588
    },
    {
      "bits_per_byte": 2.0484189691236807,
      "learning_rate": 0.00029194311766568174,
      "loss_nats": 1.4198558330535889,
      "step": 1240,
      "unclipped_grad_norm": 1.308692455291748
    },
    {
      "bits_per_byte": 2.1357799643865563,
      "learning_rate": 0.000291752134386253,
      "loss_nats": 1.480409860610962,
      "step": 1250,
      "unclipped_grad_norm": 1.1052577495574951
    },
    {
      "bits_per_byte": 2.115429601275262,
      "learning_rate": 0.00029155898526256856,
      "loss_nats": 1.466304063796997,
      "step": 1260,
      "unclipped_grad_norm": 1.1282013654708862
    },
    {
      "bits_per_byte": 2.1198928950304534,
      "learning_rate": 0.0002913636735950141,
      "loss_nats": 1.469397783279419,
      "step": 1270,
      "unclipped_grad_norm": 1.2154173851013184
    },
    {
      "bits_per_byte": 2.0054133314142284,
      "learning_rate": 0.0002911662027209272,
      "loss_nats": 1.3900465965270996,
      "step": 1280,
      "unclipped_grad_norm": 1.0942060947418213
    },
    {
      "bits_per_byte": 2.1170947373009157,
      "learning_rate": 0.00029096657601454013,
      "loss_nats": 1.4674582481384277,
      "step": 1290,
      "unclipped_grad_norm": 1.0796113014221191
    },
    {
      "bits_per_byte": 2.0596614750105826,
      "learning_rate": 0.00029076479688692266,
      "loss_nats": 1.4276485443115234,
      "step": 1300,
      "unclipped_grad_norm": 1.0578112602233887
    },
    {
      "bits_per_byte": 2.0645378710931466,
      "learning_rate": 0.00029056086878592323,
      "loss_nats": 1.4310286045074463,
      "step": 1310,
      "unclipped_grad_norm": 1.0866352319717407
    },
    {
      "bits_per_byte": 2.0503159377626976,
      "learning_rate": 0.00029035479519611045,
      "loss_nats": 1.421170711517334,
      "step": 1320,
      "unclipped_grad_norm": 0.958570659160614
    },
    {
      "bits_per_byte": 2.123195993822925,
      "learning_rate": 0.0002901465796387134,
      "loss_nats": 1.4716873168945312,
      "step": 1330,
      "unclipped_grad_norm": 1.0188450813293457
    },
    {
      "bits_per_byte": 2.0677759604435915,
      "learning_rate": 0.00028993622567156154,
      "loss_nats": 1.4332730770111084,
      "step": 1340,
      "unclipped_grad_norm": 1.1106895208358765
    },
    {
      "bits_per_byte": 2.0615133841950732,
      "learning_rate": 0.0002897237368890237,
      "loss_nats": 1.4289321899414062,
      "step": 1350,
      "unclipped_grad_norm": 1.2472416162490845
    },
    {
      "bits_per_byte": 2.122740239798138,
      "learning_rate": 0.0002895091169219469,
      "loss_nats": 1.4713714122772217,
      "step": 1360,
      "unclipped_grad_norm": 1.2623285055160522
    },
    {
      "bits_per_byte": 2.1191595610071747,
      "learning_rate": 0.00028929236943759425,
      "loss_nats": 1.4688894748687744,
      "step": 1370,
      "unclipped_grad_norm": 0.9515148401260376
    },
    {
      "bits_per_byte": 2.0626532852049917,
      "learning_rate": 0.00028907349813958225,
      "loss_nats": 1.4297223091125488,
      "step": 1380,
      "unclipped_grad_norm": 0.9549561142921448
    },
    {
      "bits_per_byte": 2.085573929033431,
      "learning_rate": 0.0002888525067678175,
      "loss_nats": 1.44560968875885,
      "step": 1390,
      "unclipped_grad_norm": 1.1485545635223389
    },
    {
      "bits_per_byte": 2.0049195692236013,
      "learning_rate": 0.00028862939909843273,
      "loss_nats": 1.3897043466567993,
      "step": 1400,
      "unclipped_grad_norm": 0.9498034715652466
    },
    {
      "bits_per_byte": 2.044304112219137,
      "learning_rate": 0.0002884041789437225,
      "loss_nats": 1.4170036315917969,
      "step": 1410,
      "unclipped_grad_norm": 1.0090101957321167
    },
    {
      "bits_per_byte": 2.0692653302000634,
      "learning_rate": 0.0002881768501520778,
      "loss_nats": 1.4343054294586182,
      "step": 1420,
      "unclipped_grad_norm": 1.0467784404754639
    },
    {
      "bits_per_byte": 2.1090376940732947,
      "learning_rate": 0.0002879474166079205,
      "loss_nats": 1.4618735313415527,
      "step": 1430,
      "unclipped_grad_norm": 1.0122960805892944
    },
    {
      "bits_per_byte": 2.054903402991812,
      "learning_rate": 0.00028771588223163694,
      "loss_nats": 1.4243505001068115,
      "step": 1440,
      "unclipped_grad_norm": 1.1181519031524658
    },
    {
      "bits_per_byte": 1.9969074134678557,
      "learning_rate": 0.00028748225097951054,
      "loss_nats": 1.384150743484497,
      "step": 1450,
      "unclipped_grad_norm": 0.900530219078064
    },
    {
      "bits_per_byte": 2.037356701054883,
      "learning_rate": 0.000287246526843655,
      "loss_nats": 1.4121880531311035,
      "step": 1460,
      "unclipped_grad_norm": 1.1993578672409058
    },
    {
      "bits_per_byte": 1.9743343465768104,
      "learning_rate": 0.00028700871385194533,
      "loss_nats": 1.368504285812378,
      "step": 1470,
      "unclipped_grad_norm": 0.9825910329818726
    },
    {
      "bits_per_byte": 2.0340613414817006,
      "learning_rate": 0.00028676881606794945,
      "loss_nats": 1.409903883934021,
      "step": 1480,
      "unclipped_grad_norm": 0.9332759380340576
    },
    {
      "bits_per_byte": 2.041156141777744,
      "learning_rate": 0.00028652683759085854,
      "loss_nats": 1.4148216247558594,
      "step": 1490,
      "unclipped_grad_norm": 0.9170629978179932
    },
    {
      "bits_per_byte": 2.0456156519146167,
      "learning_rate": 0.0002862827825554171,
      "loss_nats": 1.4179127216339111,
      "step": 1500,
      "unclipped_grad_norm": 1.007337212562561
    },
    {
      "bits_per_byte": 2.0521109206897528,
      "learning_rate": 0.00028603665513185245,
      "loss_nats": 1.4224148988723755,
      "step": 1510,
      "unclipped_grad_norm": 0.9886259436607361
    },
    {
      "bits_per_byte": 2.0030100458510716,
      "learning_rate": 0.000285788459525803,
      "loss_nats": 1.388380765914917,
      "step": 1520,
      "unclipped_grad_norm": 0.9661282896995544
    },
    {
      "bits_per_byte": 2.0761377569285413,
      "learning_rate": 0.00028553819997824707,
      "loss_nats": 1.4390690326690674,
      "step": 1530,
      "unclipped_grad_norm": 0.9895579218864441
    },
    {
      "bits_per_byte": 2.0208730238829373,
      "learning_rate": 0.00028528588076542964,
      "loss_nats": 1.4007624387741089,
      "step": 1540,
      "unclipped_grad_norm": 0.9900161623954773
    },
    {
      "bits_per_byte": 2.0817916865756563,
      "learning_rate": 0.00028503150619878996,
      "loss_nats": 1.4429880380630493,
      "step": 1550,
      "unclipped_grad_norm": 1.0090205669403076
    },
    {
      "bits_per_byte": 1.98651725359863,
      "learning_rate": 0.0002847750806248876,
      "loss_nats": 1.3769488334655762,
      "step": 1560,
      "unclipped_grad_norm": 0.9516891837120056
    },
    {
      "bits_per_byte": 2.0695575287238794,
      "learning_rate": 0.000284516608425328,
      "loss_nats": 1.434507966041565,
      "step": 1570,
      "unclipped_grad_norm": 0.897994339466095
    },
    {
      "bits_per_byte": 2.0129038637899073,
      "learning_rate": 0.000284256094016688,
      "loss_nats": 1.3952386379241943,
      "step": 1580,
      "unclipped_grad_norm": 0.9900956749916077
    },
    {
      "bits_per_byte": 2.0172801343237623,
      "learning_rate": 0.0002839935418504399,
      "loss_nats": 1.3982720375061035,
      "step": 1590,
      "unclipped_grad_norm": 0.9180708527565002
    },
    {
      "bits_per_byte": 2.0575405849601425,
      "learning_rate": 0.000283728956412876,
      "loss_nats": 1.4261784553527832,
      "step": 1600,
      "unclipped_grad_norm": 1.0603415966033936
    },
    {
      "bits_per_byte": 1.9872597027024046,
      "learning_rate": 0.00028346234222503136,
      "loss_nats": 1.377463459968567,
      "step": 1610,
      "unclipped_grad_norm": 1.0459290742874146
    },
    {
      "bits_per_byte": 1.943624092542842,
      "learning_rate": 0.00028319370384260675,
      "loss_nats": 1.3472175598144531,
      "step": 1620,
      "unclipped_grad_norm": 0.9182554483413696
    },
    {
      "bits_per_byte": 1.9757394448343595,
      "learning_rate": 0.00028292304585589105,
      "loss_nats": 1.3694782257080078,
      "step": 1630,
      "unclipped_grad_norm": 0.9694951772689819
    },
    {
      "bits_per_byte": 2.015004803852847,
      "learning_rate": 0.0002826503728896824,
      "loss_nats": 1.3966948986053467,
      "step": 1640,
      "unclipped_grad_norm": 0.9169139266014099
    },
    {
      "bits_per_byte": 2.0755808771050477,
      "learning_rate": 0.0002823756896032095,
      "loss_nats": 1.438683032989502,
      "step": 1650,
      "unclipped_grad_norm": 0.9065545797348022
    },
    {
      "bits_per_byte": 1.9985508796795006,
      "learning_rate": 0.0002820990006900518,
      "loss_nats": 1.3852899074554443,
      "step": 1660,
      "unclipped_grad_norm": 0.9444847702980042
    },
    {
      "bits_per_byte": 2.017943299425489,
      "learning_rate": 0.00028182031087805947,
      "loss_nats": 1.3987317085266113,
      "step": 1670,
      "unclipped_grad_norm": 0.9302135109901428
    },
    {
      "bits_per_byte": 2.035893988609295,
      "learning_rate": 0.0002815396249292725,
      "loss_nats": 1.4111741781234741,
      "step": 1680,
      "unclipped_grad_norm": 0.9957410097122192
    },
    {
      "bits_per_byte": 2.1006239588277844,
      "learning_rate": 0.0002812569476398391,
      "loss_nats": 1.4560415744781494,
      "step": 1690,
      "unclipped_grad_norm": 1.046242117881775
    },
    {
      "bits_per_byte": 1.9860793857695331,
      "learning_rate": 0.00028097228383993446,
      "loss_nats": 1.3766453266143799,
      "step": 1700,
      "unclipped_grad_norm": 0.9597991108894348
    },
    {
      "bits_per_byte": 1.902240595196328,
      "learning_rate": 0.0002806856383936774,
      "loss_nats": 1.3185327053070068,
      "step": 1710,
      "unclipped_grad_norm": 0.8989817500114441
    },
    {
      "bits_per_byte": 1.90692127502221,
      "learning_rate": 0.00028039701619904774,
      "loss_nats": 1.321777105331421,
      "step": 1720,
      "unclipped_grad_norm": 0.8322402238845825
    },
    {
      "bits_per_byte": 1.8988062736412483,
      "learning_rate": 0.0002801064221878024,
      "loss_nats": 1.3161522150039673,
      "step": 1730,
      "unclipped_grad_norm": 0.9716639518737793
    },
    {
      "bits_per_byte": 1.9978667326943684,
      "learning_rate": 0.00027981386132539134,
      "loss_nats": 1.3848156929016113,
      "step": 1740,
      "unclipped_grad_norm": 0.8983874917030334
    },
    {
      "bits_per_byte": 2.0305991587371817,
      "learning_rate": 0.0002795193386108724,
      "loss_nats": 1.4075040817260742,
      "step": 1750,
      "unclipped_grad_norm": 1.0045808553695679
    },
    {
      "bits_per_byte": 2.0290275812735974,
      "learning_rate": 0.00027922285907682625,
      "loss_nats": 1.4064147472381592,
      "step": 1760,
      "unclipped_grad_norm": 1.0337824821472168
    },
    {
      "bits_per_byte": 2.0294850551248924,
      "learning_rate": 0.00027892442778927005,
      "loss_nats": 1.4067318439483643,
      "step": 1770,
      "unclipped_grad_norm": 0.985150933265686
    },
    {
      "bits_per_byte": 1.8961565369394051,
      "learning_rate": 0.00027862404984757116,
      "loss_nats": 1.3143155574798584,
      "step": 1780,
      "unclipped_grad_norm": 0.9629419445991516
    },
    {
      "bits_per_byte": 1.9830617781774944,
      "learning_rate": 0.0002783217303843597,
      "loss_nats": 1.3745536804199219,
      "step": 1790,
      "unclipped_grad_norm": 0.8722974061965942
    },
    {
      "bits_per_byte": 1.903928776897197,
      "learning_rate": 0.00027801747456544134,
      "loss_nats": 1.3197028636932373,
      "step": 1800,
      "unclipped_grad_norm": 0.8771339058876038
    },
    {
      "bits_per_byte": 1.9579086275582052,
      "learning_rate": 0.00027771128758970843,
      "loss_nats": 1.357118844985962,
      "step": 1810,
      "unclipped_grad_norm": 0.8671345710754395
    },
    {
      "bits_per_byte": 1.9298429507465542,
      "learning_rate": 0.00027740317468905165,
      "loss_nats": 1.3376652002334595,
      "step": 1820,
      "unclipped_grad_norm": 0.9717899560928345
    },
    {
      "bits_per_byte": 1.9258892415858693,
      "learning_rate": 0.0002770931411282704,
      "loss_nats": 1.3349246978759766,
      "step": 1830,
      "unclipped_grad_norm": 0.8506329655647278
    },
    {
      "bits_per_byte": 1.920128166747267,
      "learning_rate": 0.0002767811922049827,
      "loss_nats": 1.3309314250946045,
      "step": 1840,
      "unclipped_grad_norm": 0.9434375166893005
    },
    {
      "bits_per_byte": 1.9401304369732149,
      "learning_rate": 0.00027646733324953503,
      "loss_nats": 1.3447959423065186,
      "step": 1850,
      "unclipped_grad_norm": 0.9371633529663086
    },
    {
      "bits_per_byte": 1.9244834553977166,
      "learning_rate": 0.00027615156962491097,
      "loss_nats": 1.3339502811431885,
      "step": 1860,
      "unclipped_grad_norm": 0.9993670582771301
    },
    {
      "bits_per_byte": 1.923443476307949,
      "learning_rate": 0.0002758339067266396,
      "loss_nats": 1.333229422569275,
      "step": 1870,
      "unclipped_grad_norm": 0.8980090618133545
    },
    {
      "bits_per_byte": 1.9331135448180123,
      "learning_rate": 0.0002755143499827035,
      "loss_nats": 1.3399322032928467,
      "step": 1880,
      "unclipped_grad_norm": 0.8608688116073608
    },
    {
      "bits_per_byte": 1.937305793915444,
      "learning_rate": 0.0002751929048534456,
      "loss_nats": 1.3428380489349365,
      "step": 1890,
      "unclipped_grad_norm": 0.9540092945098877
    },
    {
      "bits_per_byte": 1.9529099518108777,
      "learning_rate": 0.0002748695768314763,
      "loss_nats": 1.3536540269851685,
      "step": 1900,
      "unclipped_grad_norm": 0.9091746211051941
    },
    {
      "bits_per_byte": 1.8773674323326432,
      "learning_rate": 0.00027454437144157947,
      "loss_nats": 1.3012919425964355,
      "step": 1910,
      "unclipped_grad_norm": 0.8313417434692383
    },
    {
      "bits_per_byte": 1.8889088440667445,
      "learning_rate": 0.00027421729424061787,
      "loss_nats": 1.3092918395996094,
      "step": 1920,
      "unclipped_grad_norm": 0.9861888289451599
    },
    {
      "bits_per_byte": 1.9783957168769355,
      "learning_rate": 0.0002738883508174385,
      "loss_nats": 1.3713194131851196,
      "step": 1930,
      "unclipped_grad_norm": 0.9654703140258789
    },
    {
      "bits_per_byte": 1.8474522540936138,
      "learning_rate": 0.0002735575467927768,
      "loss_nats": 1.280556321144104,
      "step": 1940,
      "unclipped_grad_norm": 0.8361129760742188
    },
    {
      "bits_per_byte": 1.9440165569521108,
      "learning_rate": 0.0002732248878191609,
      "loss_nats": 1.347489595413208,
      "step": 1950,
      "unclipped_grad_norm": 0.7723413705825806
    },
    {
      "bits_per_byte": 1.8547983210425678,
      "learning_rate": 0.0002728903795808148,
      "loss_nats": 1.285648226737976,
      "step": 1960,
      "unclipped_grad_norm": 0.8670309782028198
    },
    {
      "bits_per_byte": 1.9366295581322517,
      "learning_rate": 0.0002725540277935614,
      "loss_nats": 1.3423693180084229,
      "step": 1970,
      "unclipped_grad_norm": 0.8607841730117798
    },
    {
      "bits_per_byte": 1.973360924772927,
      "learning_rate": 0.00027221583820472456,
      "loss_nats": 1.3678295612335205,
      "step": 1980,
      "unclipped_grad_norm": 0.8964365124702454
    },
    {
      "bits_per_byte": 1.9723059831925343,
      "learning_rate": 0.0002718758165930314,
      "loss_nats": 1.367098331451416,
      "step": 1990,
      "unclipped_grad_norm": 0.9170846343040466
    },
    {
      "bits_per_byte": 1.931895563684602,
      "learning_rate": 0.0002715339687685131,
      "loss_nats": 1.339087963104248,
      "step": 2000,
      "unclipped_grad_norm": 0.8662557601928711
    },
    {
      "bits_per_byte": 1.9008518352906107,
      "learning_rate": 0.00027119030057240576,
      "loss_nats": 1.3175700902938843,
      "step": 2010,
      "unclipped_grad_norm": 0.8140403628349304
    },
    {
      "bits_per_byte": 1.945781786880567,
      "learning_rate": 0.0002708448178770506,
      "loss_nats": 1.3487131595611572,
      "step": 2020,
      "unclipped_grad_norm": 0.8827610015869141
    },
    {
      "bits_per_byte": 2.023320680970017,
      "learning_rate": 0.00027049752658579376,
      "loss_nats": 1.4024590253829956,
      "step": 2030,
      "unclipped_grad_norm": 0.9354773759841919
    },
    {
      "bits_per_byte": 1.9386018551723463,
      "learning_rate": 0.00027014843263288514,
      "loss_nats": 1.3437364101409912,
      "step": 2040,
      "unclipped_grad_norm": 0.8753958940505981
    },
    {
      "bits_per_byte": 1.9926081911615872,
      "learning_rate": 0.00026979754198337724,
      "loss_nats": 1.3811707496643066,
      "step": 2050,
      "unclipped_grad_norm": 0.8750029802322388
    },
    {
      "bits_per_byte": 1.948962090060322,
      "learning_rate": 0.00026944486063302305,
      "loss_nats": 1.3509175777435303,
      "step": 2060,
      "unclipped_grad_norm": 0.9496069550514221
    },
    {
      "bits_per_byte": 1.9059292790920332,
      "learning_rate": 0.0002690903946081738,
      "loss_nats": 1.321089506149292,
      "step": 2070,
      "unclipped_grad_norm": 0.8713699579238892
    },
    {
      "bits_per_byte": 1.8954827089133246,
      "learning_rate": 0.00026873414996567596,
      "loss_nats": 1.3138484954833984,
      "step": 2080,
      "unclipped_grad_norm": 0.8487783074378967
    },
    {
      "bits_per_byte": 1.892261817827966,
      "learning_rate": 0.00026837613279276736,
      "loss_nats": 1.3116159439086914,
      "step": 2090,
      "unclipped_grad_norm": 0.859647810459137
    },
    {
      "bits_per_byte": 1.8838277886296537,
      "learning_rate": 0.0002680163492069739,
      "loss_nats": 1.305769920349121,
      "step": 2100,
      "unclipped_grad_norm": 0.8237807750701904
    },
    {
      "bits_per_byte": 1.9342706440930173,
      "learning_rate": 0.00026765480535600414,
      "loss_nats": 1.3407342433929443,
      "step": 2110,
      "unclipped_grad_norm": 0.885201632976532
    },
    {
      "bits_per_byte": 1.89223670836094,
      "learning_rate": 0.0002672915074176452,
      "loss_nats": 1.311598539352417,
      "step": 2120,
      "unclipped_grad_norm": 0.8643916249275208
    },
    {
      "bits_per_byte": 1.8591027028283622,
      "learning_rate": 0.00026692646159965624,
      "loss_nats": 1.288631796836853,
      "step": 2130,
      "unclipped_grad_norm": 0.9507161378860474
    },
    {
      "bits_per_byte": 1.991344806608349,
      "learning_rate": 0.0002665596741396633,
      "loss_nats": 1.3802950382232666,
      "step": 2140,
      "unclipped_grad_norm": 0.967012882232666
    },
    {
      "bits_per_byte": 1.8634151677987472,
      "learning_rate": 0.000266191151305052,
      "loss_nats": 1.2916209697723389,
      "step": 2150,
      "unclipped_grad_norm": 0.8769172430038452
    },
    {
      "bits_per_byte": 1.9079485273958137,
      "learning_rate": 0.00026582089939286075,
      "loss_nats": 1.3224891424179077,
      "step": 2160,
      "unclipped_grad_norm": 0.8441570401191711
    },
    {
      "bits_per_byte": 1.982388982047319,
      "learning_rate": 0.0002654489247296732,
      "loss_nats": 1.3740873336791992,
      "step": 2170,
      "unclipped_grad_norm": 0.8798146843910217
    },
    {
      "bits_per_byte": 1.855854810466818,
      "learning_rate": 0.00026507523367151007,
      "loss_nats": 1.2863805294036865,
      "step": 2180,
      "unclipped_grad_norm": 0.9269494414329529
    },
    {
      "bits_per_byte": 1.881138839883414,
      "learning_rate": 0.00026469983260372037,
      "loss_nats": 1.3039060831069946,
      "step": 2190,
      "unclipped_grad_norm": 0.9328587055206299
    },
    {
      "bits_per_byte": 1.909031158182995,
      "learning_rate": 0.0002643227279408727,
      "loss_nats": 1.3232395648956299,
      "step": 2200,
      "unclipped_grad_norm": 0.8771402835845947
    },
    {
      "bits_per_byte": 1.8627256893514381,
      "learning_rate": 0.0002639439261266453,
      "loss_nats": 1.2911430597305298,
      "step": 2210,
      "unclipped_grad_norm": 0.9345882534980774
    },
    {
      "bits_per_byte": 1.85186464098415,
      "learning_rate": 0.000263563433633716,
      "loss_nats": 1.2836147546768188,
      "step": 2220,
      "unclipped_grad_norm": 0.8119871020317078
    },
    {
      "bits_per_byte": 1.929916043373171,
      "learning_rate": 0.00026318125696365156,
      "loss_nats": 1.3377158641815186,
      "step": 2230,
      "unclipped_grad_norm": 0.8476319909095764
    },
    {
      "bits_per_byte": 1.8588350978236197,
      "learning_rate": 0.000262797402646797,
      "loss_nats": 1.288446307182312,
      "step": 2240,
      "unclipped_grad_norm": 0.9140865206718445
    },
    {
      "bits_per_byte": 1.8803798804451561,
      "learning_rate": 0.00026241187724216334,
      "loss_nats": 1.303380012512207,
      "step": 2250,
      "unclipped_grad_norm": 0.891239583492279
    },
    {
      "bits_per_byte": 1.8744335802915746,
      "learning_rate": 0.0002620246873373162,
      "loss_nats": 1.2992583513259888,
      "step": 2260,
      "unclipped_grad_norm": 0.8316894769668579
    },
    {
      "bits_per_byte": 1.8524942694689588,
      "learning_rate": 0.00026163583954826264,
      "loss_nats": 1.2840511798858643,
      "step": 2270,
      "unclipped_grad_norm": 0.8655860424041748
    },
    {
      "bits_per_byte": 1.8715530428722735,
      "learning_rate": 0.0002612453405193386,
      "loss_nats": 1.2972617149353027,
      "step": 2280,
      "unclipped_grad_norm": 0.8277263045310974
    },
    {
      "bits_per_byte": 1.9110787836241676,
      "learning_rate": 0.000260853196923095,
      "loss_nats": 1.3246588706970215,
      "step": 2290,
      "unclipped_grad_norm": 0.9131273031234741
    },
    {
      "bits_per_byte": 1.8344360911283637,
      "learning_rate": 0.0002604594154601839,
      "loss_nats": 1.2715342044830322,
      "step": 2300,
      "unclipped_grad_norm": 0.882969856262207
    },
    {
      "bits_per_byte": 1.851766782855809,
      "learning_rate": 0.0002600640028592441,
      "loss_nats": 1.2835469245910645,
      "step": 2310,
      "unclipped_grad_norm": 0.854379415512085
    },
    {
      "bits_per_byte": 1.8345169229742693,
      "learning_rate": 0.0002596669658767859,
      "loss_nats": 1.271590232849121,
      "step": 2320,
      "unclipped_grad_norm": 0.8358980417251587
    },
    {
      "bits_per_byte": 1.902897396939973,
      "learning_rate": 0.000259268311297076,
      "loss_nats": 1.3189879655838013,
      "step": 2330,
      "unclipped_grad_norm": 0.8988526463508606
    },
    {
      "bits_per_byte": 1.820362922790911,
      "learning_rate": 0.00025886804593202114,
      "loss_nats": 1.2617794275283813,
      "step": 2340,
      "unclipped_grad_norm": 0.8210547566413879
    },
    {
      "bits_per_byte": 1.8788371960669168,
      "learning_rate": 0.00025846617662105193,
      "loss_nats": 1.3023107051849365,
      "step": 2350,
      "unclipped_grad_norm": 0.8696999549865723
    },
    {
      "bits_per_byte": 1.8997241450489033,
      "learning_rate": 0.00025806271023100637,
      "loss_nats": 1.3167884349822998,
      "step": 2360,
      "unclipped_grad_norm": 0.8768331408500671
    },
    {
      "bits_per_byte": 1.9060586100454822,
      "learning_rate": 0.00025765765365601165,
      "loss_nats": 1.3211791515350342,
      "step": 2370,
      "unclipped_grad_norm": 0.8498597741127014
    },
    {
      "bits_per_byte": 1.8700977256806723,
      "learning_rate": 0.00025725101381736707,
      "loss_nats": 1.296252965927124,
      "step": 2380,
      "unclipped_grad_norm": 0.9666873216629028
    },
    {
      "bits_per_byte": 1.9181192374025384,
      "learning_rate": 0.00025684279766342567,
      "loss_nats": 1.3295389413833618,
      "step": 2390,
      "unclipped_grad_norm": 0.8190925717353821
    },
    {
      "bits_per_byte": 1.8665703615114762,
      "learning_rate": 0.00025643301216947504,
      "loss_nats": 1.2938079833984375,
      "step": 2400,
      "unclipped_grad_norm": 0.8776637315750122
    },
    {
      "bits_per_byte": 1.8564054989148806,
      "learning_rate": 0.00025602166433761866,
      "loss_nats": 1.2867622375488281,
      "step": 2410,
      "unclipped_grad_norm": 0.8437303900718689
    },
    {
      "bits_per_byte": 1.8440313471853016,
      "learning_rate": 0.0002556087611966561,
      "loss_nats": 1.2781851291656494,
      "step": 2420,
      "unclipped_grad_norm": 0.8381853103637695
    },
    {
      "bits_per_byte": 1.8694933786455403,
      "learning_rate": 0.00025519430980196277,
      "loss_nats": 1.2958340644836426,
      "step": 2430,
      "unclipped_grad_norm": 0.8620160818099976
    },
    {
      "bits_per_byte": 1.9125165585853807,
      "learning_rate": 0.0002547783172353695,
      "loss_nats": 1.325655460357666,
      "step": 2440,
      "unclipped_grad_norm": 0.8893632292747498
    },
    {
      "bits_per_byte": 1.8473590394968462,
      "learning_rate": 0.0002543607906050415,
      "loss_nats": 1.2804917097091675,
      "step": 2450,
      "unclipped_grad_norm": 0.8294656872749329
    },
    {
      "bits_per_byte": 1.8569149115267363,
      "learning_rate": 0.0002539417370453568,
      "loss_nats": 1.2871153354644775,
      "step": 2460,
      "unclipped_grad_norm": 0.8894950151443481
    },
    {
      "bits_per_byte": 1.8524935815383552,
      "learning_rate": 0.00025352116371678464,
      "loss_nats": 1.284050703048706,
      "step": 2470,
      "unclipped_grad_norm": 0.857410728931427
    },
    {
      "bits_per_byte": 1.8769517502655078,
      "learning_rate": 0.0002530990778057627,
      "loss_nats": 1.3010038137435913,
      "step": 2480,
      "unclipped_grad_norm": 0.8339816331863403
    },
    {
      "bits_per_byte": 1.8457628685141885,
      "learning_rate": 0.00025267548652457475,
      "loss_nats": 1.2793853282928467,
      "step": 2490,
      "unclipped_grad_norm": 0.8252822756767273
    },
    {
      "bits_per_byte": 1.8621884155501427,
      "learning_rate": 0.00025225039711122696,
      "loss_nats": 1.2907706499099731,
      "step": 2500,
      "unclipped_grad_norm": 0.8138288259506226
    },
    {
      "bits_per_byte": 1.815880023013521,
      "learning_rate": 0.00025182381682932464,
      "loss_nats": 1.2586721181869507,
      "step": 2510,
      "unclipped_grad_norm": 0.8689138889312744
    },
    {
      "bits_per_byte": 1.8820847444631592,
      "learning_rate": 0.0002513957529679477,
      "loss_nats": 1.304561734199524,
      "step": 2520,
      "unclipped_grad_norm": 0.8937636613845825
    },
    {
      "bits_per_byte": 1.878350485164975,
      "learning_rate": 0.00025096621284152656,
      "loss_nats": 1.3019733428955078,
      "step": 2530,
      "unclipped_grad_norm": 0.9091646671295166
    },
    {
      "bits_per_byte": 1.8572593927764145,
      "learning_rate": 0.0002505352037897168,
      "loss_nats": 1.2873541116714478,
      "step": 2540,
      "unclipped_grad_norm": 0.8881837129592896
    },
    {
      "bits_per_byte": 1.8423865051124497,
      "learning_rate": 0.0002501027331772739,
      "loss_nats": 1.2770450115203857,
      "step": 2550,
      "unclipped_grad_norm": 1.0689516067504883
    },
    {
      "bits_per_byte": 1.8515612635880279,
      "learning_rate": 0.00024966880839392746,
      "loss_nats": 1.2834044694900513,
      "step": 2560,
      "unclipped_grad_norm": 0.829948365688324
    },
    {
      "bits_per_byte": 1.8581186181001252,
      "learning_rate": 0.0002492334368542548,
      "loss_nats": 1.2879496812820435,
      "step": 2570,
      "unclipped_grad_norm": 0.8880435824394226
    },
    {
      "bits_per_byte": 1.880620656156364,
      "learning_rate": 0.00024879662599755416,
      "loss_nats": 1.3035469055175781,
      "step": 2580,
      "unclipped_grad_norm": 0.9021074175834656
    },
    {
      "bits_per_byte": 1.8318979711669305,
      "learning_rate": 0.00024835838328771803,
      "loss_nats": 1.2697749137878418,
      "step": 2590,
      "unclipped_grad_norm": 0.7873226404190063
    },
    {
      "bits_per_byte": 1.8382764637221303,
      "learning_rate": 0.0002479187162131051,
      "loss_nats": 1.2741961479187012,
      "step": 2600,
      "unclipped_grad_norm": 0.8448187708854675
    },
    {
      "bits_per_byte": 1.7796977969770509,
      "learning_rate": 0.00024747763228641264,
      "loss_nats": 1.2335925102233887,
      "step": 2610,
      "unclipped_grad_norm": 0.8379483222961426
    },
    {
      "bits_per_byte": 1.8338702682070251,
      "learning_rate": 0.00024703513904454795,
      "loss_nats": 1.2711420059204102,
      "step": 2620,
      "unclipped_grad_norm": 0.8167812824249268
    },
    {
      "bits_per_byte": 1.8538771819645465,
      "learning_rate": 0.00024659124404849956,
      "loss_nats": 1.285009741783142,
      "step": 2630,
      "unclipped_grad_norm": 0.8943092226982117
    },
    {
      "bits_per_byte": 1.7228497916476038,
      "learning_rate": 0.00024614595488320833,
      "loss_nats": 1.1941884756088257,
      "step": 2640,
      "unclipped_grad_norm": 0.8208516240119934
    },
    {
      "bits_per_byte": 1.79524330878854,
      "learning_rate": 0.00024569927915743745,
      "loss_nats": 1.2443678379058838,
      "step": 2650,
      "unclipped_grad_norm": 0.8445570468902588
    },
    {
      "bits_per_byte": 1.8489555544448055,
      "learning_rate": 0.0002452512245036427,
      "loss_nats": 1.2815983295440674,
      "step": 2660,
      "unclipped_grad_norm": 0.8848552703857422
    },
    {
      "bits_per_byte": 1.8531913151529056,
      "learning_rate": 0.00024480179857784177,
      "loss_nats": 1.2845343351364136,
      "step": 2670,
      "unclipped_grad_norm": 0.8651367425918579
    },
    {
      "bits_per_byte": 1.762816667898961,
      "learning_rate": 0.00024435100905948387,
      "loss_nats": 1.2218914031982422,
      "step": 2680,
      "unclipped_grad_norm": 0.7578476071357727
    },
    {
      "bits_per_byte": 1.9435059404616992,
      "learning_rate": 0.00024389886365131793,
      "loss_nats": 1.3471356630325317,
      "step": 2690,
      "unclipped_grad_norm": 0.8484299778938293
    },
    {
      "bits_per_byte": 1.8394798263302174,
      "learning_rate": 0.00024344537007926163,
      "loss_nats": 1.275030255317688,
      "step": 2700,
      "unclipped_grad_norm": 0.8363795876502991
    },
    {
      "bits_per_byte": 1.7593267959476528,
      "learning_rate": 0.00024299053609226878,
      "loss_nats": 1.2194724082946777,
      "step": 2710,
      "unclipped_grad_norm": 0.7895286083221436
    },
    {
      "bits_per_byte": 1.7812189835239323,
      "learning_rate": 0.00024253436946219733,
      "loss_nats": 1.2346469163894653,
      "step": 2720,
      "unclipped_grad_norm": 0.8936913013458252
    },
    {
      "bits_per_byte": 1.806379013449256,
      "learning_rate": 0.0002420768779836764,
      "loss_nats": 1.2520865201950073,
      "step": 2730,
      "unclipped_grad_norm": 0.8484287858009338
    },
    {
      "bits_per_byte": 1.8120035340630734,
      "learning_rate": 0.00024161806947397305,
      "loss_nats": 1.255985140800476,
      "step": 2740,
      "unclipped_grad_norm": 0.8566655516624451
    },
    {
      "bits_per_byte": 1.8444568322635362,
      "learning_rate": 0.00024115795177285886,
      "loss_nats": 1.278480052947998,
      "step": 2750,
      "unclipped_grad_norm": 0.8961714506149292
    },
    {
      "bits_per_byte": 1.855966255224577,
      "learning_rate": 0.0002406965327424758,
      "loss_nats": 1.2864577770233154,
      "step": 2760,
      "unclipped_grad_norm": 0.9054936766624451
    },
    {
      "bits_per_byte": 1.7360374213157639,
      "learning_rate": 0.00024023382026720197,
      "loss_nats": 1.2033294439315796,
      "step": 2770,
      "unclipped_grad_norm": 0.8361671566963196
    },
    {
      "bits_per_byte": 1.75377691580431,
      "learning_rate": 0.00023976982225351698,
      "loss_nats": 1.215625524520874,
      "step": 2780,
      "unclipped_grad_norm": 0.7793561220169067
    },
    {
      "bits_per_byte": 1.8377969760915247,
      "learning_rate": 0.0002393045466298666,
      "loss_nats": 1.2738637924194336,
      "step": 2790,
      "unclipped_grad_norm": 0.7846143245697021
    },
    {
      "bits_per_byte": 1.9117343814892567,
      "learning_rate": 0.00023883800134652763,
      "loss_nats": 1.325113296508789,
      "step": 2800,
      "unclipped_grad_norm": 0.872995913028717
    },
    {
      "bits_per_byte": 1.8341784611173713,
      "learning_rate": 0.0002383701943754715,
      "loss_nats": 1.2713556289672852,
      "step": 2810,
      "unclipped_grad_norm": 0.8531777858734131
    },
    {
      "bits_per_byte": 1.8258434939259562,
      "learning_rate": 0.00023790113371022884,
      "loss_nats": 1.265578269958496,
      "step": 2820,
      "unclipped_grad_norm": 0.8563913702964783
    },
    {
      "bits_per_byte": 1.8998995673527832,
      "learning_rate": 0.0002374308273657521,
      "loss_nats": 1.3169100284576416,
      "step": 2830,
      "unclipped_grad_norm": 0.864924430847168
    },
    {
      "bits_per_byte": 1.8091240285396772,
      "learning_rate": 0.00023695928337827911,
      "loss_nats": 1.2539892196655273,
      "step": 2840,
      "unclipped_grad_norm": 0.8685922622680664
    },
    {
      "bits_per_byte": 1.8212324670736735,
      "learning_rate": 0.00023648650980519564,
      "loss_nats": 1.26238214969635,
      "step": 2850,
      "unclipped_grad_norm": 0.8504610061645508
    },
    {
      "bits_per_byte": 1.7804541766755455,
      "learning_rate": 0.00023601251472489757,
      "loss_nats": 1.234116792678833,
      "step": 2860,
      "unclipped_grad_norm": 0.8544119000434875
    },
    {
      "bits_per_byte": 1.8702564656674188,
      "learning_rate": 0.00023553730623665305,
      "loss_nats": 1.2963629961013794,
      "step": 2870,
      "unclipped_grad_norm": 0.855370044708252
    },
    {
      "bits_per_byte": 1.8717095470845586,
      "learning_rate": 0.00023506089246046403,
      "loss_nats": 1.297370195388794,
      "step": 2880,
      "unclipped_grad_norm": 0.8495773673057556
    },
    {
      "bits_per_byte": 1.7666914370229,
      "learning_rate": 0.00023458328153692736,
      "loss_nats": 1.2245771884918213,
      "step": 2890,
      "unclipped_grad_norm": 0.8828970789909363
    },
    {
      "bits_per_byte": 1.7353923143923775,
      "learning_rate": 0.0002341044816270961,
      "loss_nats": 1.2028822898864746,
      "step": 2900,
      "unclipped_grad_norm": 0.8020629286766052
    },
    {
      "bits_per_byte": 1.7995504422967483,
      "learning_rate": 0.00023362450091233953,
      "loss_nats": 1.2473533153533936,
      "step": 2910,
      "unclipped_grad_norm": 0.8652002811431885
    },
    {
      "bits_per_byte": 1.7761088630188457,
      "learning_rate": 0.00023314334759420388,
      "loss_nats": 1.231104850769043,
      "step": 2920,
      "unclipped_grad_norm": 0.8863427042961121
    },
    {
      "bits_per_byte": 1.7940502651395047,
      "learning_rate": 0.0002326610298942717,
      "loss_nats": 1.24354088306427,
      "step": 2930,
      "unclipped_grad_norm": 0.8493190407752991
    },
    {
      "bits_per_byte": 1.8227453984533135,
      "learning_rate": 0.00023217755605402175,
      "loss_nats": 1.2634308338165283,
      "step": 2940,
      "unclipped_grad_norm": 0.9217897653579712
    },
    {
      "bits_per_byte": 1.8002454241888848,
      "learning_rate": 0.00023169293433468802,
      "loss_nats": 1.2478350400924683,
      "step": 2950,
      "unclipped_grad_norm": 0.9446085095405579
    },
    {
      "bits_per_byte": 1.8533994141604497,
      "learning_rate": 0.00023120717301711857,
      "loss_nats": 1.28467857837677,
      "step": 2960,
      "unclipped_grad_norm": 0.9335201382637024
    },
    {
      "bits_per_byte": 1.7928699482066333,
      "learning_rate": 0.00023072028040163408,
      "loss_nats": 1.242722749710083,
      "step": 2970,
      "unclipped_grad_norm": 0.9046677947044373
    },
    {
      "bits_per_byte": 1.784769565350995,
      "learning_rate": 0.00023023226480788593,
      "loss_nats": 1.2371079921722412,
      "step": 2980,
      "unclipped_grad_norm": 0.8317873477935791
    },
    {
      "bits_per_byte": 1.8006910312372704,
      "learning_rate": 0.00022974313457471416,
      "loss_nats": 1.2481439113616943,
      "step": 2990,
      "unclipped_grad_norm": 0.8293972015380859
    },
    {
      "bits_per_byte": 1.8134308180825838,
      "learning_rate": 0.0002292528980600049,
      "loss_nats": 1.256974458694458,
      "step": 3000,
      "unclipped_grad_norm": 0.8042416572570801
    },
    {
      "bits_per_byte": 1.811533849443567,
      "learning_rate": 0.00022876156364054758,
      "loss_nats": 1.255659580230713,
      "step": 3010,
      "unclipped_grad_norm": 0.7810729146003723
    },
    {
      "bits_per_byte": 1.7608942458276164,
      "learning_rate": 0.0002282691397118918,
      "loss_nats": 1.2205588817596436,
      "step": 3020,
      "unclipped_grad_norm": 0.846473753452301
    },
    {
      "bits_per_byte": 1.842709660513421,
      "learning_rate": 0.00022777563468820376,
      "loss_nats": 1.2772690057754517,
      "step": 3030,
      "unclipped_grad_norm": 0.8461940884590149
    },
    {
      "bits_per_byte": 1.775488005649231,
      "learning_rate": 0.00022728105700212282,
      "loss_nats": 1.2306745052337646,
      "step": 3040,
      "unclipped_grad_norm": 0.8409631252288818
    },
    {
      "bits_per_byte": 1.7848693152884954,
      "learning_rate": 0.00022678541510461698,
      "loss_nats": 1.2371771335601807,
      "step": 3050,
      "unclipped_grad_norm": 0.8528071641921997
    },
    {
      "bits_per_byte": 1.8093759831231913,
      "learning_rate": 0.00022628871746483876,
      "loss_nats": 1.2541638612747192,
      "step": 3060,
      "unclipped_grad_norm": 0.9029015898704529
    },
    {
      "bits_per_byte": 1.7287833650850215,
      "learning_rate": 0.0002257909725699804,
      "loss_nats": 1.1983013153076172,
      "step": 3070,
      "unclipped_grad_norm": 0.802097737789154
    },
    {
      "bits_per_byte": 1.755199900257549,
      "learning_rate": 0.00022529218892512886,
      "loss_nats": 1.2166118621826172,
      "step": 3080,
      "unclipped_grad_norm": 0.8510171175003052
    },
    {
      "bits_per_byte": 1.8287252352238135,
      "learning_rate": 0.00022479237505312034,
      "loss_nats": 1.267575740814209,
      "step": 3090,
      "unclipped_grad_norm": 0.9035980105400085
    },
    {
      "bits_per_byte": 1.7486468453117232,
      "learning_rate": 0.00022429153949439496,
      "loss_nats": 1.2120696306228638,
      "step": 3100,
      "unclipped_grad_norm": 0.8420509099960327
    },
    {
      "bits_per_byte": 1.8143894493784931,
      "learning_rate": 0.00022378969080685053,
      "loss_nats": 1.257638931274414,
      "step": 3110,
      "unclipped_grad_norm": 0.8073800206184387
    },
    {
      "bits_per_byte": 1.7807735484581977,
      "learning_rate": 0.0002232868375656965,
      "loss_nats": 1.2343381643295288,
      "step": 3120,
      "unclipped_grad_norm": 0.7948007583618164
    },
    {
      "bits_per_byte": 1.8404088766101783,
      "learning_rate": 0.00022278298836330733,
      "loss_nats": 1.2756742238998413,
      "step": 3130,
      "unclipped_grad_norm": 0.7875926494598389
    },
    {
      "bits_per_byte": 1.8127681689288093,
      "learning_rate": 0.00022227815180907572,
      "loss_nats": 1.2565151453018188,
      "step": 3140,
      "unclipped_grad_norm": 0.8293087482452393
    },
    {
      "bits_per_byte": 1.7580950562021733,
      "learning_rate": 0.00022177233652926546,
      "loss_nats": 1.218618631362915,
      "step": 3150,
      "unclipped_grad_norm": 0.831396222114563
    },
    {
      "bits_per_byte": 1.7956681059361712,
      "learning_rate": 0.00022126555116686416,
      "loss_nats": 1.2446622848510742,
      "step": 3160,
      "unclipped_grad_norm": 0.8769922256469727
    },
    {
      "bits_per_byte": 1.6987359321874784,
      "learning_rate": 0.00022075780438143532,
      "loss_nats": 1.177474021911621,
      "step": 3170,
      "unclipped_grad_norm": 0.8662227988243103
    },
    {
      "bits_per_byte": 1.8090672742648926,
      "learning_rate": 0.00022024910484897065,
      "loss_nats": 1.2539498805999756,
      "step": 3180,
      "unclipped_grad_norm": 0.87493896484375
    },
    {
      "bits_per_byte": 1.77672765659665,
      "learning_rate": 0.00021973946126174158,
      "loss_nats": 1.2315337657928467,
      "step": 3190,
      "unclipped_grad_norm": 0.8155474066734314
    },
    {
      "bits_per_byte": 1.7352984118650063,
      "learning_rate": 0.00021922888232815086,
      "loss_nats": 1.2028172016143799,
      "step": 3200,
      "unclipped_grad_norm": 0.8111788630485535
    },
    {
      "bits_per_byte": 1.7460088034301386,
      "learning_rate": 0.0002187173767725837,
      "loss_nats": 1.2102410793304443,
      "step": 3210,
      "unclipped_grad_norm": 0.8163402080535889
    },
    {
      "bits_per_byte": 1.7462268774314327,
      "learning_rate": 0.0002182049533352588,
      "loss_nats": 1.2103922367095947,
      "step": 3220,
      "unclipped_grad_norm": 0.8070465326309204
    },
    {
      "bits_per_byte": 1.8407530138945547,
      "learning_rate": 0.00021769162077207884,
      "loss_nats": 1.2759127616882324,
      "step": 3230,
      "unclipped_grad_norm": 0.9377122521400452
    },
    {
      "bits_per_byte": 1.717580415207818,
      "learning_rate": 0.00021717738785448102,
      "loss_nats": 1.1905360221862793,
      "step": 3240,
      "unclipped_grad_norm": 0.817809522151947
    },
    {
      "bits_per_byte": 1.8646075235171788,
      "learning_rate": 0.00021666226336928708,
      "loss_nats": 1.2924474477767944,
      "step": 3250,
      "unclipped_grad_norm": 0.8675336837768555
    },
    {
      "bits_per_byte": 1.7240638171800442,
      "learning_rate": 0.0002161462561185532,
      "loss_nats": 1.1950299739837646,
      "step": 3260,
      "unclipped_grad_norm": 0.8383887410163879
    },
    {
      "bits_per_byte": 1.7558293567597067,
      "learning_rate": 0.00021562937491941957,
      "loss_nats": 1.217048168182373,
      "step": 3270,
      "unclipped_grad_norm": 0.7917745113372803
    },
    {
      "bits_per_byte": 1.8139120255396979,
      "learning_rate": 0.00021511162860395975,
      "loss_nats": 1.257308006286621,
      "step": 3280,
      "unclipped_grad_norm": 0.819724440574646
    },
    {
      "bits_per_byte": 1.8132829130028418,
      "learning_rate": 0.0002145930260190298,
      "loss_nats": 1.2568719387054443,
      "step": 3290,
      "unclipped_grad_norm": 0.8676843047142029
    },
    {
      "bits_per_byte": 1.7244280764345716,
      "learning_rate": 0.00021407357602611698,
      "loss_nats": 1.1952824592590332,
      "step": 3300,
      "unclipped_grad_norm": 0.7852486968040466
    },
    {
      "bits_per_byte": 1.7655542877353951,
      "learning_rate": 0.0002135532875011885,
      "loss_nats": 1.2237889766693115,
      "step": 3310,
      "unclipped_grad_norm": 0.8482234477996826
    },
    {
      "bits_per_byte": 1.7764401016044076,
      "learning_rate": 0.0002130321693345397,
      "loss_nats": 1.2313344478607178,
      "step": 3320,
      "unclipped_grad_norm": 0.80937659740448
    },
    {
      "bits_per_byte": 1.7828375122512021,
      "learning_rate": 0.00021251023043064224,
      "loss_nats": 1.2357687950134277,
      "step": 3330,
      "unclipped_grad_norm": 0.8368547558784485
    },
    {
      "bits_per_byte": 1.7427524839187023,
      "learning_rate": 0.00021198747970799188,
      "loss_nats": 1.2079839706420898,
      "step": 3340,
      "unclipped_grad_norm": 0.8288648128509521
    },
    {
      "bits_per_byte": 1.7505973005551585,
      "learning_rate": 0.00021146392609895618,
      "loss_nats": 1.2134215831756592,
      "step": 3350,
      "unclipped_grad_norm": 0.7933014631271362
    },
    {
      "bits_per_byte": 1.7744660847378042,
      "learning_rate": 0.00021093957854962178,
      "loss_nats": 1.229966163635254,
      "step": 3360,
      "unclipped_grad_norm": 0.8237564563751221
    },
    {
      "bits_per_byte": 1.725253937124015,
      "learning_rate": 0.00021041444601964157,
      "loss_nats": 1.195854902267456,
      "step": 3370,
      "unclipped_grad_norm": 0.8081465363502502
    },
    {
      "bits_per_byte": 1.7597923529835384,
      "learning_rate": 0.00020988853748208164,
      "loss_nats": 1.2197951078414917,
      "step": 3380,
      "unclipped_grad_norm": 0.816970944404602
    },
    {
      "bits_per_byte": 1.7815524578839552,
      "learning_rate": 0.00020936186192326782,
      "loss_nats": 1.2348780632019043,
      "step": 3390,
      "unclipped_grad_norm": 0.8336570858955383
    },
    {
      "bits_per_byte": 1.834212685664893,
      "learning_rate": 0.00020883442834263225,
      "loss_nats": 1.2713793516159058,
      "step": 3400,
      "unclipped_grad_norm": 0.8331590890884399
    },
    {
      "bits_per_byte": 1.8815787715043208,
      "learning_rate": 0.0002083062457525596,
      "loss_nats": 1.3042110204696655,
      "step": 3410,
      "unclipped_grad_norm": 0.8871732354164124
    },
    {
      "bits_per_byte": 1.7373591079876445,
      "learning_rate": 0.00020777732317823306,
      "loss_nats": 1.2042455673217773,
      "step": 3420,
      "unclipped_grad_norm": 0.8121070265769958
    },
    {
      "bits_per_byte": 1.6796587566405214,
      "learning_rate": 0.00020724766965748005,
      "loss_nats": 1.1642507314682007,
      "step": 3430,
      "unclipped_grad_norm": 0.9302552938461304
    },
    {
      "bits_per_byte": 1.7533135945428855,
      "learning_rate": 0.00020671729424061788,
      "loss_nats": 1.2153043746948242,
      "step": 3440,
      "unclipped_grad_norm": 0.8166707754135132
    },
    {
      "bits_per_byte": 1.8162779908676177,
      "learning_rate": 0.00020618620599029906,
      "loss_nats": 1.2589479684829712,
      "step": 3450,
      "unclipped_grad_norm": 0.8299344182014465
    },
    {
      "bits_per_byte": 1.7445694806250678,
      "learning_rate": 0.00020565441398135649,
      "loss_nats": 1.2092434167861938,
      "step": 3460,
      "unclipped_grad_norm": 0.8381974101066589
    },
    {
      "bits_per_byte": 1.7494488004126965,
      "learning_rate": 0.00020512192730064827,
      "loss_nats": 1.212625503540039,
      "step": 3470,
      "unclipped_grad_norm": 0.8255388140678406
    },
    {
      "bits_per_byte": 1.6905657243755898,
      "learning_rate": 0.00020458875504690273,
      "loss_nats": 1.1718108654022217,
      "step": 3480,
      "unclipped_grad_norm": 0.7999947667121887
    },
    {
      "bits_per_byte": 1.9018883747273607,
      "learning_rate": 0.00020405490633056245,
      "loss_nats": 1.3182885646820068,
      "step": 3490,
      "unclipped_grad_norm": 0.8489032983779907
    },
    {
      "bits_per_byte": 1.7449648687394015,
      "learning_rate": 0.00020352039027362917,
      "loss_nats": 1.209517478942871,
      "step": 3500,
      "unclipped_grad_norm": 0.8267634510993958
    },
    {
      "bits_per_byte": 1.784236763098622,
      "learning_rate": 0.0002029852160095075,
      "loss_nats": 1.236738681793213,
      "step": 3510,
      "unclipped_grad_norm": 0.8314446806907654
    },
    {
      "bits_per_byte": 1.7790934499419189,
      "learning_rate": 0.000202449392682849,
      "loss_nats": 1.2331736087799072,
      "step": 3520,
      "unclipped_grad_norm": 0.8323742151260376
    },
    {
      "bits_per_byte": 1.7501370749814495,
      "learning_rate": 0.0002019129294493959,
      "loss_nats": 1.2131025791168213,
      "step": 3530,
      "unclipped_grad_norm": 0.8194237947463989
    },
    {
      "bits_per_byte": 1.7620009541859187,
      "learning_rate": 0.00020137583547582478,
      "loss_nats": 1.2213259935379028,
      "step": 3540,
      "unclipped_grad_norm": 0.8233645558357239
    },
    {
      "bits_per_byte": 1.7622630557458336,
      "learning_rate": 0.00020083811993958973,
      "loss_nats": 1.2215076684951782,
      "step": 3550,
      "unclipped_grad_norm": 0.8949882984161377
    },
    {
      "bits_per_byte": 1.7110605529136091,
      "learning_rate": 0.00020029979202876563,
      "loss_nats": 1.1860167980194092,
      "step": 3560,
      "unclipped_grad_norm": 0.862553060054779
    },
    {
      "bits_per_byte": 1.73054515536046,
      "learning_rate": 0.00019976086094189118,
      "loss_nats": 1.1995224952697754,
      "step": 3570,
      "unclipped_grad_norm": 0.8900191783905029
    },
    {
      "bits_per_byte": 1.763418263211679,
      "learning_rate": 0.00019922133588781175,
      "loss_nats": 1.2223083972930908,
      "step": 3580,
      "unclipped_grad_norm": 0.9058765769004822
    },
    {
      "bits_per_byte": 1.7831680629061606,
      "learning_rate": 0.0001986812260855219,
      "loss_nats": 1.2359979152679443,
      "step": 3590,
      "unclipped_grad_norm": 0.8453392386436462
    },
    {
      "bits_per_byte": 1.7970675287662419,
      "learning_rate": 0.00019814054076400787,
      "loss_nats": 1.245632290840149,
      "step": 3600,
      "unclipped_grad_norm": 0.8932130932807922
    },
    {
      "bits_per_byte": 1.7267670404863058,
      "learning_rate": 0.00019759928916209004,
      "loss_nats": 1.1969037055969238,
      "step": 3610,
      "unclipped_grad_norm": 0.8229808211326599
    },
    {
      "bits_per_byte": 1.6908183668897072,
      "learning_rate": 0.00019705748052826493,
      "loss_nats": 1.1719859838485718,
      "step": 3620,
      "unclipped_grad_norm": 0.8186308741569519
    },
    {
      "bits_per_byte": 1.7541439267812513,
      "learning_rate": 0.00019651512412054723,
      "loss_nats": 1.2158799171447754,
      "step": 3630,
      "unclipped_grad_norm": 0.8367486596107483
    },
    {
      "bits_per_byte": 1.7116509693540212,
      "learning_rate": 0.00019597222920631144,
      "loss_nats": 1.186426043510437,
      "step": 3640,
      "unclipped_grad_norm": 0.8415036797523499
    },
    {
      "bits_per_byte": 1.7837473004742666,
      "learning_rate": 0.0001954288050621338,
      "loss_nats": 1.2363994121551514,
      "step": 3650,
      "unclipped_grad_norm": 0.8471388816833496
    },
    {
      "bits_per_byte": 1.6866421121788058,
      "learning_rate": 0.00019488486097363363,
      "loss_nats": 1.1690912246704102,
      "step": 3660,
      "unclipped_grad_norm": 0.823422908782959
    },
    {
      "bits_per_byte": 1.7654146378228945,
      "learning_rate": 0.00019434040623531452,
      "loss_nats": 1.2236921787261963,
      "step": 3670,
      "unclipped_grad_norm": 0.819604218006134
    },
    {
      "bits_per_byte": 1.6976106497028831,
      "learning_rate": 0.0001937954501504058,
      "loss_nats": 1.1766940355300903,
      "step": 3680,
      "unclipped_grad_norm": 0.8133332133293152
    },
    {
      "bits_per_byte": 1.6802502049768386,
      "learning_rate": 0.00019325000203070333,
      "loss_nats": 1.1646606922149658,
      "step": 3690,
      "unclipped_grad_norm": 0.7951709032058716
    },
    {
      "bits_per_byte": 1.6953377269890801,
      "learning_rate": 0.0001927040711964106,
      "loss_nats": 1.1751185655593872,
      "step": 3700,
      "unclipped_grad_norm": 0.8330065011978149
    },
    {
      "bits_per_byte": 1.747265652642644,
      "learning_rate": 0.00019215766697597923,
      "loss_nats": 1.2111122608184814,
      "step": 3710,
      "unclipped_grad_norm": 0.8084886074066162
    },
    {
      "bits_per_byte": 1.7300491573953718,
      "learning_rate": 0.00019161079870594978,
      "loss_nats": 1.199178695678711,
      "step": 3720,
      "unclipped_grad_norm": 0.8567172288894653
    },
    {
      "bits_per_byte": 1.6577672569948454,
      "learning_rate": 0.0001910634757307922,
      "loss_nats": 1.1490767002105713,
      "step": 3730,
      "unclipped_grad_norm": 0.8171049952507019
    },
    {
      "bits_per_byte": 1.720541440507723,
      "learning_rate": 0.0001905157074027459,
      "loss_nats": 1.192588448524475,
      "step": 3740,
      "unclipped_grad_norm": 0.8777816295623779
    },
    {
      "bits_per_byte": 1.7468836791750777,
      "learning_rate": 0.00018996750308166037,
      "loss_nats": 1.2108474969863892,
      "step": 3750,
      "unclipped_grad_norm": 0.8143476843833923
    },
    {
      "bits_per_byte": 1.8090201510185562,
      "learning_rate": 0.00018941887213483482,
      "loss_nats": 1.2539172172546387,
      "step": 3760,
      "unclipped_grad_norm": 0.8902925252914429
    },
    {
      "bits_per_byte": 1.7778630860576463,
      "learning_rate": 0.00018886982393685845,
      "loss_nats": 1.232320785522461,
      "step": 3770,
      "unclipped_grad_norm": 0.8659905791282654
    },
    {
      "bits_per_byte": 1.6107073002739534,
      "learning_rate": 0.00018832036786945008,
      "loss_nats": 1.116457223892212,
      "step": 3780,
      "unclipped_grad_norm": 0.7876954674720764
    },
    {
      "bits_per_byte": 1.7097620838995948,
      "learning_rate": 0.0001877705133212979,
      "loss_nats": 1.1851167678833008,
      "step": 3790,
      "unclipped_grad_norm": 0.8166313767433167
    },
    {
      "bits_per_byte": 1.703528744701723,
      "learning_rate": 0.00018722026968789906,
      "loss_nats": 1.1807961463928223,
      "step": 3800,
      "unclipped_grad_norm": 0.7970600128173828
    },
    {
      "bits_per_byte": 1.7306136044555034,
      "learning_rate": 0.00018666964637139907,
      "loss_nats": 1.1995699405670166,
      "step": 3810,
      "unclipped_grad_norm": 0.9229923486709595
    },
    {
      "bits_per_byte": 1.7223408949837007,
      "learning_rate": 0.00018611865278043115,
      "loss_nats": 1.193835735321045,
      "step": 3820,
      "unclipped_grad_norm": 0.88306725025177
    },
    {
      "bits_per_byte": 1.7747577673136674,
      "learning_rate": 0.00018556729832995555,
      "loss_nats": 1.230168342590332,
      "step": 3830,
      "unclipped_grad_norm": 0.9060350060462952
    },
    {
      "bits_per_byte": 1.6914633018304428,
      "learning_rate": 0.00018501559244109864,
      "loss_nats": 1.1724330186843872,
      "step": 3840,
      "unclipped_grad_norm": 0.8194422721862793
    },
    {
      "bits_per_byte": 1.648618811864849,
      "learning_rate": 0.00018446354454099178,
      "loss_nats": 1.142735481262207,
      "step": 3850,
      "unclipped_grad_norm": 0.8284996747970581
    },
    {
      "bits_per_byte": 1.7050731489064712,
      "learning_rate": 0.00018391116406261048,
      "loss_nats": 1.1818666458129883,
      "step": 3860,
      "unclipped_grad_norm": 0.8395226001739502
    },
    {
      "bits_per_byte": 1.7881865166583373,
      "learning_rate": 0.00018335846044461305,
      "loss_nats": 1.2394764423370361,
      "step": 3870,
      "unclipped_grad_norm": 0.8676077127456665
    },
    {
      "bits_per_byte": 1.6805537543556115,
      "learning_rate": 0.0001828054431311793,
      "loss_nats": 1.164871096611023,
      "step": 3880,
      "unclipped_grad_norm": 0.8120661377906799
    },
    {
      "bits_per_byte": 1.6503448297489083,
      "learning_rate": 0.00018225212157184934,
      "loss_nats": 1.1439318656921387,
      "step": 3890,
      "unclipped_grad_norm": 0.8249826431274414
    },
    {
      "bits_per_byte": 1.7267109741421247,
      "learning_rate": 0.00018169850522136192,
      "loss_nats": 1.1968648433685303,
      "step": 3900,
      "unclipped_grad_norm": 0.9262217879295349
    },
    {
      "bits_per_byte": 1.7407927416121205,
      "learning_rate": 0.000181144603539493,
      "loss_nats": 1.2066255807876587,
      "step": 3910,
      "unclipped_grad_norm": 0.894594669342041
    },
    {
      "bits_per_byte": 1.6851732083577864,
      "learning_rate": 0.00018059042599089412,
      "loss_nats": 1.168073058128357,
      "step": 3920,
      "unclipped_grad_norm": 0.7890235185623169
    },
    {
      "bits_per_byte": 1.7178566193451037,
      "learning_rate": 0.00018003598204493048,
      "loss_nats": 1.1907274723052979,
      "step": 3930,
      "unclipped_grad_norm": 0.8578466773033142
    },
    {
      "bits_per_byte": 1.7552539028099199,
      "learning_rate": 0.00017948128117551936,
      "loss_nats": 1.2166492938995361,
      "step": 3940,
      "unclipped_grad_norm": 0.8101285696029663
    },
    {
      "bits_per_byte": 1.7865160491705068,
      "learning_rate": 0.00017892633286096816,
      "loss_nats": 1.2383185625076294,
      "step": 3950,
      "unclipped_grad_norm": 0.8577295541763306
    },
    {
      "bits_per_byte": 1.7671413436375571,
      "learning_rate": 0.00017837114658381247,
      "loss_nats": 1.2248890399932861,
      "step": 3960,
      "unclipped_grad_norm": 0.9096070528030396
    },
    {
      "bits_per_byte": 1.6949598811051345,
      "learning_rate": 0.00017781573183065386,
      "loss_nats": 1.1748566627502441,
      "step": 3970,
      "unclipped_grad_norm": 0.8878244757652283
    },
    {
      "bits_per_byte": 1.7218173797944742,
      "learning_rate": 0.0001772600980919981,
      "loss_nats": 1.1934728622436523,
      "step": 3980,
      "unclipped_grad_norm": 0.8564468026161194
    },
    {
      "bits_per_byte": 1.7444124604648301,
      "learning_rate": 0.0001767042548620927,
      "loss_nats": 1.209134578704834,
      "step": 3990,
      "unclipped_grad_norm": 0.8601830005645752
    },
    {
      "bits_per_byte": 1.6364575746570382,
      "learning_rate": 0.00017614821163876486,
      "loss_nats": 1.1343059539794922,
      "step": 4000,
      "unclipped_grad_norm": 0.8214763402938843
    },
    {
      "bits_per_byte": 1.7331732222482947,
      "learning_rate": 0.00017559197792325906,
      "loss_nats": 1.2013441324234009,
      "step": 4010,
      "unclipped_grad_norm": 0.8387731909751892
    },
    {
      "bits_per_byte": 1.7078159282224312,
      "learning_rate": 0.0001750355632200748,
      "loss_nats": 1.1837677955627441,
      "step": 4020,
      "unclipped_grad_norm": 0.8044644594192505
    },
    {
      "bits_per_byte": 1.7505291954254167,
      "learning_rate": 0.00017447897703680418,
      "loss_nats": 1.213374376296997,
      "step": 4030,
      "unclipped_grad_norm": 0.8809406161308289
    },
    {
      "bits_per_byte": 1.6714938802908095,
      "learning_rate": 0.0001739222288839694,
      "loss_nats": 1.1585912704467773,
      "step": 4040,
      "unclipped_grad_norm": 0.877934455871582
    },
    {
      "bits_per_byte": 1.7160740191689108,
      "learning_rate": 0.00017336532827486026,
      "loss_nats": 1.189491868019104,
      "step": 4050,
      "unclipped_grad_norm": 0.8732415437698364
    },
    {
      "bits_per_byte": 1.7110662283410876,
      "learning_rate": 0.00017280828472537163,
      "loss_nats": 1.1860207319259644,
      "step": 4060,
      "unclipped_grad_norm": 0.8489102721214294
    },
    {
      "bits_per_byte": 1.7342386547703899,
      "learning_rate": 0.0001722511077538408,
      "loss_nats": 1.202082633972168,
      "step": 4070,
      "unclipped_grad_norm": 0.8639312386512756
    },
    {
      "bits_per_byte": 1.7989254573435127,
      "learning_rate": 0.00017169380688088497,
      "loss_nats": 1.246920108795166,
      "step": 4080,
      "unclipped_grad_norm": 0.8819956183433533
    },
    {
      "bits_per_byte": 1.705004355846126,
      "learning_rate": 0.0001711363916292384,
      "loss_nats": 1.181818962097168,
      "step": 4090,
      "unclipped_grad_norm": 0.8767969012260437
    },
    {
      "bits_per_byte": 1.6958074116085866,
      "learning_rate": 0.00017057887152358974,
      "loss_nats": 1.1754441261291504,
      "step": 4100,
      "unclipped_grad_norm": 0.8374283313751221
    },
    {
      "bits_per_byte": 1.693619276341659,
      "learning_rate": 0.0001700212560904194,
      "loss_nats": 1.1739274263381958,
      "step": 4110,
      "unclipped_grad_norm": 0.8304628729820251
    },
    {
      "bits_per_byte": 1.744371184628623,
      "learning_rate": 0.00016946355485783656,
      "loss_nats": 1.2091059684753418,
      "step": 4120,
      "unclipped_grad_norm": 0.9000032544136047
    },
    {
      "bits_per_byte": 1.6641087732801096,
      "learning_rate": 0.0001689057773554166,
      "loss_nats": 1.1534723043441772,
      "step": 4130,
      "unclipped_grad_norm": 0.8784226775169373
    },
    {
      "bits_per_byte": 1.7338391390724355,
      "learning_rate": 0.00016834793311403806,
      "loss_nats": 1.2018057107925415,
      "step": 4140,
      "unclipped_grad_norm": 0.864092230796814
    },
    {
      "bits_per_byte": 1.6882804189109248,
      "learning_rate": 0.00016779003166571981,
      "loss_nats": 1.170226812362671,
      "step": 4150,
      "unclipped_grad_norm": 0.8413121104240417
    },
    {
      "bits_per_byte": 1.698938527750195,
      "learning_rate": 0.00016723208254345838,
      "loss_nats": 1.177614450454712,
      "step": 4160,
      "unclipped_grad_norm": 0.9109639525413513
    },
    {
      "bits_per_byte": 1.7387922394172843,
      "learning_rate": 0.00016667409528106481,
      "loss_nats": 1.205238938331604,
      "step": 4170,
      "unclipped_grad_norm": 0.8913268446922302
    },
    {
      "bits_per_byte": 1.7814171075377263,
      "learning_rate": 0.00016611607941300184,
      "loss_nats": 1.2347842454910278,
      "step": 4180,
      "unclipped_grad_norm": 0.9075717926025391
    },
    {
      "bits_per_byte": 1.694894183732505,
      "learning_rate": 0.0001655580444742211,
      "loss_nats": 1.1748111248016357,
      "step": 4190,
      "unclipped_grad_norm": 0.8244585990905762
    },
    {
      "bits_per_byte": 1.7220003693349923,
      "learning_rate": 0.00016499999999999997,
      "loss_nats": 1.1935997009277344,
      "step": 4200,
      "unclipped_grad_norm": 0.8734019994735718
    },
    {
      "bits_per_byte": 1.725182564323907,
      "learning_rate": 0.0001644419555257789,
      "loss_nats": 1.1958054304122925,
      "step": 4210,
      "unclipped_grad_norm": 0.834058403968811
    },
    {
      "bits_per_byte": 1.7486925926968528,
      "learning_rate": 0.0001638839205869981,
      "loss_nats": 1.2121013402938843,
      "step": 4220,
      "unclipped_grad_norm": 0.8368670344352722
    },
    {
      "bits_per_byte": 1.720319754870761,
      "learning_rate": 0.0001633259047189352,
      "loss_nats": 1.1924347877502441,
      "step": 4230,
      "unclipped_grad_norm": 0.8471981287002563
    },
    {
      "bits_per_byte": 1.6660473617206353,
      "learning_rate": 0.0001627679174565416,
      "loss_nats": 1.1548160314559937,
      "step": 4240,
      "unclipped_grad_norm": 0.8083959221839905
    },
    {
      "bits_per_byte": 1.6911604403822735,
      "learning_rate": 0.00016220996833428016,
      "loss_nats": 1.1722230911254883,
      "step": 4250,
      "unclipped_grad_norm": 0.8626519441604614
    },
    {
      "bits_per_byte": 1.6717248529909183,
      "learning_rate": 0.00016165206688596191,
      "loss_nats": 1.158751368522644,
      "step": 4260,
      "unclipped_grad_norm": 0.8391480445861816
    },
    {
      "bits_per_byte": 1.6638891514349579,
      "learning_rate": 0.00016109422264458335,
      "loss_nats": 1.153320074081421,
      "step": 4270,
      "unclipped_grad_norm": 0.8006730079650879
    },
    {
      "bits_per_byte": 1.6802054894876142,
      "learning_rate": 0.00016053644514216336,
      "loss_nats": 1.1646296977996826,
      "step": 4280,
      "unclipped_grad_norm": 0.8485363125801086
    },
    {
      "bits_per_byte": 1.732838028061763,
      "learning_rate": 0.00015997874390958058,
      "loss_nats": 1.2011117935180664,
      "step": 4290,
      "unclipped_grad_norm": 0.8599663972854614
    },
    {
      "bits_per_byte": 1.6415455094001634,
      "learning_rate": 0.00015942112847641018,
      "loss_nats": 1.1378326416015625,
      "step": 4300,
      "unclipped_grad_norm": 0.8445435166358948
    },
    {
      "bits_per_byte": 1.7360513519104837,
      "learning_rate": 0.00015886360837076156,
      "loss_nats": 1.2033390998840332,
      "step": 4310,
      "unclipped_grad_norm": 0.9501782655715942
    },
    {
      "bits_per_byte": 1.7070877536786782,
      "learning_rate": 0.00015830619311911495,
      "loss_nats": 1.1832630634307861,
      "step": 4320,
      "unclipped_grad_norm": 0.8405798673629761
    },
    {
      "bits_per_byte": 1.6698505860618154,
      "learning_rate": 0.00015774889224615915,
      "loss_nats": 1.1574522256851196,
      "step": 4330,
      "unclipped_grad_norm": 0.8460624814033508
    },
    {
      "bits_per_byte": 1.709274341101748,
      "learning_rate": 0.00015719171527462834,
      "loss_nats": 1.1847786903381348,
      "step": 4340,
      "unclipped_grad_norm": 0.8663898706436157
    },
    {
      "bits_per_byte": 1.6889219141986431,
      "learning_rate": 0.0001566346717251397,
      "loss_nats": 1.1706714630126953,
      "step": 4350,
      "unclipped_grad_norm": 0.9172630310058594
    },
    {
      "bits_per_byte": 1.675118930605696,
      "learning_rate": 0.00015607777111603056,
      "loss_nats": 1.1611039638519287,
      "step": 4360,
      "unclipped_grad_norm": 0.8827696442604065
    },
    {
      "bits_per_byte": 1.690404060683779,
      "learning_rate": 0.00015552102296319576,
      "loss_nats": 1.171698808670044,
      "step": 4370,
      "unclipped_grad_norm": 0.8694136738777161
    },
    {
      "bits_per_byte": 1.7489961420756257,
      "learning_rate": 0.00015496443677992516,
      "loss_nats": 1.2123117446899414,
      "step": 4380,
      "unclipped_grad_norm": 0.8883715867996216
    },
    {
      "bits_per_byte": 1.7051840777162777,
      "learning_rate": 0.0001544080220767409,
      "loss_nats": 1.1819435358047485,
      "step": 4390,
      "unclipped_grad_norm": 0.8627059459686279
    },
    {
      "bits_per_byte": 1.6575398959304046,
      "learning_rate": 0.00015385178836123511,
      "loss_nats": 1.1489191055297852,
      "step": 4400,
      "unclipped_grad_norm": 0.8316671848297119
    },
    {
      "bits_per_byte": 1.7285451691135765,
      "learning_rate": 0.00015329574513790726,
      "loss_nats": 1.1981362104415894,
      "step": 4410,
      "unclipped_grad_norm": 0.8469527959823608
    },
    {
      "bits_per_byte": 1.6753765606166884,
      "learning_rate": 0.00015273990190800186,
      "loss_nats": 1.1612825393676758,
      "step": 4420,
      "unclipped_grad_norm": 0.829068124294281
    },
    {
      "bits_per_byte": 1.7212594680750752,
      "learning_rate": 0.0001521842681693461,
      "loss_nats": 1.1930861473083496,
      "step": 4430,
      "unclipped_grad_norm": 0.8506612777709961
    },
    {
      "bits_per_byte": 1.6578714784812683,
      "learning_rate": 0.00015162885341618752,
      "loss_nats": 1.149148941040039,
      "step": 4440,
      "unclipped_grad_norm": 0.8394047021865845
    },
    {
      "bits_per_byte": 1.6499258800114065,
      "learning_rate": 0.00015107366713903178,
      "loss_nats": 1.143641471862793,
      "step": 4450,
      "unclipped_grad_norm": 0.8416959047317505
    },
    {
      "bits_per_byte": 1.64743299148715,
      "learning_rate": 0.0001505187188244806,
      "loss_nats": 1.1419135332107544,
      "step": 4460,
      "unclipped_grad_norm": 0.8612979054450989
    },
    {
      "bits_per_byte": 1.6939761403421993,
      "learning_rate": 0.00014996401795506952,
      "loss_nats": 1.1741747856140137,
      "step": 4470,
      "unclipped_grad_norm": 0.9045028686523438
    },
    {
      "bits_per_byte": 1.7161144350918636,
      "learning_rate": 0.00014940957400910585,
      "loss_nats": 1.1895198822021484,
      "step": 4480,
      "unclipped_grad_norm": 0.9105556607246399
    },
    {
      "bits_per_byte": 1.6800321309755446,
      "learning_rate": 0.00014885539646050699,
      "loss_nats": 1.1645095348358154,
      "step": 4490,
      "unclipped_grad_norm": 0.8570636510848999
    },
    {
      "bits_per_byte": 1.7013357939205715,
      "learning_rate": 0.00014830149477863805,
      "loss_nats": 1.1792761087417603,
      "step": 4500,
      "unclipped_grad_norm": 0.8411341905593872
    },
    {
      "bits_per_byte": 1.6660826181640622,
      "learning_rate": 0.00014774787842815066,
      "loss_nats": 1.1548404693603516,
      "step": 4510,
      "unclipped_grad_norm": 0.8592174053192139
    },
    {
      "bits_per_byte": 1.7057920363870778,
      "learning_rate": 0.00014719455686882066,
      "loss_nats": 1.1823649406433105,
      "step": 4520,
      "unclipped_grad_norm": 0.8456743955612183
    },
    {
      "bits_per_byte": 1.7124625554834427,
      "learning_rate": 0.00014664153955538695,
      "loss_nats": 1.1869885921478271,
      "step": 4530,
      "unclipped_grad_norm": 0.8667335510253906
    },
    {
      "bits_per_byte": 1.761984099886134,
      "learning_rate": 0.00014608883593738946,
      "loss_nats": 1.2213143110275269,
      "step": 4540,
      "unclipped_grad_norm": 0.8827010989189148
    },
    {
      "bits_per_byte": 1.608659158884828,
      "learning_rate": 0.00014553645545900817,
      "loss_nats": 1.1150375604629517,
      "step": 4550,
      "unclipped_grad_norm": 0.8445418477058411
    },
    {
      "bits_per_byte": 1.6846809940110172,
      "learning_rate": 0.0001449844075589013,
      "loss_nats": 1.1677318811416626,
      "step": 4560,
      "unclipped_grad_norm": 0.9384724497795105
    },
    {
      "bits_per_byte": 1.6632275341770886,
      "learning_rate": 0.0001444327016700444,
      "loss_nats": 1.152861475944519,
      "step": 4570,
      "unclipped_grad_norm": 0.836455762386322
    },
    {
      "bits_per_byte": 1.6185414539860559,
      "learning_rate": 0.0001438813472195688,
      "loss_nats": 1.121887445449829,
      "step": 4580,
      "unclipped_grad_norm": 0.8520491123199463
    },
    {
      "bits_per_byte": 1.6763542819868433,
      "learning_rate": 0.00014333035362860093,
      "loss_nats": 1.161960244178772,
      "step": 4590,
      "unclipped_grad_norm": 0.9324198961257935
    },
    {
      "bits_per_byte": 1.667847676109867,
      "learning_rate": 0.00014277973031210088,
      "loss_nats": 1.1560639142990112,
      "step": 4600,
      "unclipped_grad_norm": 0.8337410092353821
    },
    {
      "bits_per_byte": 1.7332526782329933,
      "learning_rate": 0.00014222948667870207,
      "loss_nats": 1.2013992071151733,
      "step": 4610,
      "unclipped_grad_norm": 0.934492290019989
    },
    {
      "bits_per_byte": 1.7177494741536163,
      "learning_rate": 0.00014167963213054991,
      "loss_nats": 1.1906532049179077,
      "step": 4620,
      "unclipped_grad_norm": 0.8968923091888428
    },
    {
      "bits_per_byte": 1.707787379102388,
      "learning_rate": 0.00014113017606314152,
      "loss_nats": 1.1837480068206787,
      "step": 4630,
      "unclipped_grad_norm": 0.9312987327575684
    },
    {
      "bits_per_byte": 1.7495194852822011,
      "learning_rate": 0.00014058112786516518,
      "loss_nats": 1.2126744985580444,
      "step": 4640,
      "unclipped_grad_norm": 0.8537502884864807
    },
    {
      "bits_per_byte": 1.6591016703828898,
      "learning_rate": 0.0001400324969183396,
      "loss_nats": 1.1500016450881958,
      "step": 4650,
      "unclipped_grad_norm": 0.8534259796142578
    },
    {
      "bits_per_byte": 1.63647786860984,
      "learning_rate": 0.00013948429259725406,
      "loss_nats": 1.1343200206756592,
      "step": 4660,
      "unclipped_grad_norm": 0.862847626209259
    },
    {
      "bits_per_byte": 1.758119477738596,
      "learning_rate": 0.00013893652426920776,
      "loss_nats": 1.2186355590820312,
      "step": 4670,
      "unclipped_grad_norm": 0.8828908205032349
    },
    {
      "bits_per_byte": 1.7750497938548324,
      "learning_rate": 0.00013838920129405016,
      "loss_nats": 1.2303707599639893,
      "step": 4680,
      "unclipped_grad_norm": 0.9205049872398376
    },
    {
      "bits_per_byte": 1.7426339878722579,
      "learning_rate": 0.0001378423330240207,
      "loss_nats": 1.2079018354415894,
      "step": 4690,
      "unclipped_grad_norm": 0.92873615026474
    },
    {
      "bits_per_byte": 1.659708425175134,
      "learning_rate": 0.00013729592880358936,
      "loss_nats": 1.150422215461731,
      "step": 4700,
      "unclipped_grad_norm": 0.8619311451911926
    },
    {
      "bits_per_byte": 1.7262008735996655,
      "learning_rate": 0.00013674999796929659,
      "loss_nats": 1.1965112686157227,
      "step": 4710,
      "unclipped_grad_norm": 0.9467828869819641
    },
    {
      "bits_per_byte": 1.6608278602495998,
      "learning_rate": 0.00013620454984959417,
      "loss_nats": 1.151198148727417,
      "step": 4720,
      "unclipped_grad_norm": 0.8912813067436218
    },
    {
      "bits_per_byte": 1.5957210760430698,
      "learning_rate": 0.00013565959376468542,
      "loss_nats": 1.106069564819336,
      "step": 4730,
      "unclipped_grad_norm": 0.8848060369491577
    },
    {
      "bits_per_byte": 1.6368768683598418,
      "learning_rate": 0.00013511513902636634,
      "loss_nats": 1.134596586227417,
      "step": 4740,
      "unclipped_grad_norm": 0.9091616868972778
    },
    {
      "bits_per_byte": 1.6949358035340139,
      "learning_rate": 0.0001345711949378662,
      "loss_nats": 1.174839973449707,
      "step": 4750,
      "unclipped_grad_norm": 0.9046142101287842
    },
    {
      "bits_per_byte": 1.665315747523865,
      "learning_rate": 0.00013402777079368853,
      "loss_nats": 1.1543089151382446,
      "step": 4760,
      "unclipped_grad_norm": 0.8868388533592224
    },
    {
      "bits_per_byte": 1.6325195159175816,
      "learning_rate": 0.0001334848758794528,
      "loss_nats": 1.1315762996673584,
      "step": 4770,
      "unclipped_grad_norm": 0.839006781578064
    },
    {
      "bits_per_byte": 1.6782909786182094,
      "learning_rate": 0.00013294251947173504,
      "loss_nats": 1.1633026599884033,
      "step": 4780,
      "unclipped_grad_norm": 0.930033266544342
    },
    {
      "bits_per_byte": 1.6275290953374955,
      "learning_rate": 0.00013240071083790996,
      "loss_nats": 1.1281172037124634,
      "step": 4790,
      "unclipped_grad_norm": 0.8176500201225281
    },
    {
      "bits_per_byte": 1.6333247386889214,
      "learning_rate": 0.0001318594592359921,
      "loss_nats": 1.1321344375610352,
      "step": 4800,
      "unclipped_grad_norm": 0.855272650718689
    },
    {
      "bits_per_byte": 1.6820415762682257,
      "learning_rate": 0.0001313187739144781,
      "loss_nats": 1.1659023761749268,
      "step": 4810,
      "unclipped_grad_norm": 0.9120509624481201
    },
    {
      "bits_per_byte": 1.6350118884938853,
      "learning_rate": 0.0001307786641121882,
      "loss_nats": 1.1333038806915283,
      "step": 4820,
      "unclipped_grad_norm": 0.89643794298172
    },
    {
      "bits_per_byte": 1.6668641073295827,
      "learning_rate": 0.00013023913905810876,
      "loss_nats": 1.1553821563720703,
      "step": 4830,
      "unclipped_grad_norm": 0.9077607989311218
    },
    {
      "bits_per_byte": 1.667841656717087,
      "learning_rate": 0.00012970020797123432,
      "loss_nats": 1.156059741973877,
      "step": 4840,
      "unclipped_grad_norm": 0.9171648621559143
    },
    {
      "bits_per_byte": 1.6750915853642085,
      "learning_rate": 0.00012916188006041027,
      "loss_nats": 1.1610850095748901,
      "step": 4850,
      "unclipped_grad_norm": 0.8878066539764404
    },
    {
      "bits_per_byte": 1.6131449823672828,
      "learning_rate": 0.00012862416452417516,
      "loss_nats": 1.1181468963623047,
      "step": 4860,
      "unclipped_grad_norm": 0.8896912932395935
    },
    {
      "bits_per_byte": 1.6943018754829335,
      "learning_rate": 0.00012808707055060405,
      "loss_nats": 1.1744005680084229,
      "step": 4870,
      "unclipped_grad_norm": 0.8964843153953552
    },
    {
      "bits_per_byte": 1.5440334101527613,
      "learning_rate": 0.000127550607317151,
      "loss_nats": 1.0702424049377441,
      "step": 4880,
      "unclipped_grad_norm": 0.8637375235557556
    },
    {
      "bits_per_byte": 1.609086879737524,
      "learning_rate": 0.00012701478399049248,
      "loss_nats": 1.1153340339660645,
      "step": 4890,
      "unclipped_grad_norm": 0.8875832557678223
    },
    {
      "bits_per_byte": 1.651299333461197,
      "learning_rate": 0.0001264796097263708,
      "loss_nats": 1.1445934772491455,
      "step": 4900,
      "unclipped_grad_norm": 0.9340603351593018
    },
    {
      "bits_per_byte": 1.7515653908968651,
      "learning_rate": 0.0001259450936694375,
      "loss_nats": 1.2140926122665405,
      "step": 4910,
      "unclipped_grad_norm": 0.9391474723815918
    },
    {
      "bits_per_byte": 1.6798804422774836,
      "learning_rate": 0.00012541124495309727,
      "loss_nats": 1.1644043922424316,
      "step": 4920,
      "unclipped_grad_norm": 0.8976443409919739
    },
    {
      "bits_per_byte": 1.6454846000355252,
      "learning_rate": 0.00012487807269935167,
      "loss_nats": 1.1405630111694336,
      "step": 4930,
      "unclipped_grad_norm": 0.8620614409446716
    },
    {
      "bits_per_byte": 1.6197019929140781,
      "learning_rate": 0.00012434558601864349,
      "loss_nats": 1.1226918697357178,
      "step": 4940,
      "unclipped_grad_norm": 0.8644847869873047
    },
    {
      "bits_per_byte": 1.5870352642438945,
      "learning_rate": 0.0001238137940097009,
      "loss_nats": 1.1000490188598633,
      "step": 4950,
      "unclipped_grad_norm": 0.8775791525840759
    },
    {
      "bits_per_byte": 1.5784220291233835,
      "learning_rate": 0.0001232827057593821,
      "loss_nats": 1.094078779220581,
      "step": 4960,
      "unclipped_grad_norm": 0.8890455961227417
    },
    {
      "bits_per_byte": 1.6649912162616867,
      "learning_rate": 0.00012275233034251992,
      "loss_nats": 1.1540839672088623,
      "step": 4970,
      "unclipped_grad_norm": 0.9286753535270691
    },
    {
      "bits_per_byte": 1.7156910138054393,
      "learning_rate": 0.0001222226768217669,
      "loss_nats": 1.1892263889312744,
      "step": 4980,
      "unclipped_grad_norm": 0.9152454137802124
    },
    {
      "bits_per_byte": 1.6570270436655319,
      "learning_rate": 0.00012169375424744034,
      "loss_nats": 1.1485636234283447,
      "step": 4990,
      "unclipped_grad_norm": 0.9534220099449158
    },
    {
      "bits_per_byte": 1.6136648859208411,
      "learning_rate": 0.00012116557165736772,
      "loss_nats": 1.1185072660446167,
      "step": 5000,
      "unclipped_grad_norm": 0.8542637825012207
    },
    {
      "bits_per_byte": 1.5871047452348432,
      "learning_rate": 0.00012063813807673214,
      "loss_nats": 1.1000971794128418,
      "step": 5010,
      "unclipped_grad_norm": 0.8614861965179443
    },
    {
      "bits_per_byte": 1.6463324745042789,
      "learning_rate": 0.00012011146251791832,
      "loss_nats": 1.141150712966919,
      "step": 5020,
      "unclipped_grad_norm": 0.8538332581520081
    },
    {
      "bits_per_byte": 1.630251408718003,
      "learning_rate": 0.0001195855539803584,
      "loss_nats": 1.1300041675567627,
      "step": 5030,
      "unclipped_grad_norm": 0.8974499702453613
    },
    {
      "bits_per_byte": 1.6081984173631667,
      "learning_rate": 0.00011906042145037817,
      "loss_nats": 1.1147181987762451,
      "step": 5040,
      "unclipped_grad_norm": 0.8664147853851318
    },
    {
      "bits_per_byte": 1.566623675308893,
      "learning_rate": 0.0001185360739010438,
      "loss_nats": 1.0859007835388184,
      "step": 5050,
      "unclipped_grad_norm": 0.8855953216552734
    },
    {
      "bits_per_byte": 1.5903796388725728,
      "learning_rate": 0.00011801252029200808,
      "loss_nats": 1.1023671627044678,
      "step": 5060,
      "unclipped_grad_norm": 0.8359810709953308
    },
    {
      "bits_per_byte": 1.6634062241513352,
      "learning_rate": 0.00011748976956935777,
      "loss_nats": 1.1529853343963623,
      "step": 5070,
      "unclipped_grad_norm": 0.8875454068183899
    },
    {
      "bits_per_byte": 1.6210933325595582,
      "learning_rate": 0.00011696783066546024,
      "loss_nats": 1.1236562728881836,
      "step": 5080,
      "unclipped_grad_norm": 0.9011101722717285
    },
    {
      "bits_per_byte": 1.6245405528134524,
      "learning_rate": 0.00011644671249881147,
      "loss_nats": 1.1260457038879395,
      "step": 5090,
      "unclipped_grad_norm": 0.8935633301734924
    },
    {
      "bits_per_byte": 1.6269426344980533,
      "learning_rate": 0.00011592642397388293,
      "loss_nats": 1.1277107000350952,
      "step": 5100,
      "unclipped_grad_norm": 0.8533574342727661
    },
    {
      "bits_per_byte": 1.565103176692615,
      "learning_rate": 0.00011540697398097015,
      "loss_nats": 1.0848468542099,
      "step": 5110,
      "unclipped_grad_norm": 0.9117326736450195
    },
    {
      "bits_per_byte": 1.6439374441083634,
      "learning_rate": 0.00011488837139604016,
      "loss_nats": 1.1394906044006348,
      "step": 5120,
      "unclipped_grad_norm": 0.8894507884979248
    },
    {
      "bits_per_byte": 1.6476570848812242,
      "learning_rate": 0.00011437062508058039,
      "loss_nats": 1.142068862915039,
      "step": 5130,
      "unclipped_grad_norm": 0.8907402753829956
    },
    {
      "bits_per_byte": 1.615579052824944,
      "learning_rate": 0.00011385374388144673,
      "loss_nats": 1.119834065437317,
      "step": 5140,
      "unclipped_grad_norm": 0.8878148794174194
    },
    {
      "bits_per_byte": 1.6886897376199783,
      "learning_rate": 0.00011333773663071286,
      "loss_nats": 1.1705105304718018,
      "step": 5150,
      "unclipped_grad_norm": 0.906775712966919
    },
    {
      "bits_per_byte": 1.5749478075933039,
      "learning_rate": 0.00011282261214551897,
      "loss_nats": 1.0916706323623657,
      "step": 5160,
      "unclipped_grad_norm": 0.8668916821479797
    },
    {
      "bits_per_byte": 1.7387628303839868,
      "learning_rate": 0.00011230837922792112,
      "loss_nats": 1.2052185535430908,
      "step": 5170,
      "unclipped_grad_norm": 0.951082706451416
    },
    {
      "bits_per_byte": 1.633963310271575,
      "learning_rate": 0.0001117950466647412,
      "loss_nats": 1.1325770616531372,
      "step": 5180,
      "unclipped_grad_norm": 0.9125628471374512
    },
    {
      "bits_per_byte": 1.571220427601154,
      "learning_rate": 0.00011128262322741627,
      "loss_nats": 1.0890870094299316,
      "step": 5190,
      "unclipped_grad_norm": 0.837740421295166
    },
    {
      "bits_per_byte": 1.600543469573263,
      "learning_rate": 0.00011077111767184913,
      "loss_nats": 1.1094121932983398,
      "step": 5200,
      "unclipped_grad_norm": 0.8841894865036011
    },
    {
      "bits_per_byte": 1.630884304873178,
      "learning_rate": 0.00011026053873825837,
      "loss_nats": 1.1304428577423096,
      "step": 5210,
      "unclipped_grad_norm": 0.8948630690574646
    },
    {
      "bits_per_byte": 1.7074470254363305,
      "learning_rate": 0.0001097508951510293,
      "loss_nats": 1.1835120916366577,
      "step": 5220,
      "unclipped_grad_norm": 0.9275010228157043
    },
    {
      "bits_per_byte": 1.6155556631844268,
      "learning_rate": 0.0001092421956185646,
      "loss_nats": 1.119817852973938,
      "step": 5230,
      "unclipped_grad_norm": 0.8827324509620667
    },
    {
      "bits_per_byte": 1.615808649663846,
      "learning_rate": 0.00010873444883313581,
      "loss_nats": 1.1199932098388672,
      "step": 5240,
      "unclipped_grad_norm": 0.8995877504348755
    },
    {
      "bits_per_byte": 1.592627108154048,
      "learning_rate": 0.00010822766347073447,
      "loss_nats": 1.1039249897003174,
      "step": 5250,
      "unclipped_grad_norm": 0.8981198668479919
    },
    {
      "bits_per_byte": 1.60489222288298,
      "learning_rate": 0.00010772184819092424,
      "loss_nats": 1.112426519393921,
      "step": 5260,
      "unclipped_grad_norm": 0.8831214308738708
    },
    {
      "bits_per_byte": 1.5994845723919007,
      "learning_rate": 0.0001072170116366926,
      "loss_nats": 1.1086782217025757,
      "step": 5270,
      "unclipped_grad_norm": 0.8702658414840698
    },
    {
      "bits_per_byte": 1.6097115207254575,
      "learning_rate": 0.00010671316243430345,
      "loss_nats": 1.115767002105713,
      "step": 5280,
      "unclipped_grad_norm": 0.9307021498680115
    },
    {
      "bits_per_byte": 1.6134614304448702,
      "learning_rate": 0.00010621030919314944,
      "loss_nats": 1.1183662414550781,
      "step": 5290,
      "unclipped_grad_norm": 0.9075614213943481
    },
    {
      "bits_per_byte": 1.6247061721062332,
      "learning_rate": 0.000105708460505605,
      "loss_nats": 1.1261605024337769,
      "step": 5300,
      "unclipped_grad_norm": 0.8755994439125061
    },
    {
      "bits_per_byte": 1.6182275856482313,
      "learning_rate": 0.00010520762494687964,
      "loss_nats": 1.121669888496399,
      "step": 5310,
      "unclipped_grad_norm": 0.914459764957428
    },
    {
      "bits_per_byte": 1.6139610400456268,
      "learning_rate": 0.00010470781107487111,
      "loss_nats": 1.1187125444412231,
      "step": 5320,
      "unclipped_grad_norm": 0.8940036296844482
    },
    {
      "bits_per_byte": 1.6078936641058377,
      "learning_rate": 0.00010420902743001955,
      "loss_nats": 1.1145069599151611,
      "step": 5330,
      "unclipped_grad_norm": 0.9279354214668274
    },
    {
      "bits_per_byte": 1.569059465593063,
      "learning_rate": 0.00010371128253516117,
      "loss_nats": 1.087589144706726,
      "step": 5340,
      "unclipped_grad_norm": 0.8639222979545593
    },
    {
      "bits_per_byte": 1.6467650108711989,
      "learning_rate": 0.000103214584895383,
      "loss_nats": 1.1414505243301392,
      "step": 5350,
      "unclipped_grad_norm": 0.9308397769927979
    },
    {
      "bits_per_byte": 1.565089418080546,
      "learning_rate": 0.00010271894299787715,
      "loss_nats": 1.0848373174667358,
      "step": 5360,
      "unclipped_grad_norm": 0.9050061702728271
    },
    {
      "bits_per_byte": 1.5389489150626532,
      "learning_rate": 0.00010222436531179622,
      "loss_nats": 1.0667181015014648,
      "step": 5370,
      "unclipped_grad_norm": 0.8974517583847046
    },
    {
      "bits_per_byte": 1.6578491207366561,
      "learning_rate": 0.0001017308602881082,
      "loss_nats": 1.1491334438323975,
      "step": 5380,
      "unclipped_grad_norm": 0.9271295070648193
    },
    {
      "bits_per_byte": 1.5921135679585716,
      "learning_rate": 0.00010123843635945242,
      "loss_nats": 1.1035690307617188,
      "step": 5390,
      "unclipped_grad_norm": 0.884811282157898
    },
    {
      "bits_per_byte": 1.6126496723327979,
      "learning_rate": 0.00010074710193999505,
      "loss_nats": 1.1178035736083984,
      "step": 5400,
      "unclipped_grad_norm": 0.9159505367279053
    },
    {
      "bits_per_byte": 1.6339560870002385,
      "learning_rate": 0.0001002568654252858,
      "loss_nats": 1.132572054862976,
      "step": 5410,
      "unclipped_grad_norm": 0.9119471907615662
    },
    {
      "bits_per_byte": 1.595336178870439,
      "learning_rate": 9.976773519211404e-05,
      "loss_nats": 1.1058027744293213,
      "step": 5420,
      "unclipped_grad_norm": 0.9820973873138428
    },
    {
      "bits_per_byte": 1.649176723584248,
      "learning_rate": 9.92797195983659e-05,
      "loss_nats": 1.1431221961975098,
      "step": 5430,
      "unclipped_grad_norm": 0.9529035091400146
    },
    {
      "bits_per_byte": 1.5866154545931386,
      "learning_rate": 9.879282698288142e-05,
      "loss_nats": 1.0997580289840698,
      "step": 5440,
      "unclipped_grad_norm": 0.9037122130393982
    },
    {
      "bits_per_byte": 1.5834752233710345,
      "learning_rate": 9.830706566531197e-05,
      "loss_nats": 1.097581386566162,
      "step": 5450,
      "unclipped_grad_norm": 0.9287867546081543
    },
    {
      "bits_per_byte": 1.5614380544200777,
      "learning_rate": 9.782244394597826e-05,
      "loss_nats": 1.0823063850402832,
      "step": 5460,
      "unclipped_grad_norm": 0.8947009444236755
    },
    {
      "bits_per_byte": 1.607795462012195,
      "learning_rate": 9.733897010572829e-05,
      "loss_nats": 1.1144388914108276,
      "step": 5470,
      "unclipped_grad_norm": 0.9344596862792969
    },
    {
      "bits_per_byte": 1.6195384374131077,
      "learning_rate": 9.685665240579612e-05,
      "loss_nats": 1.122578501701355,
      "step": 5480,
      "unclipped_grad_norm": 0.8986119031906128
    },
    {
      "bits_per_byte": 1.630767700635893,
      "learning_rate": 9.637549908766041e-05,
      "loss_nats": 1.1303620338439941,
      "step": 5490,
      "unclipped_grad_norm": 0.9683650135993958
    },
    {
      "bits_per_byte": 1.6068012303075572,
      "learning_rate": 9.589551837290389e-05,
      "loss_nats": 1.1137497425079346,
      "step": 5500,
      "unclipped_grad_norm": 0.913102924823761
    },
    {
      "bits_per_byte": 1.7072599083121918,
      "learning_rate": 9.541671846307257e-05,
      "loss_nats": 1.1833823919296265,
      "step": 5510,
      "unclipped_grad_norm": 1.0086424350738525
    },
    {
      "bits_per_byte": 1.6241427569420068,
      "learning_rate": 9.493910753953597e-05,
      "loss_nats": 1.1257699728012085,
      "step": 5520,
      "unclipped_grad_norm": 0.9219017624855042
    },
    {
      "bits_per_byte": 1.5865065895751422,
      "learning_rate": 9.446269376334689e-05,
      "loss_nats": 1.0996825695037842,
      "step": 5530,
      "unclipped_grad_norm": 0.8849281668663025
    },
    {
      "bits_per_byte": 1.663044200671269,
      "learning_rate": 9.39874852751024e-05,
      "loss_nats": 1.152734398841858,
      "step": 5540,
      "unclipped_grad_norm": 1.0463494062423706
    },
    {
      "bits_per_byte": 1.588076447212218,
      "learning_rate": 9.351349019480433e-05,
      "loss_nats": 1.1007707118988037,
      "step": 5550,
      "unclipped_grad_norm": 0.8995247483253479
    },
    {
      "bits_per_byte": 1.5728786843207734,
      "learning_rate": 9.304071662172086e-05,
      "loss_nats": 1.0902364253997803,
      "step": 5560,
      "unclipped_grad_norm": 0.9297628998756409
    },
    {
      "bits_per_byte": 1.553519629209052,
      "learning_rate": 9.256917263424787e-05,
      "loss_nats": 1.0768177509307861,
      "step": 5570,
      "unclipped_grad_norm": 0.9231326580047607
    },
    {
      "bits_per_byte": 1.6281497807244594,
      "learning_rate": 9.209886628977114e-05,
      "loss_nats": 1.1285474300384521,
      "step": 5580,
      "unclipped_grad_norm": 0.932182788848877
    },
    {
      "bits_per_byte": 1.5666377778862637,
      "learning_rate": 9.162980562452843e-05,
      "loss_nats": 1.0859105587005615,
      "step": 5590,
      "unclipped_grad_norm": 0.9885890483856201
    },
    {
      "bits_per_byte": 1.6130727496539203,
      "learning_rate": 9.116199865347238e-05,
      "loss_nats": 1.1180968284606934,
      "step": 5600,
      "unclipped_grad_norm": 0.9734658598899841
    },
    {
      "bits_per_byte": 1.6005983320388883,
      "learning_rate": 9.069545337013333e-05,
      "loss_nats": 1.1094502210617065,
      "step": 5610,
      "unclipped_grad_norm": 0.8992385864257812
    },
    {
      "bits_per_byte": 1.607953514068338,
      "learning_rate": 9.0230177746483e-05,
      "loss_nats": 1.1145484447479248,
      "step": 5620,
      "unclipped_grad_norm": 0.9051816463470459
    },
    {
      "bits_per_byte": 1.594605252604272,
      "learning_rate": 8.976617973279797e-05,
      "loss_nats": 1.1052961349487305,
      "step": 5630,
      "unclipped_grad_norm": 0.9290134906768799
    },
    {
      "bits_per_byte": 1.5677630603708592,
      "learning_rate": 8.930346725752419e-05,
      "loss_nats": 1.0866905450820923,
      "step": 5640,
      "unclipped_grad_norm": 0.9029556512832642
    },
    {
      "bits_per_byte": 1.5944298303003919,
      "learning_rate": 8.88420482271411e-05,
      "loss_nats": 1.1051745414733887,
      "step": 5650,
      "unclipped_grad_norm": 0.9501181244850159
    },
    {
      "bits_per_byte": 1.676524028863245,
      "learning_rate": 8.838193052602691e-05,
      "loss_nats": 1.1620779037475586,
      "step": 5660,
      "unclipped_grad_norm": 0.9740756750106812
    },
    {
      "bits_per_byte": 1.5742370032972879,
      "learning_rate": 8.792312201632352e-05,
      "loss_nats": 1.0911779403686523,
      "step": 5670,
      "unclipped_grad_norm": 0.8980613350868225
    },
    {
      "bits_per_byte": 1.5945870224432805,
      "learning_rate": 8.746563053780261e-05,
      "loss_nats": 1.105283498764038,
      "step": 5680,
      "unclipped_grad_norm": 0.9082752466201782
    },
    {
      "bits_per_byte": 1.6598031876157593,
      "learning_rate": 8.700946390773114e-05,
      "loss_nats": 1.1504878997802734,
      "step": 5690,
      "unclipped_grad_norm": 0.9417368769645691
    },
    {
      "bits_per_byte": 1.5763857545371678,
      "learning_rate": 8.655462992073833e-05,
      "loss_nats": 1.0926673412322998,
      "step": 5700,
      "unclipped_grad_norm": 0.9450368881225586
    },
    {
      "bits_per_byte": 1.6005926566114097,
      "learning_rate": 8.610113634868199e-05,
      "loss_nats": 1.1094462871551514,
      "step": 5710,
      "unclipped_grad_norm": 0.9505442976951599
    },
    {
      "bits_per_byte": 1.6062563892696238,
      "learning_rate": 8.564899094051613e-05,
      "loss_nats": 1.1133720874786377,
      "step": 5720,
      "unclipped_grad_norm": 0.8568877577781677
    },
    {
      "bits_per_byte": 1.6007586198694923,
      "learning_rate": 8.519820142215816e-05,
      "loss_nats": 1.1095613241195679,
      "step": 5730,
      "unclipped_grad_norm": 0.885662853717804
    },
    {
      "bits_per_byte": 1.596347780822814,
      "learning_rate": 8.474877549635728e-05,
      "loss_nats": 1.106503963470459,
      "step": 5740,
      "unclipped_grad_norm": 0.8908458948135376
    },
    {
      "bits_per_byte": 1.6113288455741714,
      "learning_rate": 8.430072084256247e-05,
      "loss_nats": 1.1168880462646484,
      "step": 5750,
      "unclipped_grad_norm": 0.9814403057098389
    },
    {
      "bits_per_byte": 1.6378140018243932,
      "learning_rate": 8.385404511679161e-05,
      "loss_nats": 1.1352461576461792,
      "step": 5760,
      "unclipped_grad_norm": 1.000325322151184
    },
    {
      "bits_per_byte": 1.57939407506606,
      "learning_rate": 8.340875595150034e-05,
      "loss_nats": 1.094752550125122,
      "step": 5770,
      "unclipped_grad_norm": 0.9007365703582764
    },
    {
      "bits_per_byte": 1.5515046804715436,
      "learning_rate": 8.296486095545203e-05,
      "loss_nats": 1.0754210948944092,
      "step": 5780,
      "unclipped_grad_norm": 0.9067476987838745
    },
    {
      "bits_per_byte": 1.586475632697987,
      "learning_rate": 8.252236771358729e-05,
      "loss_nats": 1.099661111831665,
      "step": 5790,
      "unclipped_grad_norm": 0.9168080687522888
    },
    {
      "bits_per_byte": 1.544505330546729,
      "learning_rate": 8.208128378689484e-05,
      "loss_nats": 1.0705695152282715,
      "step": 5800,
      "unclipped_grad_norm": 0.9089300036430359
    },
    {
      "bits_per_byte": 1.6342659997370934,
      "learning_rate": 8.16416167122819e-05,
      "loss_nats": 1.1327868700027466,
      "step": 5810,
      "unclipped_grad_norm": 0.9730345010757446
    },
    {
      "bits_per_byte": 1.5628627586998252,
      "learning_rate": 8.120337400244578e-05,
      "loss_nats": 1.0832939147949219,
      "step": 5820,
      "unclipped_grad_norm": 0.9135822057723999
    },
    {
      "bits_per_byte": 1.5753394120893185,
      "learning_rate": 8.076656314574521e-05,
      "loss_nats": 1.0919420719146729,
      "step": 5830,
      "unclipped_grad_norm": 0.9162984490394592
    },
    {
      "bits_per_byte": 1.5812992988723182,
      "learning_rate": 8.03311916060725e-05,
      "loss_nats": 1.0960731506347656,
      "step": 5840,
      "unclipped_grad_norm": 0.946280300617218
    },
    {
      "bits_per_byte": 1.5446322537430657,
      "learning_rate": 7.989726682272606e-05,
      "loss_nats": 1.07065749168396,
      "step": 5850,
      "unclipped_grad_norm": 0.9933390021324158
    },
    {
      "bits_per_byte": 1.596009318965916,
      "learning_rate": 7.946479621028318e-05,
      "loss_nats": 1.106269359588623,
      "step": 5860,
      "unclipped_grad_norm": 1.0133196115493774
    },
    {
      "bits_per_byte": 1.5986716104012721,
      "learning_rate": 7.903378715847343e-05,
      "loss_nats": 1.1081147193908691,
      "step": 5870,
      "unclipped_grad_norm": 0.9077220559120178
    },
    {
      "bits_per_byte": 1.616603209510832,
      "learning_rate": 7.860424703205224e-05,
      "loss_nats": 1.1205439567565918,
      "step": 5880,
      "unclipped_grad_norm": 0.9455923438072205
    },
    {
      "bits_per_byte": 1.6185034458202152,
      "learning_rate": 7.817618317067534e-05,
      "loss_nats": 1.1218611001968384,
      "step": 5890,
      "unclipped_grad_norm": 0.9350056648254395
    },
    {
      "bits_per_byte": 1.6333419369540076,
      "learning_rate": 7.774960288877295e-05,
      "loss_nats": 1.1321463584899902,
      "step": 5900,
      "unclipped_grad_norm": 1.0195506811141968
    },
    {
      "bits_per_byte": 1.6111236702716922,
      "learning_rate": 7.732451347542522e-05,
      "loss_nats": 1.1167458295822144,
      "step": 5910,
      "unclipped_grad_norm": 0.9529072642326355
    },
    {
      "bits_per_byte": 1.5460519705259381,
      "learning_rate": 7.690092219423722e-05,
      "loss_nats": 1.0716415643692017,
      "step": 5920,
      "unclipped_grad_norm": 0.8979566097259521
    },
    {
      "bits_per_byte": 1.6630168554297817,
      "learning_rate": 7.647883628321536e-05,
      "loss_nats": 1.1527154445648193,
      "step": 5930,
      "unclipped_grad_norm": 0.9750632047653198
    },
    {
      "bits_per_byte": 1.5225558727477122,
      "learning_rate": 7.605826295464316e-05,
      "loss_nats": 1.0553553104400635,
      "step": 5940,
      "unclipped_grad_norm": 0.8854946494102478
    },
    {
      "bits_per_byte": 1.5768313615855534,
      "learning_rate": 7.563920939495851e-05,
      "loss_nats": 1.0929762125015259,
      "step": 5950,
      "unclipped_grad_norm": 0.8891376852989197
    },
    {
      "bits_per_byte": 1.6173250206965033,
      "learning_rate": 7.522168276463047e-05,
      "loss_nats": 1.1210442781448364,
      "step": 5960,
      "unclipped_grad_norm": 1.0107448101043701
    },
    {
      "bits_per_byte": 1.598686744874548,
      "learning_rate": 7.480569019803719e-05,
      "loss_nats": 1.1081252098083496,
      "step": 5970,
      "unclipped_grad_norm": 1.0125014781951904
    },
    {
      "bits_per_byte": 1.623099338199222,
      "learning_rate": 7.439123880334388e-05,
      "loss_nats": 1.125046730041504,
      "step": 5980,
      "unclipped_grad_norm": 0.9798391461372375
    },
    {
      "bits_per_byte": 1.6025701131310304,
      "learning_rate": 7.39783356623813e-05,
      "loss_nats": 1.1108169555664062,
      "step": 5990,
      "unclipped_grad_norm": 0.9963014125823975
    },
    {
      "bits_per_byte": 1.561719762002191,
      "learning_rate": 7.356698783052497e-05,
      "loss_nats": 1.0825016498565674,
      "step": 6000,
      "unclipped_grad_norm": 0.9239053130149841
    },
    {
      "bits_per_byte": 1.5905753551292547,
      "learning_rate": 7.315720233657431e-05,
      "loss_nats": 1.1025028228759766,
      "step": 6010,
      "unclipped_grad_norm": 0.9512903094291687
    },
    {
      "bits_per_byte": 1.5463665267943663,
      "learning_rate": 7.274898618263286e-05,
      "loss_nats": 1.07185959815979,
      "step": 6020,
      "unclipped_grad_norm": 0.931431770324707
    },
    {
      "bits_per_byte": 1.5669568057036143,
      "learning_rate": 7.234234634398831e-05,
      "loss_nats": 1.0861316919326782,
      "step": 6030,
      "unclipped_grad_norm": 0.9379295110702515
    },
    {
      "bits_per_byte": 1.6392313108501535,
      "learning_rate": 7.193728976899362e-05,
      "loss_nats": 1.1362285614013672,
      "step": 6040,
      "unclipped_grad_norm": 1.0010037422180176
    },
    {
      "bits_per_byte": 1.547184132316568,
      "learning_rate": 7.153382337894797e-05,
      "loss_nats": 1.0724263191223145,
      "step": 6050,
      "unclipped_grad_norm": 0.9731131792068481
    },
    {
      "bits_per_byte": 1.550029757257744,
      "learning_rate": 7.113195406797887e-05,
      "loss_nats": 1.0743987560272217,
      "step": 6060,
      "unclipped_grad_norm": 0.9350199699401855
    },
    {
      "bits_per_byte": 1.6229744787946958,
      "learning_rate": 7.073168870292397e-05,
      "loss_nats": 1.12496018409729,
      "step": 6070,
      "unclipped_grad_norm": 1.0221160650253296
    },
    {
      "bits_per_byte": 1.5381166910151283,
      "learning_rate": 7.033303412321404e-05,
      "loss_nats": 1.0661412477493286,
      "step": 6080,
      "unclipped_grad_norm": 0.9525545239448547
    },
    {
      "bits_per_byte": 1.54977178328145,
      "learning_rate": 6.993599714075586e-05,
      "loss_nats": 1.0742199420928955,
      "step": 6090,
      "unclipped_grad_norm": 0.9721336364746094
    },
    {
      "bits_per_byte": 1.5613978104797757,
      "learning_rate": 6.954058453981609e-05,
      "loss_nats": 1.0822784900665283,
      "step": 6100,
      "unclipped_grad_norm": 0.9061561822891235
    },
    {
      "bits_per_byte": 1.6307178256671429,
      "learning_rate": 6.914680307690504e-05,
      "loss_nats": 1.1303274631500244,
      "step": 6110,
      "unclipped_grad_norm": 0.9887846112251282
    },
    {
      "bits_per_byte": 1.6160071176429416,
      "learning_rate": 6.875465948066139e-05,
      "loss_nats": 1.1201307773590088,
      "step": 6120,
      "unclipped_grad_norm": 1.008078694343567
    },
    {
      "bits_per_byte": 1.5519858879286577,
      "learning_rate": 6.836416045173733e-05,
      "loss_nats": 1.0757546424865723,
      "step": 6130,
      "unclipped_grad_norm": 0.958514928817749
    },
    {
      "bits_per_byte": 1.5641512527200894,
      "learning_rate": 6.797531266268377e-05,
      "loss_nats": 1.0841870307922363,
      "step": 6140,
      "unclipped_grad_norm": 0.9408689141273499
    },
    {
      "bits_per_byte": 1.6176376851557719,
      "learning_rate": 6.758812275783663e-05,
      "loss_nats": 1.1212610006332397,
      "step": 6150,
      "unclipped_grad_norm": 0.9648155570030212
    },
    {
      "bits_per_byte": 1.4733855169180825,
      "learning_rate": 6.720259735320298e-05,
      "loss_nats": 1.0212730169296265,
      "step": 6160,
      "unclipped_grad_norm": 0.9190199375152588
    },
    {
      "bits_per_byte": 1.587719411229027,
      "learning_rate": 6.681874303634842e-05,
      "loss_nats": 1.1005232334136963,
      "step": 6170,
      "unclipped_grad_norm": 0.9420804977416992
    },
    {
      "bits_per_byte": 1.561453876823957,
      "learning_rate": 6.643656636628402e-05,
      "loss_nats": 1.0823173522949219,
      "step": 6180,
      "unclipped_grad_norm": 0.9441719055175781
    },
    {
      "bits_per_byte": 1.573854341899118,
      "learning_rate": 6.605607387335466e-05,
      "loss_nats": 1.0909126996994019,
      "step": 6190,
      "unclipped_grad_norm": 0.9590060114860535
    },
    {
      "bits_per_byte": 1.5911162405662183,
      "learning_rate": 6.567727205912724e-05,
      "loss_nats": 1.1028777360916138,
      "step": 6200,
      "unclipped_grad_norm": 0.9321414232254028
    },
    {
      "bits_per_byte": 1.586496958546694,
      "learning_rate": 6.530016739627959e-05,
      "loss_nats": 1.0996758937835693,
      "step": 6210,
      "unclipped_grad_norm": 1.0076017379760742
    },
    {
      "bits_per_byte": 1.574202778749766,
      "learning_rate": 6.492476632848993e-05,
      "loss_nats": 1.0911542177200317,
      "step": 6220,
      "unclipped_grad_norm": 0.9875207543373108
    },
    {
      "bits_per_byte": 1.5328827430014202,
      "learning_rate": 6.455107527032678e-05,
      "loss_nats": 1.0625133514404297,
      "step": 6230,
      "unclipped_grad_norm": 0.9433984756469727
    },
    {
      "bits_per_byte": 1.5747416003949193,
      "learning_rate": 6.417910060713926e-05,
      "loss_nats": 1.0915277004241943,
      "step": 6240,
      "unclipped_grad_norm": 0.9963345527648926
    },
    {
      "bits_per_byte": 1.6134170589209478,
      "learning_rate": 6.380884869494799e-05,
      "loss_nats": 1.118335485458374,
      "step": 6250,
      "unclipped_grad_norm": 1.0228519439697266
    },
    {
      "bits_per_byte": 1.4956729206260448,
      "learning_rate": 6.34403258603367e-05,
      "loss_nats": 1.0367214679718018,
      "step": 6260,
      "unclipped_grad_norm": 0.9954123497009277
    },
    {
      "bits_per_byte": 1.574694133183281,
      "learning_rate": 6.30735384003437e-05,
      "loss_nats": 1.0914947986602783,
      "step": 6270,
      "unclipped_grad_norm": 0.9892840385437012
    },
    {
      "bits_per_byte": 1.5724050441002972,
      "learning_rate": 6.270849258235484e-05,
      "loss_nats": 1.0899081230163574,
      "step": 6280,
      "unclipped_grad_norm": 0.976605236530304
    },
    {
      "bits_per_byte": 1.528724890434161,
      "learning_rate": 6.234519464399581e-05,
      "loss_nats": 1.05963134765625,
      "step": 6290,
      "unclipped_grad_norm": 0.9648171067237854
    },
    {
      "bits_per_byte": 1.589885188751342,
      "learning_rate": 6.198365079302614e-05,
      "loss_nats": 1.1020244359970093,
      "step": 6300,
      "unclipped_grad_norm": 1.012677550315857
    },
    {
      "bits_per_byte": 1.549742718213454,
      "learning_rate": 6.16238672072326e-05,
      "loss_nats": 1.0741997957229614,
      "step": 6310,
      "unclipped_grad_norm": 0.9430392384529114
    },
    {
      "bits_per_byte": 1.5003391539092545,
      "learning_rate": 6.126585003432406e-05,
      "loss_nats": 1.0399558544158936,
      "step": 6320,
      "unclipped_grad_norm": 0.9601491689682007
    },
    {
      "bits_per_byte": 1.5786977173127166,
      "learning_rate": 6.090960539182614e-05,
      "loss_nats": 1.094269871711731,
      "step": 6330,
      "unclipped_grad_norm": 0.9872254133224487
    },
    {
      "bits_per_byte": 1.5486566477732553,
      "learning_rate": 6.0555139366976935e-05,
      "loss_nats": 1.0734469890594482,
      "step": 6340,
      "unclipped_grad_norm": 0.9915545582771301
    },
    {
      "bits_per_byte": 1.643350295338318,
      "learning_rate": 6.020245801662274e-05,
      "loss_nats": 1.1390836238861084,
      "step": 6350,
      "unclipped_grad_norm": 1.0489985942840576
    },
    {
      "bits_per_byte": 1.5515856843001,
      "learning_rate": 5.985156736711483e-05,
      "loss_nats": 1.0754772424697876,
      "step": 6360,
      "unclipped_grad_norm": 1.0075738430023193
    },
    {
      "bits_per_byte": 1.515523502153932,
      "learning_rate": 5.950247341420618e-05,
      "loss_nats": 1.050480842590332,
      "step": 6370,
      "unclipped_grad_norm": 0.9415894746780396
    },
    {
      "bits_per_byte": 1.5423135836441333,
      "learning_rate": 5.915518212294935e-05,
      "loss_nats": 1.0690503120422363,
      "step": 6380,
      "unclipped_grad_norm": 0.9571269154548645
    },
    {
      "bits_per_byte": 1.6015189551689568,
      "learning_rate": 5.880969942759425e-05,
      "loss_nats": 1.1100883483886719,
      "step": 6390,
      "unclipped_grad_norm": 0.962506115436554
    },
    {
      "bits_per_byte": 1.5875768376114616,
      "learning_rate": 5.846603123148688e-05,
      "loss_nats": 1.1004244089126587,
      "step": 6400,
      "unclipped_grad_norm": 0.9905776977539062
    },
    {
      "bits_per_byte": 1.5643576319011245,
      "learning_rate": 5.812418340696858e-05,
      "loss_nats": 1.0843300819396973,
      "step": 6410,
      "unclipped_grad_norm": 1.0040770769119263
    },
    {
      "bits_per_byte": 1.5604245606585432,
      "learning_rate": 5.778416179527541e-05,
      "loss_nats": 1.0816038846969604,
      "step": 6420,
      "unclipped_grad_norm": 0.9999781250953674
    },
    {
      "bits_per_byte": 1.592441538873767,
      "learning_rate": 5.7445972206438635e-05,
      "loss_nats": 1.103796362876892,
      "step": 6430,
      "unclipped_grad_norm": 0.9464544653892517
    },
    {
      "bits_per_byte": 1.5425199628251687,
      "learning_rate": 5.710962041918516e-05,
      "loss_nats": 1.0691933631896973,
      "step": 6440,
      "unclipped_grad_norm": 0.9930383563041687
    },
    {
      "bits_per_byte": 1.6124689185667411,
      "learning_rate": 5.677511218083908e-05,
      "loss_nats": 1.1176782846450806,
      "step": 6450,
      "unclipped_grad_norm": 1.0360738039016724
    },
    {
      "bits_per_byte": 1.6512981295826408,
      "learning_rate": 5.644245320722316e-05,
      "loss_nats": 1.1445926427841187,
      "step": 6460,
      "unclipped_grad_norm": 1.0094163417816162
    },
    {
      "bits_per_byte": 1.6379674103489628,
      "learning_rate": 5.61116491825615e-05,
      "loss_nats": 1.1353524923324585,
      "step": 6470,
      "unclipped_grad_norm": 1.026789665222168
    },
    {
      "bits_per_byte": 1.5269778906666966,
      "learning_rate": 5.5782705759382104e-05,
      "loss_nats": 1.0584204196929932,
      "step": 6480,
      "unclipped_grad_norm": 0.9595211744308472
    },
    {
      "bits_per_byte": 1.6320837118802953,
      "learning_rate": 5.5455628558420546e-05,
      "loss_nats": 1.1312742233276367,
      "step": 6490,
      "unclipped_grad_norm": 0.9658059477806091
    },
    {
      "bits_per_byte": 1.5804063649490385,
      "learning_rate": 5.513042316852368e-05,
      "loss_nats": 1.095454216003418,
      "step": 6500,
      "unclipped_grad_norm": 1.007287621498108
    },
    {
      "bits_per_byte": 1.5282990613906247,
      "learning_rate": 5.480709514655443e-05,
      "loss_nats": 1.0593361854553223,
      "step": 6510,
      "unclipped_grad_norm": 0.9717181921005249
    },
    {
      "bits_per_byte": 1.518525631307393,
      "learning_rate": 5.448565001729653e-05,
      "loss_nats": 1.0525617599487305,
      "step": 6520,
      "unclipped_grad_norm": 0.9680256843566895
    },
    {
      "bits_per_byte": 1.5436008737858413,
      "learning_rate": 5.41660932733604e-05,
      "loss_nats": 1.069942593574524,
      "step": 6530,
      "unclipped_grad_norm": 0.9684496521949768
    },
    {
      "bits_per_byte": 1.5757463230412598,
      "learning_rate": 5.384843037508902e-05,
      "loss_nats": 1.09222412109375,
      "step": 6540,
      "unclipped_grad_norm": 0.9695699214935303
    },
    {
      "bits_per_byte": 1.5133627121284916,
      "learning_rate": 5.353266675046498e-05,
      "loss_nats": 1.048983097076416,
      "step": 6550,
      "unclipped_grad_norm": 0.9651139974594116
    },
    {
      "bits_per_byte": 1.5985231893735776,
      "learning_rate": 5.321880779501729e-05,
      "loss_nats": 1.1080118417739868,
      "step": 6560,
      "unclipped_grad_norm": 0.9865891337394714
    },
    {
      "bits_per_byte": 1.5411050615565203,
      "learning_rate": 5.290685887172962e-05,
      "loss_nats": 1.068212628364563,
      "step": 6570,
      "unclipped_grad_norm": 1.0009613037109375
    },
    {
      "bits_per_byte": 1.585936467087532,
      "learning_rate": 5.2596825310948295e-05,
      "loss_nats": 1.0992873907089233,
      "step": 6580,
      "unclipped_grad_norm": 1.0042976140975952
    },
    {
      "bits_per_byte": 1.5806860087393415,
      "learning_rate": 5.228871241029154e-05,
      "loss_nats": 1.0956480503082275,
      "step": 6590,
      "unclipped_grad_norm": 1.0104939937591553
    },
    {
      "bits_per_byte": 1.4917766536707477,
      "learning_rate": 5.198252543455865e-05,
      "loss_nats": 1.0340207815170288,
      "step": 6600,
      "unclipped_grad_norm": 0.9221757054328918
    },
    {
      "bits_per_byte": 1.5136690132296782,
      "learning_rate": 5.167826961564027e-05,
      "loss_nats": 1.049195408821106,
      "step": 6610,
      "unclipped_grad_norm": 0.9053012132644653
    },
    {
      "bits_per_byte": 1.5109461839012182,
      "learning_rate": 5.137595015242884e-05,
      "loss_nats": 1.047308087348938,
      "step": 6620,
      "unclipped_grad_norm": 0.9886469841003418
    },
    {
      "bits_per_byte": 1.5485409034492248,
      "learning_rate": 5.1075572210729925e-05,
      "loss_nats": 1.0733667612075806,
      "step": 6630,
      "unclipped_grad_norm": 1.005003809928894
    },
    {
      "bits_per_byte": 1.5964243131024478,
      "learning_rate": 5.07771409231737e-05,
      "loss_nats": 1.106557011604309,
      "step": 6640,
      "unclipped_grad_norm": 1.033184289932251
    },
    {
      "bits_per_byte": 1.579020356765735,
      "learning_rate": 5.048066138912756e-05,
      "loss_nats": 1.0944935083389282,
      "step": 6650,
      "unclipped_grad_norm": 1.0069372653961182
    },
    {
      "bits_per_byte": 1.596521827265487,
      "learning_rate": 5.018613867460863e-05,
      "loss_nats": 1.1066246032714844,
      "step": 6660,
      "unclipped_grad_norm": 0.9888476729393005
    },
    {
      "bits_per_byte": 1.5643971879108232,
      "learning_rate": 4.989357781219755e-05,
      "loss_nats": 1.084357500076294,
      "step": 6670,
      "unclipped_grad_norm": 0.9964956045150757
    },
    {
      "bits_per_byte": 1.541306109275379,
      "learning_rate": 4.960298380095222e-05,
      "loss_nats": 1.0683519840240479,
      "step": 6680,
      "unclipped_grad_norm": 0.9876991510391235
    },
    {
      "bits_per_byte": 1.57202289865008,
      "learning_rate": 4.931436160632256e-05,
      "loss_nats": 1.0896432399749756,
      "step": 6690,
      "unclipped_grad_norm": 1.0232517719268799
    },
    {
      "bits_per_byte": 1.634657604233108,
      "learning_rate": 4.9027716160065485e-05,
      "loss_nats": 1.1330583095550537,
      "step": 6700,
      "unclipped_grad_norm": 1.028996467590332
    },
    {
      "bits_per_byte": 1.582099534146783,
      "learning_rate": 4.874305236016084e-05,
      "loss_nats": 1.0966278314590454,
      "step": 6710,
      "unclipped_grad_norm": 0.982172429561615
    },
    {
      "bits_per_byte": 1.5951110535804596,
      "learning_rate": 4.846037507072752e-05,
      "loss_nats": 1.1056467294692993,
      "step": 6720,
      "unclipped_grad_norm": 0.9665309190750122
    },
    {
      "bits_per_byte": 1.5713714283686118,
      "learning_rate": 4.817968912194049e-05,
      "loss_nats": 1.0891916751861572,
      "step": 6730,
      "unclipped_grad_norm": 1.033214807510376
    },
    {
      "bits_per_byte": 1.5070635035753395,
      "learning_rate": 4.790099930994813e-05,
      "loss_nats": 1.0446168184280396,
      "step": 6740,
      "unclipped_grad_norm": 0.9829891920089722
    },
    {
      "bits_per_byte": 1.5742997769648528,
      "learning_rate": 4.762431039679047e-05,
      "loss_nats": 1.0912214517593384,
      "step": 6750,
      "unclipped_grad_norm": 0.9734475612640381
    },
    {
      "bits_per_byte": 1.5504435475157199,
      "learning_rate": 4.734962711031755e-05,
      "loss_nats": 1.0746855735778809,
      "step": 6760,
      "unclipped_grad_norm": 0.947333574295044
    },
    {
      "bits_per_byte": 1.4574869247243718,
      "learning_rate": 4.707695414410892e-05,
      "loss_nats": 1.0102529525756836,
      "step": 6770,
      "unclipped_grad_norm": 0.9365482330322266
    },
    {
      "bits_per_byte": 1.5535751796052808,
      "learning_rate": 4.680629615739322e-05,
      "loss_nats": 1.076856255531311,
      "step": 6780,
      "unclipped_grad_norm": 1.000013828277588
    },
    {
      "bits_per_byte": 1.5272709491037668,
      "learning_rate": 4.653765777496863e-05,
      "loss_nats": 1.0586235523223877,
      "step": 6790,
      "unclipped_grad_norm": 1.0473871231079102
    },
    {
      "bits_per_byte": 1.588691973119656,
      "learning_rate": 4.6271043587123986e-05,
      "loss_nats": 1.101197361946106,
      "step": 6800,
      "unclipped_grad_norm": 1.0225850343704224
    },
    {
      "bits_per_byte": 1.5961993597951194,
      "learning_rate": 4.600645814956006e-05,
      "loss_nats": 1.1064010858535767,
      "step": 6810,
      "unclipped_grad_norm": 1.0318177938461304
    },
    {
      "bits_per_byte": 1.5861701915100546,
      "learning_rate": 4.574390598331204e-05,
      "loss_nats": 1.0994493961334229,
      "step": 6820,
      "unclipped_grad_norm": 1.0743839740753174
    },
    {
      "bits_per_byte": 1.5382866098741805,
      "learning_rate": 4.548339157467199e-05,
      "loss_nats": 1.0662590265274048,
      "step": 6830,
      "unclipped_grad_norm": 0.9585179090499878
    },
    {
      "bits_per_byte": 1.521101759434667,
      "learning_rate": 4.52249193751124e-05,
      "loss_nats": 1.0543473958969116,
      "step": 6840,
      "unclipped_grad_norm": 1.0282230377197266
    },
    {
      "bits_per_byte": 1.5590901472704985,
      "learning_rate": 4.4968493801209985e-05,
      "loss_nats": 1.080678939819336,
      "step": 6850,
      "unclipped_grad_norm": 1.064353346824646
    },
    {
      "bits_per_byte": 1.5486480486407121,
      "learning_rate": 4.471411923457034e-05,
      "loss_nats": 1.0734410285949707,
      "step": 6860,
      "unclipped_grad_norm": 1.054447889328003
    },
    {
      "bits_per_byte": 1.486493862584195,
      "learning_rate": 4.446180002175291e-05,
      "loss_nats": 1.0303590297698975,
      "step": 6870,
      "unclipped_grad_norm": 0.992324709892273
    },
    {
      "bits_per_byte": 1.501989499426934,
      "learning_rate": 4.421154047419693e-05,
      "loss_nats": 1.0410997867584229,
      "step": 6880,
      "unclipped_grad_norm": 0.9187514781951904
    },
    {
      "bits_per_byte": 1.4984072727921125,
      "learning_rate": 4.396334486814753e-05,
      "loss_nats": 1.0386167764663696,
      "step": 6890,
      "unclipped_grad_norm": 0.9562593102455139
    },
    {
      "bits_per_byte": 1.6084883801125214,
      "learning_rate": 4.371721744458285e-05,
      "loss_nats": 1.1149191856384277,
      "step": 6900,
      "unclipped_grad_norm": 1.046886920928955
    },
    {
      "bits_per_byte": 1.5851809473022918,
      "learning_rate": 4.347316240914145e-05,
      "loss_nats": 1.0987637042999268,
      "step": 6910,
      "unclipped_grad_norm": 1.0160750150680542
    },
    {
      "bits_per_byte": 1.5136987662282775,
      "learning_rate": 4.3231183932050546e-05,
      "loss_nats": 1.0492160320281982,
      "step": 6920,
      "unclipped_grad_norm": 0.9592021107673645
    },
    {
      "bits_per_byte": 1.5198106856746398,
      "learning_rate": 4.299128614805464e-05,
      "loss_nats": 1.053452491760254,
      "step": 6930,
      "unclipped_grad_norm": 1.0287657976150513
    },
    {
      "bits_per_byte": 1.494010192357503,
      "learning_rate": 4.2753473156344965e-05,
      "loss_nats": 1.0355689525604248,
      "step": 6940,
      "unclipped_grad_norm": 0.972965657711029
    },
    {
      "bits_per_byte": 1.513553956836251,
      "learning_rate": 4.251774902048942e-05,
      "loss_nats": 1.0491156578063965,
      "step": 6950,
      "unclipped_grad_norm": 1.0125614404678345
    },
    {
      "bits_per_byte": 1.583024456843123,
      "learning_rate": 4.228411776836306e-05,
      "loss_nats": 1.0972689390182495,
      "step": 6960,
      "unclipped_grad_norm": 0.9876657724380493
    },
    {
      "bits_per_byte": 1.5602042508827878,
      "learning_rate": 4.2052583392079427e-05,
      "loss_nats": 1.081451177597046,
      "step": 6970,
      "unclipped_grad_norm": 1.0233051776885986
    },
    {
      "bits_per_byte": 1.5255848311947078,
      "learning_rate": 4.182314984792214e-05,
      "loss_nats": 1.0574548244476318,
      "step": 6980,
      "unclipped_grad_norm": 0.9819255471229553
    },
    {
      "bits_per_byte": 1.5587054220805185,
      "learning_rate": 4.159582105627748e-05,
      "loss_nats": 1.0804122686386108,
      "step": 6990,
      "unclipped_grad_norm": 1.0355700254440308
    },
    {
      "bits_per_byte": 1.6012561656784385,
      "learning_rate": 4.137060090156724e-05,
      "loss_nats": 1.1099061965942383,
      "step": 7000,
      "unclipped_grad_norm": 1.0185670852661133
    },
    {
      "bits_per_byte": 1.585546754400677,
      "learning_rate": 4.114749323218249e-05,
      "loss_nats": 1.0990172624588013,
      "step": 7010,
      "unclipped_grad_norm": 1.014304757118225
    },
    {
      "bits_per_byte": 1.542434143482388,
      "learning_rate": 4.0926501860417706e-05,
      "loss_nats": 1.0691338777542114,
      "step": 7020,
      "unclipped_grad_norm": 1.027226448059082
    },
    {
      "bits_per_byte": 1.5806457647990395,
      "learning_rate": 4.070763056240571e-05,
      "loss_nats": 1.0956201553344727,
      "step": 7030,
      "unclipped_grad_norm": 1.0283117294311523
    },
    {
      "bits_per_byte": 1.5888914729946568,
      "learning_rate": 4.049088307805306e-05,
      "loss_nats": 1.1013356447219849,
      "step": 7040,
      "unclipped_grad_norm": 1.0418747663497925
    },
    {
      "bits_per_byte": 1.5596535624347252,
      "learning_rate": 4.027626311097629e-05,
      "loss_nats": 1.0810694694519043,
      "step": 7050,
      "unclipped_grad_norm": 1.007202386856079
    },
    {
      "bits_per_byte": 1.5766827685752078,
      "learning_rate": 4.006377432843846e-05,
      "loss_nats": 1.092873215675354,
      "step": 7060,
      "unclipped_grad_norm": 1.057618498802185
    },
    {
      "bits_per_byte": 1.5601829250340808,
      "learning_rate": 3.985342036128656e-05,
      "loss_nats": 1.0814363956451416,
      "step": 7070,
      "unclipped_grad_norm": 1.0584709644317627
    },
    {
      "bits_per_byte": 1.5180239579148262,
      "learning_rate": 3.964520480388956e-05,
      "loss_nats": 1.0522140264511108,
      "step": 7080,
      "unclipped_grad_norm": 1.0032520294189453
    },
    {
      "bits_per_byte": 1.5539740073726318,
      "learning_rate": 3.943913121407678e-05,
      "loss_nats": 1.0771327018737793,
      "step": 7090,
      "unclipped_grad_norm": 1.0915608406066895
    },
    {
      "bits_per_byte": 1.5426179929361605,
      "learning_rate": 3.923520311307735e-05,
      "loss_nats": 1.0692613124847412,
      "step": 7100,
      "unclipped_grad_norm": 1.0186309814453125
    },
    {
      "bits_per_byte": 1.5505398578002032,
      "learning_rate": 3.9033423985459816e-05,
      "loss_nats": 1.0747523307800293,
      "step": 7110,
      "unclipped_grad_norm": 1.0057092905044556
    },
    {
      "bits_per_byte": 1.58839977459584,
      "learning_rate": 3.88337972790728e-05,
      "loss_nats": 1.1009948253631592,
      "step": 7120,
      "unclipped_grad_norm": 1.0634574890136719
    },
    {
      "bits_per_byte": 1.5622621952830122,
      "learning_rate": 3.8636326404985845e-05,
      "loss_nats": 1.0828776359558105,
      "step": 7130,
      "unclipped_grad_norm": 1.0760761499404907
    },
    {
      "bits_per_byte": 1.5468659644124718,
      "learning_rate": 3.844101473743143e-05,
      "loss_nats": 1.0722057819366455,
      "step": 7140,
      "unclipped_grad_norm": 1.0254970788955688
    },
    {
      "bits_per_byte": 1.5017244741619544,
      "learning_rate": 3.824786561374701e-05,
      "loss_nats": 1.040916085243225,
      "step": 7150,
      "unclipped_grad_norm": 0.9808151125907898
    },
    {
      "bits_per_byte": 1.5967245948108544,
      "learning_rate": 3.805688233431824e-05,
      "loss_nats": 1.1067651510238647,
      "step": 7160,
      "unclipped_grad_norm": 1.0412861108779907
    },
    {
      "bits_per_byte": 1.5266782968888937,
      "learning_rate": 3.786806816252236e-05,
      "loss_nats": 1.0582127571105957,
      "step": 7170,
      "unclipped_grad_norm": 0.993831217288971
    },
    {
      "bits_per_byte": 1.5582240426407534,
      "learning_rate": 3.768142632467269e-05,
      "loss_nats": 1.0800786018371582,
      "step": 7180,
      "unclipped_grad_norm": 1.0170890092849731
    },
    {
      "bits_per_byte": 1.5191605912543786,
      "learning_rate": 3.7496960009963236e-05,
      "loss_nats": 1.053001880645752,
      "step": 7190,
      "unclipped_grad_norm": 1.0318375825881958
    },
    {
      "bits_per_byte": 1.5658617921655709,
      "learning_rate": 3.7314672370414325e-05,
      "loss_nats": 1.0853726863861084,
      "step": 7200,
      "unclipped_grad_norm": 1.0288457870483398
    },
    {
      "bits_per_byte": 1.6065680218329872,
      "learning_rate": 3.713456652081882e-05,
      "loss_nats": 1.1135880947113037,
      "step": 7210,
      "unclipped_grad_norm": 1.079895257949829
    },
    {
      "bits_per_byte": 1.5943379915648312,
      "learning_rate": 3.695664553868873e-05,
      "loss_nats": 1.1051108837127686,
      "step": 7220,
      "unclipped_grad_norm": 1.0698199272155762
    },
    {
      "bits_per_byte": 1.5270222621906193,
      "learning_rate": 3.678091246420277e-05,
      "loss_nats": 1.0584511756896973,
      "step": 7230,
      "unclipped_grad_norm": 1.0323591232299805
    },
    {
      "bits_per_byte": 1.5416120664112638,
      "learning_rate": 3.660737030015427e-05,
      "loss_nats": 1.0685640573501587,
      "step": 7240,
      "unclipped_grad_norm": 1.0111463069915771
    },
    {
      "bits_per_byte": 1.5732295289285334,
      "learning_rate": 3.6436022011900056e-05,
      "loss_nats": 1.0904796123504639,
      "step": 7250,
      "unclipped_grad_norm": 1.0152740478515625
    },
    {
      "bits_per_byte": 1.467654711026032,
      "learning_rate": 3.626687052730954e-05,
      "loss_nats": 1.0173007249832153,
      "step": 7260,
      "unclipped_grad_norm": 0.9989895820617676
    },
    {
      "bits_per_byte": 1.5079978853174771,
      "learning_rate": 3.6099918736714955e-05,
      "loss_nats": 1.045264482498169,
      "step": 7270,
      "unclipped_grad_norm": 1.015647053718567
    },
    {
      "bits_per_byte": 1.5841817281007788,
      "learning_rate": 3.5935169492861716e-05,
      "loss_nats": 1.0980710983276367,
      "step": 7280,
      "unclipped_grad_norm": 1.0212509632110596
    },
    {
      "bits_per_byte": 1.50208838945118,
      "learning_rate": 3.5772625610859886e-05,
      "loss_nats": 1.0411683320999146,
      "step": 7290,
      "unclipped_grad_norm": 1.0158213376998901
    },
    {
      "bits_per_byte": 1.529626423489984,
      "learning_rate": 3.5612289868135887e-05,
      "loss_nats": 1.0602562427520752,
      "step": 7300,
      "unclipped_grad_norm": 1.0379971265792847
    },
    {
      "bits_per_byte": 1.530171608493219,
      "learning_rate": 3.545416500438522e-05,
      "loss_nats": 1.0606341361999512,
      "step": 7310,
      "unclipped_grad_norm": 0.9976150393486023
    },
    {
      "bits_per_byte": 1.4838797262910803,
      "learning_rate": 3.52982537215255e-05,
      "loss_nats": 1.0285470485687256,
      "step": 7320,
      "unclipped_grad_norm": 1.0280481576919556
    },
    {
      "bits_per_byte": 1.510586568178264,
      "learning_rate": 3.5144558683650396e-05,
      "loss_nats": 1.0470588207244873,
      "step": 7330,
      "unclipped_grad_norm": 1.0713517665863037
    },
    {
      "bits_per_byte": 1.4810440763436543,
      "learning_rate": 3.499308251698404e-05,
      "loss_nats": 1.0265815258026123,
      "step": 7340,
      "unclipped_grad_norm": 1.0121647119522095
    },
    {
      "bits_per_byte": 1.5558935057389116,
      "learning_rate": 3.484382780983616e-05,
      "loss_nats": 1.0784631967544556,
      "step": 7350,
      "unclipped_grad_norm": 1.046762228012085
    },
    {
      "bits_per_byte": 1.5163350882833535,
      "learning_rate": 3.469679711255795e-05,
      "loss_nats": 1.0510433912277222,
      "step": 7360,
      "unclipped_grad_norm": 1.0365986824035645
    },
    {
      "bits_per_byte": 1.536221614185271,
      "learning_rate": 3.4551992937498284e-05,
      "loss_nats": 1.0648276805877686,
      "step": 7370,
      "unclipped_grad_norm": 1.0269999504089355
    },
    {
      "bits_per_byte": 1.4635044256954106,
      "learning_rate": 3.440941775896104e-05,
      "loss_nats": 1.0144239664077759,
      "step": 7380,
      "unclipped_grad_norm": 1.0192774534225464
    },
    {
      "bits_per_byte": 1.5685326827334702,
      "learning_rate": 3.4269074013162596e-05,
      "loss_nats": 1.087224006652832,
      "step": 7390,
      "unclipped_grad_norm": 1.030569076538086
    },
    {
      "bits_per_byte": 1.49844063742638,
      "learning_rate": 3.41309640981904e-05,
      "loss_nats": 1.0386399030685425,
      "step": 7400,
      "unclipped_grad_norm": 1.0422602891921997
    },
    {
      "bits_per_byte": 1.48524905215725,
      "learning_rate": 3.3995090373961795e-05,
      "loss_nats": 1.029496192932129,
      "step": 7410,
      "unclipped_grad_norm": 0.9939574003219604
    },
    {
      "bits_per_byte": 1.556964785671136,
      "learning_rate": 3.3861455162183876e-05,
      "loss_nats": 1.0792057514190674,
      "step": 7420,
      "unclipped_grad_norm": 1.0415644645690918
    },
    {
      "bits_per_byte": 1.5795878995135824,
      "learning_rate": 3.3730060746313664e-05,
      "loss_nats": 1.0948868989944458,
      "step": 7430,
      "unclipped_grad_norm": 1.0722742080688477
    },
    {
      "bits_per_byte": 1.591624105334216,
      "learning_rate": 3.3600909371519225e-05,
      "loss_nats": 1.1032297611236572,
      "step": 7440,
      "unclipped_grad_norm": 1.0902073383331299
    },
    {
      "bits_per_byte": 1.5369473809719119,
      "learning_rate": 3.3474003244641194e-05,
      "loss_nats": 1.0653307437896729,
      "step": 7450,
      "unclipped_grad_norm": 1.0743108987808228
    },
    {
      "bits_per_byte": 1.5231069051610766,
      "learning_rate": 3.334934453415514e-05,
      "loss_nats": 1.0557372570037842,
      "step": 7460,
      "unclipped_grad_norm": 0.9955658912658691
    },
    {
      "bits_per_byte": 1.5091140527215767,
      "learning_rate": 3.3226935370134486e-05,
      "loss_nats": 1.0460381507873535,
      "step": 7470,
      "unclipped_grad_norm": 1.048521876335144
    },
    {
      "bits_per_byte": 1.5543542610136893,
      "learning_rate": 3.31067778442141e-05,
      "loss_nats": 1.077396273612976,
      "step": 7480,
      "unclipped_grad_norm": 1.0729283094406128
    },
    {
      "bits_per_byte": 1.6078190236353633,
      "learning_rate": 3.298887400955457e-05,
      "loss_nats": 1.114455223083496,
      "step": 7490,
      "unclipped_grad_norm": 1.0465319156646729
    },
    {
      "bits_per_byte": 1.5529930183321103,
      "learning_rate": 3.287322588080714e-05,
      "loss_nats": 1.0764527320861816,
      "step": 7500,
      "unclipped_grad_norm": 1.0308523178100586
    },
    {
      "bits_per_byte": 1.5173444544612673,
      "learning_rate": 3.275983543407924e-05,
      "loss_nats": 1.0517430305480957,
      "step": 7510,
      "unclipped_grad_norm": 1.008348822593689
    },
    {
      "bits_per_byte": 1.5327847128904286,
      "learning_rate": 3.2648704606900735e-05,
      "loss_nats": 1.0624454021453857,
      "step": 7520,
      "unclipped_grad_norm": 1.0445557832717896
    },
    {
      "bits_per_byte": 1.5854733178087586,
      "learning_rate": 3.253983529819086e-05,
      "loss_nats": 1.098966360092163,
      "step": 7530,
      "unclipped_grad_norm": 1.1247494220733643
    },
    {
      "bits_per_byte": 1.5176457680655788,
      "learning_rate": 3.243322936822571e-05,
      "loss_nats": 1.0519518852233887,
      "step": 7540,
      "unclipped_grad_norm": 0.9896391034126282
    },
    {
      "bits_per_byte": 1.5701451920679599,
      "learning_rate": 3.232888863860648e-05,
      "loss_nats": 1.0883417129516602,
      "step": 7550,
      "unclipped_grad_norm": 1.0925025939941406
    },
    {
      "bits_per_byte": 1.5354839805957203,
      "learning_rate": 3.222681489222838e-05,
      "loss_nats": 1.0643163919448853,
      "step": 7560,
      "unclipped_grad_norm": 1.032513976097107
    },
    {
      "bits_per_byte": 1.4933951823980176,
      "learning_rate": 3.212700987325009e-05,
      "loss_nats": 1.0351426601409912,
      "step": 7570,
      "unclipped_grad_norm": 1.0785413980484009
    },
    {
      "bits_per_byte": 1.6648412473901344,
      "learning_rate": 3.202947528706403e-05,
      "loss_nats": 1.153980016708374,
      "step": 7580,
      "unclipped_grad_norm": 1.1147191524505615
    },
    {
      "bits_per_byte": 1.5229436936254077,
      "learning_rate": 3.1934212800267116e-05,
      "loss_nats": 1.0556241273880005,
      "step": 7590,
      "unclipped_grad_norm": 1.0130250453948975
    },
    {
      "bits_per_byte": 1.506402230282772,
      "learning_rate": 3.1841224040632484e-05,
      "loss_nats": 1.0441584587097168,
      "step": 7600,
      "unclipped_grad_norm": 1.0460206270217896
    },
    {
      "bits_per_byte": 1.5204523529450091,
      "learning_rate": 3.175051059708143e-05,
      "loss_nats": 1.0538972616195679,
      "step": 7610,
      "unclipped_grad_norm": 1.0537638664245605
    },
    {
      "bits_per_byte": 1.4833304137042245,
      "learning_rate": 3.1662074019656414e-05,
      "loss_nats": 1.0281662940979004,
      "step": 7620,
      "unclipped_grad_norm": 1.0038108825683594
    },
    {
      "bits_per_byte": 1.4865265392878588,
      "learning_rate": 3.157591581949455e-05,
      "loss_nats": 1.030381679534912,
      "step": 7630,
      "unclipped_grad_norm": 1.040962815284729
    },
    {
      "bits_per_byte": 1.4985199214284277,
      "learning_rate": 3.149203746880175e-05,
      "loss_nats": 1.0386948585510254,
      "step": 7640,
      "unclipped_grad_norm": 1.037416696548462
    },
    {
      "bits_per_byte": 1.6218830768923203,
      "learning_rate": 3.141044040082763e-05,
      "loss_nats": 1.1242036819458008,
      "step": 7650,
      "unclipped_grad_norm": 1.0509169101715088
    },
    {
      "bits_per_byte": 1.5436070651612726,
      "learning_rate": 3.133112600984091e-05,
      "loss_nats": 1.0699468851089478,
      "step": 7660,
      "unclipped_grad_norm": 1.0892055034637451
    },
    {
      "bits_per_byte": 1.5646802713541432,
      "learning_rate": 3.125409565110571e-05,
      "loss_nats": 1.0845537185668945,
      "step": 7670,
      "unclipped_grad_norm": 1.0592975616455078
    },
    {
      "bits_per_byte": 1.5230315767599987,
      "learning_rate": 3.117935064085835e-05,
      "loss_nats": 1.055685043334961,
      "step": 7680,
      "unclipped_grad_norm": 1.0794378519058228
    },
    {
      "bits_per_byte": 1.5199706295399424,
      "learning_rate": 3.110689225628479e-05,
      "loss_nats": 1.0535633563995361,
      "step": 7690,
      "unclipped_grad_norm": 1.030026912689209
    },
    {
      "bits_per_byte": 1.486342001903483,
      "learning_rate": 3.1036721735498926e-05,
      "loss_nats": 1.0302537679672241,
      "step": 7700,
      "unclipped_grad_norm": 1.021731972694397
    },
    {
      "bits_per_byte": 1.565274987360827,
      "learning_rate": 3.096884027752132e-05,
      "loss_nats": 1.0849659442901611,
      "step": 7710,
      "unclipped_grad_norm": 1.054081678390503
    },
    {
      "bits_per_byte": 1.5620626954080112,
      "learning_rate": 3.090324904225885e-05,
      "loss_nats": 1.0827393531799316,
      "step": 7720,
      "unclipped_grad_norm": 1.0361971855163574
    },
    {
      "bits_per_byte": 1.5781238112067872,
      "learning_rate": 3.083994915048472e-05,
      "loss_nats": 1.0938720703125,
      "step": 7730,
      "unclipped_grad_norm": 1.0852437019348145
    },
    {
      "bits_per_byte": 1.5098900384422698,
      "learning_rate": 3.077894168381942e-05,
      "loss_nats": 1.0465760231018066,
      "step": 7740,
      "unclipped_grad_norm": 1.0850290060043335
    },
    {
      "bits_per_byte": 1.5288445903591616,
      "learning_rate": 3.0720227684712254e-05,
      "loss_nats": 1.0597143173217773,
      "step": 7750,
      "unclipped_grad_norm": 1.0632920265197754
    },
    {
      "bits_per_byte": 1.5461653070928567,
      "learning_rate": 3.0663808156423456e-05,
      "loss_nats": 1.0717201232910156,
      "step": 7760,
      "unclipped_grad_norm": 1.0411477088928223
    },
    {
      "bits_per_byte": 1.4760610510175551,
      "learning_rate": 3.060968406300711e-05,
      "loss_nats": 1.023127555847168,
      "step": 7770,
      "unclipped_grad_norm": 1.0089881420135498
    },
    {
      "bits_per_byte": 1.5534400012417027,
      "learning_rate": 3.055785632929463e-05,
      "loss_nats": 1.0767625570297241,
      "step": 7780,
      "unclipped_grad_norm": 1.0804110765457153
    },
    {
      "bits_per_byte": 1.4972104455247583,
      "learning_rate": 3.0508325840878984e-05,
      "loss_nats": 1.0377871990203857,
      "step": 7790,
      "unclipped_grad_norm": 1.0377298593521118
    },
    {
      "bits_per_byte": 1.5201942069860641,
      "learning_rate": 3.046109344409957e-05,
      "loss_nats": 1.0537183284759521,
      "step": 7800,
      "unclipped_grad_norm": 1.0282031297683716
    },
    {
      "bits_per_byte": 1.4946203868027643,
      "learning_rate": 3.0416159946027727e-05,
      "loss_nats": 1.035991907119751,
      "step": 7810,
      "unclipped_grad_norm": 1.041825771331787
    },
    {
      "bits_per_byte": 1.4985438270168976,
      "learning_rate": 3.0373526114452946e-05,
      "loss_nats": 1.038711428642273,
      "step": 7820,
      "unclipped_grad_norm": 1.0459965467453003
    },
    {
      "bits_per_byte": 1.5248091894393165,
      "learning_rate": 3.0333192677869803e-05,
      "loss_nats": 1.0569171905517578,
      "step": 7830,
      "unclipped_grad_norm": 1.093446969985962
    },
    {
      "bits_per_byte": 1.5216629388244325,
      "learning_rate": 3.029516032546544e-05,
      "loss_nats": 1.0547363758087158,
      "step": 7840,
      "unclipped_grad_norm": 1.0666427612304688
    },
    {
      "bits_per_byte": 1.4598855667559556,
      "learning_rate": 3.02594297071078e-05,
      "loss_nats": 1.0119155645370483,
      "step": 7850,
      "unclipped_grad_norm": 1.0127599239349365
    },
    {
      "bits_per_byte": 1.522777730367325,
      "learning_rate": 3.0226001433334603e-05,
      "loss_nats": 1.055509090423584,
      "step": 7860,
      "unclipped_grad_norm": 1.0349164009094238
    },
    {
      "bits_per_byte": 1.5484055531029957,
      "learning_rate": 3.019487607534278e-05,
      "loss_nats": 1.073272943496704,
      "step": 7870,
      "unclipped_grad_norm": 1.0840545892715454
    },
    {
      "bits_per_byte": 1.4683410937856254,
      "learning_rate": 3.016605416497886e-05,
      "loss_nats": 1.0177764892578125,
      "step": 7880,
      "unclipped_grad_norm": 1.0721038579940796
    },
    {
      "bits_per_byte": 1.5719193650942607,
      "learning_rate": 3.0139536194729734e-05,
      "loss_nats": 1.089571475982666,
      "step": 7890,
      "unclipped_grad_norm": 1.1230442523956299
    },
    {
      "bits_per_byte": 1.5165904825198848,
      "learning_rate": 3.0115322617714362e-05,
      "loss_nats": 1.051220417022705,
      "step": 7900,
      "unclipped_grad_norm": 1.0280076265335083
    },
    {
      "bits_per_byte": 1.5320310849143477,
      "learning_rate": 3.0093413847675965e-05,
      "loss_nats": 1.0619230270385742,
      "step": 7910,
      "unclipped_grad_norm": 1.0358502864837646
    },
    {
      "bits_per_byte": 1.5396064047369018,
      "learning_rate": 3.0073810258974985e-05,
      "loss_nats": 1.0671738386154175,
      "step": 7920,
      "unclipped_grad_norm": 1.0826281309127808
    },
    {
      "bits_per_byte": 1.4911463372553357,
      "learning_rate": 3.0056512186582658e-05,
      "loss_nats": 1.0335838794708252,
      "step": 7930,
      "unclipped_grad_norm": 1.0188233852386475
    },
    {
      "bits_per_byte": 1.500506321045893,
      "learning_rate": 3.0041519926075316e-05,
      "loss_nats": 1.040071725845337,
      "step": 7940,
      "unclipped_grad_norm": 1.0400398969650269
    },
    {
      "bits_per_byte": 1.561352235077297,
      "learning_rate": 3.0028833733629313e-05,
      "loss_nats": 1.0822468996047974,
      "step": 7950,
      "unclipped_grad_norm": 1.0635569095611572
    },
    {
      "bits_per_byte": 1.4866159702663075,
      "learning_rate": 3.0018453826016686e-05,
      "loss_nats": 1.0304436683654785,
      "step": 7960,
      "unclipped_grad_norm": 1.0361822843551636
    },
    {
      "bits_per_byte": 1.5208446453716271,
      "learning_rate": 3.001038038060143e-05,
      "loss_nats": 1.0541691780090332,
      "step": 7970,
      "unclipped_grad_norm": 1.0433354377746582
    },
    {
      "bits_per_byte": 1.555866504462726,
      "learning_rate": 3.0004613535336433e-05,
      "loss_nats": 1.078444480895996,
      "step": 7980,
      "unclipped_grad_norm": 1.1136473417282104
    },
    {
      "bits_per_byte": 1.613144638401981,
      "learning_rate": 3.000115338876117e-05,
      "loss_nats": 1.1181466579437256,
      "step": 7990,
      "unclipped_grad_norm": 1.1455442905426025
    },
    {
      "bits_per_byte": 1.4895616891102856,
      "learning_rate": 3e-05,
      "loss_nats": 1.0324854850769043,
      "step": 8000,
      "unclipped_grad_norm": 1.039122223854065
    }
  ]
}