llm-atlas/experiments/k3/attnres_gradient/results/raw/formal-depth-16-baseline-seed-2026073003.json

{
  "architecture": "baseline",
  "batch_size": 32,
  "canonical_sha256_without_self": "14cb0082a87883b57c59c7a7d75d194347e04cee762e0db4354d50b7946c7443",
  "depth": 16,
  "diagnostics": [
    {
      "activation_grad_rms_by_block": [
        0.00031658337684348226,
        0.0002727926184888929,
        0.00024126221251208335,
        0.00021358189405873418,
        0.0001949657453224063,
        0.00017900438979268074,
        0.00016492439317516983,
        0.00015548785449936986,
        0.00014800082135479897,
        0.0001436655584257096,
        0.00013764512550551444,
        0.0001334012922598049,
        0.00013005948858335614,
        0.00012698007049039006,
        0.00012402374704834074,
        0.000122041383292526
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.0002610550254757982,
        "first_to_last_ratio": 2.075552313213765,
        "imbalance_abs_log_ratio": 0.7302272933071482,
        "last_quartile_mean": 0.00012577617235365324,
        "mean": 0.00017527624822832877,
        "normalized": [
          1.8061966754963605,
          1.5563581560322512,
          1.3764683746413606,
          1.2185444189820027,
          1.1123340857252286,
          1.021270089941082,
          0.9409397727427746,
          0.887101681323182,
          0.8443860640033859,
          0.8196521769370567,
          0.7853039239304516,
          0.7610916687697798,
          0.7420257444918054,
          0.7244568033255466,
          0.7075901515576575,
          0.6962802121000742
        ],
        "population_cv": 0.32194911387722014
      },
      "activation_output_rms_by_block": [
        0.0284949392080307,
        0.028932223096489906,
        0.029410898685455322,
        0.029886392876505852,
        0.030463071539998055,
        0.031026815995573997,
        0.031805604696273804,
        0.03248070180416107,
        0.03315235301852226,
        0.034095361828804016,
        0.034913208335638046,
        0.03547592833638191,
        0.03669917955994606,
        0.03757266327738762,
        0.038674499839544296,
        0.039735738188028336
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.029181113466620445,
        "first_to_last_ratio": 0.7644934703880544,
        "imbalance_abs_log_ratio": 0.2685417946723145,
        "last_quartile_mean": 0.03817052021622658,
        "mean": 0.03330122376792133,
        "normalized": [
          0.8556724343409726,
          0.8688036000755015,
          0.8831777141411389,
          0.8974562942427076,
          0.9147733354274736,
          0.9317019762337311,
          0.9550881648653333,
          0.9753606062804618,
          0.9955295712122606,
          1.0238471134399472,
          1.0484061660601651,
          1.065304043587595,
          1.102036964638457,
          1.1282667429651916,
          1.1613537120758601,
          1.193221560413203
        ],
        "population_cv": 0.1043395636179365
      },
      "bits_per_byte": 8.022455452740537,
      "branch_output_rms_by_sublayer": [
        0.003436586121097207,
        0.0038391812704503536,
        0.003560001263394952,
        0.0038144157733768225,
        0.004080869723111391,
        0.0038664413150399923,
        0.004083882085978985,
        0.003824100596830249,
        0.004302335437387228,
        0.0039005596190690994,
        0.005134448409080505,
        0.003882436081767082,
        0.005223615560680628,
        0.0038928077556192875,
        0.00572318397462368,
        0.0038727927021682262,
        0.005704221315681934,
        0.003765099449083209,
        0.0069826096296310425,
        0.003909832797944546,
        0.006672552786767483,
        0.0038559692911803722,
        0.0066586704924702644,
        0.00382705288939178,
        0.007329727057367563,
        0.0038791403640061617,
        0.007807865273207426,
        0.00388751900754869,
        0.008622666820883751,
        0.0039123171009123325,
        0.008360983803868294,
        0.0037942787166684866
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.007247792155837073,
        0.0070599825250150985,
        0.007115739546591776,
        0.006247289691276946,
        0.006127370040903895,
        0.0065579111453454445,
        0.0059278656112939756,
        0.005972691938247438,
        0.0055663256066952145,
        0.006567633669083912,
        0.005568350992310226,
        0.005243662280059585,
        0.00519845242839581,
        0.005229357864659918,
        0.005505343111599652,
        0.004934663781812996
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.006917700979680223,
        "first_to_last_ratio": 1.3260037536012093,
        "imbalance_abs_log_ratio": 0.2821697225298069,
        "last_quartile_mean": 0.005216954296617094,
        "mean": 0.00600440202432056,
        "normalized": [
          1.2070797602292813,
          1.1758011033270186,
          1.1850871273725165,
          1.0404515996718042,
          1.0204796441153106,
          1.092183887551653,
          0.9872532830552356,
          0.9947188602720667,
          0.927040791763953,
          1.0938031201911544,
          0.9273781085536696,
          0.8733029965049588,
          0.8657735453655023,
          0.8709206751111334,
          0.9168844939596827,
          0.821841002955059
        ],
        "population_cv": 0.119259291012628
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.028142500668764114,
        0.02826075814664364,
        0.0284949392080307,
        0.028706196695566177,
        0.028932223096489906,
        0.029190780594944954,
        0.029410898685455322,
        0.02965042181313038,
        0.029886392876505852,
        0.03018481843173504,
        0.030463071539998055,
        0.030850933864712715,
        0.031026815995573997,
        0.031584836542606354,
        0.031805604696273804,
        0.03225463256239891,
        0.03248070180416107,
        0.03296073526144028,
        0.03315235301852226,
        0.03385457023978233,
        0.034095361828804016,
        0.03471403941512108,
        0.034913208335638046,
        0.03529877960681915,
        0.03547592833638191,
        0.03643115609884262,
        0.03669917955994606,
        0.03748895600438118,
        0.03757266327738762,
        0.03844280168414116,
        0.038674499839544296,
        0.03956866264343262
      ],
      "loss_nats": 5.560742378234863,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 0,
      "stream_state_rms_by_sublayer": [
        0.02826075814664364,
        0.0284949392080307,
        0.028706196695566177,
        0.028932223096489906,
        0.029190780594944954,
        0.029410898685455322,
        0.02965042181313038,
        0.029886392876505852,
        0.03018481843173504,
        0.030463071539998055,
        0.030850933864712715,
        0.031026815995573997,
        0.031584836542606354,
        0.031805604696273804,
        0.03225463256239891,
        0.03248070180416107,
        0.03296073526144028,
        0.03315235301852226,
        0.03385457023978233,
        0.034095361828804016,
        0.03471403941512108,
        0.034913208335638046,
        0.03529877960681915,
        0.03547592833638191,
        0.03643115609884262,
        0.03669917955994606,
        0.03748895600438118,
        0.03757266327738762,
        0.03844280168414116,
        0.038674499839544296,
        0.03956866264343262,
        0.039735738188028336
      ]
    },
    {
      "activation_grad_rms_by_block": [
        7.706054748268798e-05,
        6.966097134863958e-05,
        6.121626211097464e-05,
        5.34596256329678e-05,
        4.684355008066632e-05,
        4.049177005072124e-05,
        3.4754324587993324e-05,
        3.020181247848086e-05,
        2.6929572413791902e-05,
        2.4378721718676388e-05,
        2.2718366381013766e-05,
        2.1439738702611066e-05,
        2.0652638340834528e-05,
        2.0034611225128174e-05,
        1.9586463167797774e-05,
        1.9270755728939548e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 6.53493516438175e-05,
        "first_to_last_ratio": 3.2861795625404726,
        "imbalance_abs_log_ratio": 1.1897256628392332,
        "last_quartile_mean": 1.9886117115675006e-05,
        "mean": 3.6793733215745306e-05,
        "normalized": [
          2.0943932769972666,
          1.893283590990143,
          1.663768711699472,
          1.45295464636599,
          1.2731393633256098,
          1.1005072470709065,
          0.9445718482603106,
          0.8208412095991516,
          0.7319064976605258,
          0.6625780965396546,
          0.6174520603223757,
          0.5827008250806217,
          0.5613085853434561,
          0.5445115098175108,
          0.5323314993058669,
          0.5237510316211383
        ],
        "population_cv": 0.5070097402008534
      },
      "activation_output_rms_by_block": [
        0.03020417131483555,
        0.03345569595694542,
        0.03829808160662651,
        0.04440607130527496,
        0.05133094638586044,
        0.059150952845811844,
        0.06815594434738159,
        0.07766561955213547,
        0.08808834105730057,
        0.10039259493350983,
        0.11500746011734009,
        0.132473424077034,
        0.1519528478384018,
        0.17349907755851746,
        0.19798777997493744,
        0.22741958498954773
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.03659100504592061,
        "first_to_last_ratio": 0.19492869311536967,
        "imbalance_abs_log_ratio": 1.6351214636257212,
        "last_quartile_mean": 0.1877148225903511,
        "mean": 0.09934303711634129,
        "normalized": [
          0.30403913743308697,
          0.33676940959400337,
          0.3855134966507554,
          0.44699731953303096,
          0.5167040174717673,
          0.5954212249071846,
          0.6860666466997954,
          0.7817922806324186,
          0.8867087579992116,
          1.010564986209747,
          1.1576801299385893,
          1.3334948067059142,
          1.5295772330822623,
          1.746464398459366,
          1.9929708787045903,
          2.2892352759782764
        ],
        "population_cv": 0.6013462998472742
      },
      "bits_per_byte": 5.091903414776641,
      "branch_output_rms_by_sublayer": [
        0.0044672503136098385,
        0.008439179509878159,
        0.004589807707816362,
        0.008912671357393265,
        0.005376614164561033,
        0.009684479795396328,
        0.004654362332075834,
        0.010457071475684643,
        0.00613226555287838,
        0.009992391802370548,
        0.005980229936540127,
        0.011163929477334023,
        0.008545844815671444,
        0.010891783982515335,
        0.00866044219583273,
        0.011625969782471657,
        0.010422462597489357,
        0.011488622985780239,
        0.010880537331104279,
        0.012680526822805405,
        0.014226034283638,
        0.012272600084543228,
        0.01669861003756523,
        0.013891858980059624,
        0.01948261633515358,
        0.012507432140409946,
        0.01911107264459133,
        0.013616994023323059,
        0.02317427285015583,
        0.013388233259320259,
        0.027934540063142776,
        0.013881584629416466
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00027208547857035954,
        0.0002379277587721014,
        0.0002494129668009364,
        0.00019949675850635897,
        0.0002037090250478814,
        0.0001757519905869641,
        0.00017904243730119992,
        0.00016456038574532694,
        0.0001493779474782176,
        0.00013508785060359837,
        0.0001355896704444294,
        0.00013648423482290936,
        0.00013660889397982336,
        0.00013630230473358484,
        0.0001407085634636149,
        0.00014945333748469326
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00023973074066243908,
        "first_to_last_ratio": 1.703016825392403,
        "imbalance_abs_log_ratio": 0.532401281486357,
        "last_quartile_mean": 0.0001407682749154291,
        "mean": 0.00017509997527137498,
        "normalized": [
          1.5538864477203589,
          1.358810921608379,
          1.4244032097342618,
          1.1393305921212904,
          1.163386943557058,
          1.0037236745155367,
          1.0225154916425019,
          0.9398081609679643,
          0.8531009056209594,
          0.7714898325612858,
          0.774355737254038,
          0.7794646150656627,
          0.7801765463878734,
          0.7784256081266694,
          0.8035898534282527,
          0.8535314596879079
        ],
        "population_cv": 0.2495872477003493
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.028559360653162003,
        0.028688903898000717,
        0.03020417131483555,
        0.03060002811253071,
        0.03345569595694542,
        0.034075960516929626,
        0.03829808160662651,
        0.03887331485748291,
        0.04440607130527496,
        0.04547364264726639,
        0.05133094638586044,
        0.05236440151929855,
        0.059150952845811844,
        0.061456043273210526,
        0.06815594434738159,
        0.07020121067762375,
        0.07766561955213547,
        0.08064334839582443,
        0.08808834105730057,
        0.09190437197685242,
        0.10039259493350983,
        0.10653280466794968,
        0.11500746011734009,
        0.1223740428686142,
        0.132473424077034,
        0.14304450154304504,
        0.1519528478384018,
        0.1632441133260727,
        0.17349907755851746,
        0.1877153217792511,
        0.19798777997493744,
        0.21634350717067719
      ],
      "loss_nats": 3.5294384956359863,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 100,
      "stream_state_rms_by_sublayer": [
        0.028688903898000717,
        0.03020417131483555,
        0.03060002811253071,
        0.03345569595694542,
        0.034075960516929626,
        0.03829808160662651,
        0.03887331485748291,
        0.04440607130527496,
        0.04547364264726639,
        0.05133094638586044,
        0.05236440151929855,
        0.059150952845811844,
        0.061456043273210526,
        0.06815594434738159,
        0.07020121067762375,
        0.07766561955213547,
        0.08064334839582443,
        0.08808834105730057,
        0.09190437197685242,
        0.10039259493350983,
        0.10653280466794968,
        0.11500746011734009,
        0.1223740428686142,
        0.132473424077034,
        0.14304450154304504,
        0.1519528478384018,
        0.1632441133260727,
        0.17349907755851746,
        0.1877153217792511,
        0.19798777997493744,
        0.21634350717067719,
        0.22741958498954773
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.00015226140385493636,
        0.00015351864567492157,
        0.00014040769019629806,
        0.00012548675294965506,
        0.00010927267430815846,
        9.748537559062243e-05,
        8.759690535953268e-05,
        7.801534957252443e-05,
        7.03658297425136e-05,
        6.262787064770237e-05,
        5.5923766922205687e-05,
        4.834980791201815e-05,
        4.202092532068491e-05,
        3.7704347050748765e-05,
        3.55200027115643e-05,
        3.515387652441859e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00014291862316895276,
        "first_to_last_ratio": 3.801048653306501,
        "imbalance_abs_log_ratio": 1.3352769900584454,
        "last_quartile_mean": 3.759978790185414e-05,
        "mean": 8.323195152115659e-05,
        "normalized": [
          1.8293624151805847,
          1.8444676938266782,
          1.686944588348479,
          1.5076752455787101,
          1.3128693045288338,
          1.1712494277614383,
          1.0524432475581997,
          0.9373245267797649,
          0.845418477597842,
          0.752449864542502,
          0.671902627538302,
          0.5809044126488876,
          0.504865313773205,
          0.4530032801305249,
          0.4267592200158317,
          0.4223603541902161
        ],
        "population_cv": 0.48987170747402153
      },
      "activation_output_rms_by_block": [
        0.05162646621465683,
        0.06355457007884979,
        0.07474122941493988,
        0.08428152650594711,
        0.08889172971248627,
        0.09293187409639359,
        0.0973934531211853,
        0.09854210913181305,
        0.10090499371290207,
        0.10472240298986435,
        0.11040307581424713,
        0.1186341941356659,
        0.130356103181839,
        0.149788036942482,
        0.17963695526123047,
        0.2188950628042221
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.0685509480535984,
        "first_to_last_ratio": 0.40402744210990815,
        "imbalance_abs_log_ratio": 0.9062724773132301,
        "last_quartile_mean": 0.1696690395474434,
        "mean": 0.1103314864449203,
        "normalized": [
          0.4679214236856113,
          0.576032936079199,
          0.6774242949427878,
          0.7638936918340364,
          0.8056787103730612,
          0.8422969461468014,
          0.8827348951725225,
          0.8931458489957648,
          0.9145620798218456,
          0.9491615346100125,
          1.0006488571090046,
          1.0752523867689434,
          1.1814950326706182,
          1.3576182263914227,
          1.6281567578708265,
          1.9839763775275423
        ],
        "population_cv": 0.3749688391491054
      },
      "bits_per_byte": 3.129900568231996,
      "branch_output_rms_by_sublayer": [
        0.014640412293374538,
        0.020567819476127625,
        0.01939546875655651,
        0.012996131554245949,
        0.0218212753534317,
        0.010227516293525696,
        0.02053862437605858,
        0.010292341001331806,
        0.02414301224052906,
        0.009784159250557423,
        0.019980086013674736,
        0.010961603373289108,
        0.01658599078655243,
        0.011390469036996365,
        0.021502232179045677,
        0.012452049180865288,
        0.020629581063985825,
        0.014398678205907345,
        0.02130005694925785,
        0.015666954219341278,
        0.016051532700657845,
        0.0179822389036417,
        0.018369123339653015,
        0.02199208177626133,
        0.01977314054965973,
        0.026316894218325615,
        0.016924018040299416,
        0.03431078419089317,
        0.013718036003410816,
        0.04166935011744499,
        0.022049644961953163,
        0.04074697569012642
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0009002454290164998,
        0.0005806619682142476,
        0.0004904299887450306,
        0.00047545530221616405,
        0.00040490888378542537,
        0.0003357588676961225,
        0.0003168596431849841,
        0.00029935138955839515,
        0.00029222591787988014,
        0.00027100955094896284,
        0.0002328601817845373,
        0.0002299739939536432,
        0.0002128774552133389,
        0.00018778710115789697,
        0.00017922956471394958,
        0.0001849955243427181
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.0006116981720479855,
        "first_to_last_ratio": 3.1988832674328185,
        "imbalance_abs_log_ratio": 1.1628017699711202,
        "last_quartile_mean": 0.0001912224113569759,
        "mean": 0.0003496644226507373,
        "normalized": [
          2.5745983025436674,
          1.6606263909046375,
          1.4025733159444054,
          1.359747436161308,
          1.157992799827591,
          0.9602317134548619,
          0.9061821067837941,
          0.8561105167322174,
          0.8357324879224854,
          0.7750561206498867,
          0.6659533160945287,
          0.6576991511182508,
          0.6088050182502319,
          0.537049493724068,
          0.512575924525708,
          0.5290659053623568
        ],
        "population_cv": 0.5241885524055848
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.04364648833870888,
        0.045909684151411057,
        0.05162646621465683,
        0.0588536262512207,
        0.06355457007884979,
        0.07136592268943787,
        0.07474122941493988,
        0.08130704611539841,
        0.08428152650594711,
        0.08667806535959244,
        0.08889172971248627,
        0.09049459546804428,
        0.09293187409639359,
        0.09443560987710953,
        0.0973934531211853,
        0.09589134901762009,
        0.09854210913181305,
        0.09718576818704605,
        0.10090499371290207,
        0.09916814416646957,
        0.10472240298986435,
        0.10267213732004166,
        0.11040307581424713,
        0.10699481517076492,
        0.1186341941356659,
        0.11436477303504944,
        0.130356103181839,
        0.12692348659038544,
        0.149788036942482,
        0.15056778490543365,
        0.17963695526123047,
        0.18913158774375916
      ],
      "loss_nats": 2.1694817543029785,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 500,
      "stream_state_rms_by_sublayer": [
        0.045909684151411057,
        0.05162646621465683,
        0.0588536262512207,
        0.06355457007884979,
        0.07136592268943787,
        0.07474122941493988,
        0.08130704611539841,
        0.08428152650594711,
        0.08667806535959244,
        0.08889172971248627,
        0.09049459546804428,
        0.09293187409639359,
        0.09443560987710953,
        0.0973934531211853,
        0.09589134901762009,
        0.09854210913181305,
        0.09718576818704605,
        0.10090499371290207,
        0.09916814416646957,
        0.10472240298986435,
        0.10267213732004166,
        0.11040307581424713,
        0.10699481517076492,
        0.1186341941356659,
        0.11436477303504944,
        0.130356103181839,
        0.12692348659038544,
        0.149788036942482,
        0.15056778490543365,
        0.17963695526123047,
        0.18913158774375916,
        0.2188950628042221
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.0001594285131432116,
        0.00017026996647473425,
        0.00018808410095516592,
        0.00017561657296027988,
        0.0001699679996818304,
        0.00016689945186953992,
        0.00015736353816464543,
        0.00014705183275509626,
        0.0001390052493661642,
        0.00012381834676489234,
        0.000104674712929409,
        9.053898247657344e-05,
        7.686029857723042e-05,
        6.236408080440015e-05,
        4.532423918135464e-05,
        3.028271021321416e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00017334978838334791,
        "first_to_last_ratio": 3.227644485017083,
        "imbalance_abs_log_ratio": 1.171752609589353,
        "last_quartile_mean": 5.370783219404984e-05,
        "mean": 0.00012547191226985888,
        "normalized": [
          1.2706310939162266,
          1.3570365143437513,
          1.499013584415959,
          1.3996484933024078,
          1.3546298658162754,
          1.3301738122120965,
          1.2541734266884814,
          1.1719900501621776,
          1.1078594950174865,
          0.9868212297473388,
          0.834248167863097,
          0.7215876512812414,
          0.6125697551490491,
          0.4970361866349061,
          0.3612301618857412,
          0.24135051156376403
        ],
        "population_cv": 0.3906610370884368
      },
      "activation_output_rms_by_block": [
        0.08273108303546906,
        0.13354359567165375,
        0.1704990118741989,
        0.17198410630226135,
        0.17284893989562988,
        0.17146942019462585,
        0.16773979365825653,
        0.15899653732776642,
        0.14656992256641388,
        0.13979844748973846,
        0.1417049616575241,
        0.149912029504776,
        0.16512760519981384,
        0.19090111553668976,
        0.2512649893760681,
        0.390801340341568
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.13968944922089577,
        "first_to_last_ratio": 0.5598242338035286,
        "imbalance_abs_log_ratio": 0.5801324127278619,
        "last_quartile_mean": 0.24952376261353493,
        "mean": 0.17536830622702837,
        "normalized": [
          0.4717561845432151,
          0.7615035951751215,
          0.9722338975748231,
          0.9807023287298795,
          0.9856338560507224,
          0.9777674420407808,
          0.9565000499069877,
          0.906643513577263,
          0.8357834190213785,
          0.7971705406606262,
          0.8080420271270434,
          0.8548410641014165,
          0.9416046077678534,
          1.0885725000363118,
          1.4327844910059484,
          2.228460482680629
        ],
        "population_cv": 0.3692959780115816
      },
      "bits_per_byte": 2.102960171157105,
      "branch_output_rms_by_sublayer": [
        0.027171069756150246,
        0.04243958741426468,
        0.07314105331897736,
        0.007419656030833721,
        0.04879630357027054,
        0.0038894445169717073,
        0.024768881499767303,
        0.005487008485943079,
        0.02289147861301899,
        0.00865041371434927,
        0.025303618982434273,
        0.011297631077468395,
        0.03125236928462982,
        0.014728116802871227,
        0.035565298050642014,
        0.019090818241238594,
        0.043166082352399826,
        0.02257433719933033,
        0.04396531730890274,
        0.02886364422738552,
        0.04178466275334358,
        0.03710934892296791,
        0.035476140677928925,
        0.04854284226894379,
        0.026270829141139984,
        0.058104008436203,
        0.018585238605737686,
        0.06995242089033127,
        0.01513067539781332,
        0.10811682045459747,
        0.016200197860598564,
        0.20113423466682434
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0004582000765955337,
        0.0003576182203712263,
        0.0003389913908322493,
        0.00022415790052185902,
        0.00024709072048410314,
        0.00027475536769928975,
        0.00033936291802973487,
        0.00039410580789665003,
        0.0004814106753566502,
        0.0005191302840338224,
        0.0005153457080868455,
        0.0004830357094583275,
        0.00041821493179685683,
        0.0003600919520715316,
        0.0003136989944772832,
        0.0002645229655850915
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00034474189708021705,
        "first_to_last_ratio": 1.0165412954471984,
        "imbalance_abs_log_ratio": 0.016405978393597878,
        "last_quartile_mean": 0.00033913221098269076,
        "mean": 0.00037435835145606595,
        "normalized": [
          1.2239611452859689,
          0.9552831370804766,
          0.9055264548359695,
          0.5987789497683099,
          0.6600379543371862,
          0.7339367924627015,
          0.9065188921518208,
          1.052750142647483,
          1.2859621629494962,
          1.3867201893978494,
          1.3766106888824827,
          1.2903030146904997,
          1.1171513341967951,
          0.9618910615215468,
          0.8379644617440793,
          0.7066036180473336
        ],
        "population_cv": 0.2506116126131003
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.055472806096076965,
        0.06610024720430374,
        0.08273108303546906,
        0.13110069930553436,
        0.13354359567165375,
        0.1698971837759018,
        0.1704990118741989,
        0.17238332331180573,
        0.17198410630226135,
        0.17322638630867004,
        0.17284893989562988,
        0.1726299524307251,
        0.17146942019462585,
        0.170275017619133,
        0.16773979365825653,
        0.16162297129631042,
        0.15899653732776642,
        0.14726711809635162,
        0.14656992256641388,
        0.13564367592334747,
        0.13979844748973846,
        0.13359025120735168,
        0.1417049616575241,
        0.13732580840587616,
        0.149912029504776,
        0.14843478798866272,
        0.16512760519981384,
        0.1652906984090805,
        0.19090111553668976,
        0.191492959856987,
        0.2512649893760681,
        0.25225311517715454
      ],
      "loss_nats": 1.4576609134674072,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 2000,
      "stream_state_rms_by_sublayer": [
        0.06610024720430374,
        0.08273108303546906,
        0.13110069930553436,
        0.13354359567165375,
        0.1698971837759018,
        0.1704990118741989,
        0.17238332331180573,
        0.17198410630226135,
        0.17322638630867004,
        0.17284893989562988,
        0.1726299524307251,
        0.17146942019462585,
        0.170275017619133,
        0.16773979365825653,
        0.16162297129631042,
        0.15899653732776642,
        0.14726711809635162,
        0.14656992256641388,
        0.13564367592334747,
        0.13979844748973846,
        0.13359025120735168,
        0.1417049616575241,
        0.13732580840587616,
        0.149912029504776,
        0.14843478798866272,
        0.16512760519981384,
        0.1652906984090805,
        0.19090111553668976,
        0.191492959856987,
        0.2512649893760681,
        0.25225311517715454,
        0.390801340341568
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.00016188873269129544,
        0.00017867473070509732,
        0.00019946519751101732,
        0.00018539158918429166,
        0.00018016427929978818,
        0.0001743910543154925,
        0.00016342199523933232,
        0.00015197545872069895,
        0.0001410306867910549,
        0.00012255505134817213,
        0.00010388438386144117,
        8.946077286964282e-05,
        7.552531315013766e-05,
        6.115865835454315e-05,
        4.3820087739732116e-05,
        2.6790259653353132e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00018135506252292544,
        "first_to_last_ratio": 3.499469999703496,
        "imbalance_abs_log_ratio": 1.2526115283727475,
        "last_quartile_mean": 5.1823579724441515e-05,
        "mean": 0.00012872489071469317,
        "normalized": [
          1.257633483256217,
          1.3880355983452597,
          1.5495464506014887,
          1.4402155492615254,
          1.3996071645467982,
          1.3547578354679994,
          1.269544641537448,
          1.1806221615486823,
          1.0955976424453635,
          0.9520695699777604,
          0.8070263900374272,
          0.6949764911273015,
          0.58671879797927,
          0.475111363583098,
          0.3404165852962757,
          0.208120274988085
        ],
        "population_cv": 0.4148571258695616
      },
      "activation_output_rms_by_block": [
        0.09483804553747177,
        0.15792693197727203,
        0.18785393238067627,
        0.19029200077056885,
        0.19116424024105072,
        0.18977893888950348,
        0.18693965673446655,
        0.177143394947052,
        0.16181440651416779,
        0.1581212431192398,
        0.165044903755188,
        0.18009524047374725,
        0.1996685266494751,
        0.2291477918624878,
        0.29672855138778687,
        0.4953230619430542
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.15772772766649723,
        "first_to_last_ratio": 0.516772448690399,
        "imbalance_abs_log_ratio": 0.6601526392924331,
        "last_quartile_mean": 0.305216982960701,
        "mean": 0.20386755419895053,
        "normalized": [
          0.46519440481893015,
          0.774654567264559,
          0.9214508562620668,
          0.9334099362611985,
          0.9376883977059789,
          0.9308932931245242,
          0.9169662012623924,
          0.8689141126114707,
          0.7937231951890498,
          0.7756076916728606,
          0.8095692539388771,
          0.8833933441806815,
          0.9794031592424085,
          1.1240032420208796,
          1.4554966951642292,
          2.429631649279893
        ],
        "population_cv": 0.4175842970930077
      },
      "bits_per_byte": 1.9507894057257391,
      "branch_output_rms_by_sublayer": [
        0.03931046277284622,
        0.047202516347169876,
        0.08608254790306091,
        0.0076385182328522205,
        0.041328247636556625,
        0.004047342110425234,
        0.034119248390197754,
        0.0045016431249678135,
        0.029622135683894157,
        0.008596057072281837,
        0.02270515263080597,
        0.012007404118776321,
        0.02946399338543415,
        0.01667790301144123,
        0.038835883140563965,
        0.02453153021633625,
        0.05270649492740631,
        0.02996903844177723,
        0.05063421651721001,
        0.03910558298230171,
        0.04952584207057953,
        0.05079483985900879,
        0.03944792598485947,
        0.06468769162893295,
        0.0361105240881443,
        0.07707343250513077,
        0.028930658474564552,
        0.0911106988787651,
        0.02081119269132614,
        0.1352817416191101,
        0.02383771352469921,
        0.28301650285720825
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00041699002778082224,
        0.00029759414151732746,
        0.000265991514011251,
        0.0001953867836934275,
        0.00023560860135608575,
        0.00025046058190266666,
        0.00033611798182792255,
        0.0004636386291792892,
        0.0005491632897177901,
        0.0005951115480573408,
        0.0005882338016339408,
        0.0005513321353847482,
        0.0004945066783331371,
        0.00043544338635105514,
        0.0003756714670850321,
        0.0003149065782712711
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.0002939906167507071,
        "first_to_last_ratio": 0.7256661946909654,
        "imbalance_abs_log_ratio": 0.3206651568188065,
        "last_quartile_mean": 0.00040513202751012384,
        "mean": 0.00039788482163144424,
        "normalized": [
          1.0480169262829406,
          0.7479404222988562,
          0.6685138501152368,
          0.4910636774036378,
          0.5921527752428994,
          0.6294801115451073,
          0.8447620103972311,
          1.165258396332474,
          1.380206682592364,
          1.4956879873356546,
          1.4784022150480887,
          1.3856576210274245,
          1.2428387599846482,
          1.094395570470896,
          0.9441713949898092,
          0.7914515989327312
        ],
        "population_cv": 0.32327688011764555
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.059103816747665405,
        0.07607264071702957,
        0.09483804553747177,
        0.15534481406211853,
        0.15792693197727203,
        0.18721190094947815,
        0.18785393238067627,
        0.19058401882648468,
        0.19029200077056885,
        0.1914086490869522,
        0.19116424024105072,
        0.19123691320419312,
        0.18977893888950348,
        0.19008098542690277,
        0.18693965673446655,
        0.18065746128559113,
        0.177143394947052,
        0.16187894344329834,
        0.16181440651416779,
        0.15150673687458038,
        0.1581212431192398,
        0.15270332992076874,
        0.165044903755188,
        0.16355812549591064,
        0.18009524047374725,
        0.17952413856983185,
        0.1996685266494751,
        0.2009342908859253,
        0.2291477918624878,
        0.23023094236850739,
        0.29672855138778687,
        0.29810962080955505
      ],
      "loss_nats": 1.3521841764450073,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 4000,
      "stream_state_rms_by_sublayer": [
        0.07607264071702957,
        0.09483804553747177,
        0.15534481406211853,
        0.15792693197727203,
        0.18721190094947815,
        0.18785393238067627,
        0.19058401882648468,
        0.19029200077056885,
        0.1914086490869522,
        0.19116424024105072,
        0.19123691320419312,
        0.18977893888950348,
        0.19008098542690277,
        0.18693965673446655,
        0.18065746128559113,
        0.177143394947052,
        0.16187894344329834,
        0.16181440651416779,
        0.15150673687458038,
        0.1581212431192398,
        0.15270332992076874,
        0.165044903755188,
        0.16355812549591064,
        0.18009524047374725,
        0.17952413856983185,
        0.1996685266494751,
        0.2009342908859253,
        0.2291477918624878,
        0.23023094236850739,
        0.29672855138778687,
        0.29810962080955505,
        0.4953230619430542
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.00025817437563091516,
        0.00027957011479884386,
        0.0003037556598428637,
        0.0002888414019253105,
        0.00028409145306795835,
        0.00028240776737220585,
        0.00026678573340177536,
        0.0002519928675610572,
        0.00023668054200243205,
        0.00020713888807222247,
        0.00017311860574409366,
        0.0001455697201890871,
        0.00011939310206798837,
        9.376799425808713e-05,
        6.36211916571483e-05,
        3.1806663173483685e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.0002825853880494833,
        "first_to_last_ratio": 3.662935915109688,
        "imbalance_abs_log_ratio": 1.2982649884510844,
        "last_quartile_mean": 7.714723778917687e-05,
        "mean": 0.00020541975504784205,
        "normalized": [
          1.2568137644346165,
          1.3609699550743417,
          1.478707146603886,
          1.4061033314835745,
          1.3829801958521164,
          1.3747838775605266,
          1.2987345513075956,
          1.226721682646191,
          1.1521800420183999,
          1.0083688787574498,
          0.8427553898298366,
          0.7086451843698482,
          0.5812152878878755,
          0.4564701882554999,
          0.30971311226776116,
          0.15483741165048096
        ],
        "population_cv": 0.41926708997335715
      },
      "activation_output_rms_by_block": [
        0.0951872169971466,
        0.14984266459941864,
        0.17071209847927094,
        0.17147330939769745,
        0.17152385413646698,
        0.17004960775375366,
        0.1669105589389801,
        0.15593038499355316,
        0.1393769234418869,
        0.1344291716814041,
        0.13867342472076416,
        0.1521374136209488,
        0.16997921466827393,
        0.1961352825164795,
        0.25573307275772095,
        0.4589640200138092
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.1468038223683834,
        "first_to_last_ratio": 0.5433095785892573,
        "imbalance_abs_log_ratio": 0.6100759951930727,
        "last_quartile_mean": 0.2702028974890709,
        "mean": 0.18106613866984844,
        "normalized": [
          0.5257041305260761,
          0.8275576300472066,
          0.9428162534052995,
          0.9470203024009789,
          0.94729945309774,
          0.9391574206142317,
          0.9218209443529402,
          0.861179158836942,
          0.7697569764605372,
          0.7424313163629063,
          0.7658716629154935,
          0.8402311704363035,
          0.9387686505990491,
          1.0832245275529278,
          1.4123738134385158,
          2.534786588952852
        ],
        "population_cv": 0.4350736575110673
      },
      "bits_per_byte": 1.8670605120664352,
      "branch_output_rms_by_sublayer": [
        0.051385972648859024,
        0.036380402743816376,
        0.07075809687376022,
        0.0071685765869915485,
        0.030611267313361168,
        0.003892687615007162,
        0.023587750270962715,
        0.004152141511440277,
        0.018766717985272408,
        0.007657408714294434,
        0.01761440560221672,
        0.010538195259869099,
        0.02535838447511196,
        0.014725946821272373,
        0.03455089405179024,
        0.021837059408426285,
        0.046537015587091446,
        0.02783774584531784,
        0.04248657822608948,
        0.03665589541196823,
        0.044122546911239624,
        0.04793383553624153,
        0.034650206565856934,
        0.06113889440894127,
        0.031748466193675995,
        0.07430189102888107,
        0.0256944727152586,
        0.0880647599697113,
        0.019152166321873665,
        0.12767614424228668,
        0.024809958413243294,
        0.2864501476287842
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0005002075550754018,
        0.0003412904728571966,
        0.0002592425753327288,
        0.0002585649119364213,
        0.00030071075691208866,
        0.0003425364164715924,
        0.0005240790570344413,
        0.0006975962934822548,
        0.0008810870404936847,
        0.000990036099264455,
        0.0009754970927650687,
        0.0008971247584696028,
        0.0007874986100445877,
        0.000666994667011454,
        0.0005527789362036235,
        0.0004234912235697502
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00033982637880043714,
        "first_to_last_ratio": 0.5592092980363276,
        "imbalance_abs_log_ratio": 0.5812314608110569,
        "last_quartile_mean": 0.0006076908592073538,
        "mean": 0.0005874210291827721,
        "normalized": [
          0.8515315765445054,
          0.580998050635001,
          0.44132327998776366,
          0.4401696553086297,
          0.5119169079296352,
          0.5831190908301898,
          0.8921693827739654,
          1.1875575759566523,
          1.4999242395517653,
          1.6853943765714456,
          1.6606438045335101,
          1.5272261527948612,
          1.3406033678095697,
          1.135462698602033,
          0.9410268082718403,
          0.7209330318986312
        ],
        "population_cv": 0.42806100897327076
      },
      "depth_weights": [],
      "layer_input_rms_by_sublayer": [
        0.060872070491313934,
        0.0852167010307312,
        0.0951872169971466,
        0.14736860990524292,
        0.14984266459941864,
        0.1702466458082199,
        0.17071209847927094,
        0.17185615003108978,
        0.17147330939769745,
        0.1719079613685608,
        0.17152385413646698,
        0.17163211107254028,
        0.17004960775375366,
        0.16988061368465424,
        0.1669105589389801,
        0.1597038209438324,
        0.15593038499355316,
        0.14019693434238434,
        0.1393769234418869,
        0.12975449860095978,
        0.1344291716814041,
        0.1281583607196808,
        0.13867342472076416,
        0.13766533136367798,
        0.1521374136209488,
        0.15170666575431824,
        0.16997921466827393,
        0.1708848923444748,
        0.1961352825164795,
        0.19691267609596252,
        0.25573307275772095,
        0.25681471824645996
      ],
      "loss_nats": 1.2941477298736572,
      "loss_scale": 1.0,
      "output_weights": null,
      "step": 8000,
      "stream_state_rms_by_sublayer": [
        0.0852167010307312,
        0.0951872169971466,
        0.14736860990524292,
        0.14984266459941864,
        0.1702466458082199,
        0.17071209847927094,
        0.17185615003108978,
        0.17147330939769745,
        0.1719079613685608,
        0.17152385413646698,
        0.17163211107254028,
        0.17004960775375366,
        0.16988061368465424,
        0.1669105589389801,
        0.1597038209438324,
        0.15593038499355316,
        0.14019693434238434,
        0.1393769234418869,
        0.12975449860095978,
        0.1344291716814041,
        0.1281583607196808,
        0.13867342472076416,
        0.13766533136367798,
        0.1521374136209488,
        0.15170666575431824,
        0.16997921466827393,
        0.1708848923444748,
        0.1961352825164795,
        0.19691267609596252,
        0.25573307275772095,
        0.25681471824645996,
        0.4589640200138092
      ]
    }
  ],
  "environment": {
    "autocast": "cuda-bfloat16-forward-fp32-cross-entropy",
    "compile": false,
    "compute_capability": [
      12,
      0
    ],
    "cublas_workspace_config": ":4096:8",
    "cuda": "12.8",
    "deterministic_algorithms": true,
    "gpu": "NVIDIA GeForce RTX 5090",
    "python": "3.10.14",
    "torch": "2.11.0+cu128"
  },
  "evaluations": [
    {
      "bits_per_byte": 8.018429854831792,
      "cross_entropy_nats": 5.557952046394348,
      "step": 0
    },
    {
      "bits_per_byte": 5.108882101013433,
      "cross_entropy_nats": 3.5412072241306305,
      "step": 100
    },
    {
      "bits_per_byte": 3.154257052216826,
      "cross_entropy_nats": 2.186364382505417,
      "step": 500
    },
    {
      "bits_per_byte": 1.977161870343983,
      "cross_entropy_nats": 1.37046417593956,
      "step": 2000
    },
    {
      "bits_per_byte": 1.8329381437371801,
      "cross_entropy_nats": 1.2704959064722061,
      "step": 4000
    },
    {
      "bits_per_byte": 1.7363699927668699,
      "cross_entropy_nats": 1.2035599648952484,
      "step": 8000
    }
  ],
  "gradient_gate": null,
  "hashes": {
    "final_mixer_parameters": null,
    "final_model_state": "d6cc15df5d5c50ac7916b8ae7c6c1c714b608e0e89d62192a5693d9718c19ea7",
    "final_optimizer_state": "de12f865acf312961efff64f13ff04f15f5b52fc60a03097833a902e65a3e5f3",
    "final_public_parameters": "d6cc15df5d5c50ac7916b8ae7c6c1c714b608e0e89d62192a5693d9718c19ea7",
    "initial_mixer_parameters": null,
    "initial_public_parameter_elements": 9541824,
    "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe",
    "initial_public_parameter_tensors": 115,
    "initial_public_parameters": "7a565cd353efdb3b95e9b8b1844581082c029991ea18d438b4b18566970c8c61"
  },
  "manifest": {
    "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716",
    "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371",
    "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
    "input_gate_tensor_hashes": {
      "0": "3eff5c07fa0cfbd5ceed5922cbd09525ad514f801cc0a440c03fd2fa15df4c84",
      "1": "e83e4001c5811b19c69a54c7ea7a802af996a34aa57072cde43a76bc8369e2d0",
      "7999": "d04c4ca879f0209183a48ac54a0c4ffe3a0f62a78888479d4641f6c304426442"
    },
    "path": "experiments/k3/attnres_gradient/manifest.json",
    "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338"
  },
  "model": {
    "attnres_aggregation_groups": 8,
    "context": 256,
    "d_ff": 768,
    "d_head": 32,
    "d_model": 192,
    "heads": 6,
    "layers": 16,
    "parameters": {
      "core": 9541824,
      "embedding": 98304,
      "mixer": 0,
      "total": 9541824
    },
    "sublayers": 32,
    "sublayers_per_attnres_group": 4,
    "transformer_blocks_per_attnres_group": 2,
    "vocabulary": 256
  },
  "optimizer": {
    "betas": [
      0.9,
      0.95
    ],
    "epsilon": 1e-08,
    "grad_clip": 1.0,
    "min_lr": 3e-05,
    "name": "AdamW",
    "peak_lr": 0.0003,
    "warmup_steps": 400,
    "weight_decay_ndim_ge_2": 0.1
  },
  "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
  "run_kind": "formal",
  "schema_version": 1,
  "seed": 2026073003,
  "steps": 8000,
  "target_bytes_seen": 65536000,
  "timing": {
    "mean_ms": 21.482708441361407,
    "measured_steps": 7980,
    "median_ms": 21.420384509838186,
    "p95_ms": 21.788083588762674,
    "peak_allocated_bytes": 3036005376,
    "peak_reserved_bytes": 3282042880,
    "warmup_steps_excluded": 20
  },
  "training_history": [
    {
      "bits_per_byte": 8.034161279888863,
      "learning_rate": 7.499999999999999e-07,
      "loss_nats": 5.568856239318848,
      "step": 1,
      "unclipped_grad_norm": 18.562814712524414
    },
    {
      "bits_per_byte": 7.448981367230313,
      "learning_rate": 7.499999999999999e-06,
      "loss_nats": 5.163240432739258,
      "step": 10,
      "unclipped_grad_norm": 12.974101066589355
    },
    {
      "bits_per_byte": 6.770661154309292,
      "learning_rate": 1.4999999999999999e-05,
      "loss_nats": 4.6930646896362305,
      "step": 20,
      "unclipped_grad_norm": 4.495212554931641
    },
    {
      "bits_per_byte": 6.551721736316109,
      "learning_rate": 2.2499999999999998e-05,
      "loss_nats": 4.54130744934082,
      "step": 30,
      "unclipped_grad_norm": 3.012619972229004
    },
    {
      "bits_per_byte": 6.419676896636662,
      "learning_rate": 2.9999999999999997e-05,
      "loss_nats": 4.4497809410095215,
      "step": 40,
      "unclipped_grad_norm": 2.8561618328094482
    },
    {
      "bits_per_byte": 6.250720452498435,
      "learning_rate": 3.75e-05,
      "loss_nats": 4.332669258117676,
      "step": 50,
      "unclipped_grad_norm": 2.8167924880981445
    },
    {
      "bits_per_byte": 5.9407609363626195,
      "learning_rate": 4.4999999999999996e-05,
      "loss_nats": 4.11782169342041,
      "step": 60,
      "unclipped_grad_norm": 2.718289375305176
    },
    {
      "bits_per_byte": 5.7053297580129,
      "learning_rate": 5.2499999999999995e-05,
      "loss_nats": 3.9546332359313965,
      "step": 70,
      "unclipped_grad_norm": 2.7082417011260986
    },
    {
      "bits_per_byte": 5.509554339299127,
      "learning_rate": 5.9999999999999995e-05,
      "loss_nats": 3.818932056427002,
      "step": 80,
      "unclipped_grad_norm": 2.5339207649230957
    },
    {
      "bits_per_byte": 5.312337705971124,
      "learning_rate": 6.749999999999999e-05,
      "loss_nats": 3.682231903076172,
      "step": 90,
      "unclipped_grad_norm": 2.2857937812805176
    },
    {
      "bits_per_byte": 5.088000784463262,
      "learning_rate": 7.5e-05,
      "loss_nats": 3.5267333984375,
      "step": 100,
      "unclipped_grad_norm": 2.166426658630371
    },
    {
      "bits_per_byte": 4.935865619440625,
      "learning_rate": 8.249999999999999e-05,
      "loss_nats": 3.421281337738037,
      "step": 110,
      "unclipped_grad_norm": 2.0095791816711426
    },
    {
      "bits_per_byte": 4.7340020148863085,
      "learning_rate": 8.999999999999999e-05,
      "loss_nats": 3.281360149383545,
      "step": 120,
      "unclipped_grad_norm": 1.7170522212982178
    },
    {
      "bits_per_byte": 4.584287677657216,
      "learning_rate": 9.75e-05,
      "loss_nats": 3.177586078643799,
      "step": 130,
      "unclipped_grad_norm": 1.5826245546340942
    },
    {
      "bits_per_byte": 4.439357553809826,
      "learning_rate": 0.00010499999999999999,
      "loss_nats": 3.0771281719207764,
      "step": 140,
      "unclipped_grad_norm": 1.4182634353637695
    },
    {
      "bits_per_byte": 4.232185532753978,
      "learning_rate": 0.0001125,
      "loss_nats": 2.9335274696350098,
      "step": 150,
      "unclipped_grad_norm": 1.1410893201828003
    },
    {
      "bits_per_byte": 4.110705931282335,
      "learning_rate": 0.00011999999999999999,
      "loss_nats": 2.8493242263793945,
      "step": 160,
      "unclipped_grad_norm": 0.887942910194397
    },
    {
      "bits_per_byte": 4.028603133586938,
      "learning_rate": 0.00012749999999999998,
      "loss_nats": 2.792414903640747,
      "step": 170,
      "unclipped_grad_norm": 0.9387072920799255
    },
    {
      "bits_per_byte": 3.9322216482863084,
      "learning_rate": 0.00013499999999999997,
      "loss_nats": 2.7256083488464355,
      "step": 180,
      "unclipped_grad_norm": 0.7101461887359619
    },
    {
      "bits_per_byte": 3.8536510302743325,
      "learning_rate": 0.0001425,
      "loss_nats": 2.671147346496582,
      "step": 190,
      "unclipped_grad_norm": 0.90241938829422
    },
    {
      "bits_per_byte": 3.794223801129896,
      "learning_rate": 0.00015,
      "loss_nats": 2.629955530166626,
      "step": 200,
      "unclipped_grad_norm": 0.8553160429000854
    },
    {
      "bits_per_byte": 3.7222494057091122,
      "learning_rate": 0.0001575,
      "loss_nats": 2.580066680908203,
      "step": 210,
      "unclipped_grad_norm": 0.7590162754058838
    },
    {
      "bits_per_byte": 3.70871849866983,
      "learning_rate": 0.00016499999999999997,
      "loss_nats": 2.570687770843506,
      "step": 220,
      "unclipped_grad_norm": 0.8053720593452454
    },
    {
      "bits_per_byte": 3.6584135732924583,
      "learning_rate": 0.0001725,
      "loss_nats": 2.5358190536499023,
      "step": 230,
      "unclipped_grad_norm": 0.757310152053833
    },
    {
      "bits_per_byte": 3.6516113154855327,
      "learning_rate": 0.00017999999999999998,
      "loss_nats": 2.53110408782959,
      "step": 240,
      "unclipped_grad_norm": 0.6297867894172668
    },
    {
      "bits_per_byte": 3.5768769424837026,
      "learning_rate": 0.0001875,
      "loss_nats": 2.479302167892456,
      "step": 250,
      "unclipped_grad_norm": 0.5848793387413025
    },
    {
      "bits_per_byte": 3.595435590338308,
      "learning_rate": 0.000195,
      "loss_nats": 2.492166042327881,
      "step": 260,
      "unclipped_grad_norm": 0.7728506326675415
    },
    {
      "bits_per_byte": 3.581017252820574,
      "learning_rate": 0.00020249999999999996,
      "loss_nats": 2.4821720123291016,
      "step": 270,
      "unclipped_grad_norm": 0.6333379149436951
    },
    {
      "bits_per_byte": 3.5262535052369333,
      "learning_rate": 0.00020999999999999998,
      "loss_nats": 2.4442126750946045,
      "step": 280,
      "unclipped_grad_norm": 0.6880936026573181
    },
    {
      "bits_per_byte": 3.5151664716664115,
      "learning_rate": 0.00021749999999999997,
      "loss_nats": 2.436527729034424,
      "step": 290,
      "unclipped_grad_norm": 0.6548541188240051
    },
    {
      "bits_per_byte": 3.500701698832944,
      "learning_rate": 0.000225,
      "loss_nats": 2.426501512527466,
      "step": 300,
      "unclipped_grad_norm": 0.6513921618461609
    },
    {
      "bits_per_byte": 3.4243479371905896,
      "learning_rate": 0.00023249999999999996,
      "loss_nats": 2.373577117919922,
      "step": 310,
      "unclipped_grad_norm": 0.6903060674667358
    },
    {
      "bits_per_byte": 3.495896847533139,
      "learning_rate": 0.00023999999999999998,
      "loss_nats": 2.423171043395996,
      "step": 320,
      "unclipped_grad_norm": 1.2031209468841553
    },
    {
      "bits_per_byte": 3.486328076804434,
      "learning_rate": 0.0002475,
      "loss_nats": 2.4165384769439697,
      "step": 330,
      "unclipped_grad_norm": 0.8043278455734253
    },
    {
      "bits_per_byte": 3.4281229563770284,
      "learning_rate": 0.00025499999999999996,
      "loss_nats": 2.3761937618255615,
      "step": 340,
      "unclipped_grad_norm": 0.7758461236953735
    },
    {
      "bits_per_byte": 3.3969462853939203,
      "learning_rate": 0.0002625,
      "loss_nats": 2.354583740234375,
      "step": 350,
      "unclipped_grad_norm": 0.7247553467750549
    },
    {
      "bits_per_byte": 3.3922525348865724,
      "learning_rate": 0.00026999999999999995,
      "loss_nats": 2.351330280303955,
      "step": 360,
      "unclipped_grad_norm": 0.7819244265556335
    },
    {
      "bits_per_byte": 3.4131862631495933,
      "learning_rate": 0.00027749999999999997,
      "loss_nats": 2.365840435028076,
      "step": 370,
      "unclipped_grad_norm": 0.8280189037322998
    },
    {
      "bits_per_byte": 3.3082022056528055,
      "learning_rate": 0.000285,
      "loss_nats": 2.2930710315704346,
      "step": 380,
      "unclipped_grad_norm": 0.7057737112045288
    },
    {
      "bits_per_byte": 3.3912378372464818,
      "learning_rate": 0.0002925,
      "loss_nats": 2.3506269454956055,
      "step": 390,
      "unclipped_grad_norm": 0.7360323071479797
    },
    {
      "bits_per_byte": 3.361113700086654,
      "learning_rate": 0.0003,
      "loss_nats": 2.3297464847564697,
      "step": 400,
      "unclipped_grad_norm": 0.6539704203605652
    },
    {
      "bits_per_byte": 3.4030207126223946,
      "learning_rate": 0.0002999988466112388,
      "loss_nats": 2.3587942123413086,
      "step": 410,
      "unclipped_grad_norm": 0.791742205619812
    },
    {
      "bits_per_byte": 3.334176745412616,
      "learning_rate": 0.00029999538646466355,
      "loss_nats": 2.311075210571289,
      "step": 420,
      "unclipped_grad_norm": 0.8703852891921997
    },
    {
      "bits_per_byte": 3.3591761435420335,
      "learning_rate": 0.0002999896196193985,
      "loss_nats": 2.3284034729003906,
      "step": 430,
      "unclipped_grad_norm": 0.8373144865036011
    },
    {
      "bits_per_byte": 3.313656119476967,
      "learning_rate": 0.00029998154617398326,
      "loss_nats": 2.296851396560669,
      "step": 440,
      "unclipped_grad_norm": 1.3769525289535522
    },
    {
      "bits_per_byte": 3.3562916505217624,
      "learning_rate": 0.0002999711662663707,
      "loss_nats": 2.326404094696045,
      "step": 450,
      "unclipped_grad_norm": 0.7585985660552979
    },
    {
      "bits_per_byte": 3.2828990861619642,
      "learning_rate": 0.0002999584800739247,
      "loss_nats": 2.2755322456359863,
      "step": 460,
      "unclipped_grad_norm": 0.9068207144737244
    },
    {
      "bits_per_byte": 3.2566393991670237,
      "learning_rate": 0.0002999434878134173,
      "loss_nats": 2.2573304176330566,
      "step": 470,
      "unclipped_grad_norm": 0.9806933999061584
    },
    {
      "bits_per_byte": 3.2517141600116144,
      "learning_rate": 0.000299926189741025,
      "loss_nats": 2.2539165019989014,
      "step": 480,
      "unclipped_grad_norm": 1.2366756200790405
    },
    {
      "bits_per_byte": 3.1895775161854854,
      "learning_rate": 0.000299906586152324,
      "loss_nats": 2.2108466625213623,
      "step": 490,
      "unclipped_grad_norm": 0.7806023955345154
    },
    {
      "bits_per_byte": 3.123298498230674,
      "learning_rate": 0.0002998846773822856,
      "loss_nats": 2.164905548095703,
      "step": 500,
      "unclipped_grad_norm": 1.1635692119598389
    },
    {
      "bits_per_byte": 3.1673697404448724,
      "learning_rate": 0.0002998604638052702,
      "loss_nats": 2.195453405380249,
      "step": 510,
      "unclipped_grad_norm": 1.5141944885253906
    },
    {
      "bits_per_byte": 3.1538006532570986,
      "learning_rate": 0.0002998339458350211,
      "loss_nats": 2.1860480308532715,
      "step": 520,
      "unclipped_grad_norm": 1.1977208852767944
    },
    {
      "bits_per_byte": 2.9931695452818388,
      "learning_rate": 0.0002998051239246572,
      "loss_nats": 2.07470703125,
      "step": 530,
      "unclipped_grad_norm": 1.6449142694473267
    },
    {
      "bits_per_byte": 2.988713818763285,
      "learning_rate": 0.00029977399856666537,
      "loss_nats": 2.0716185569763184,
      "step": 540,
      "unclipped_grad_norm": 1.1201231479644775
    },
    {
      "bits_per_byte": 2.9807878263156216,
      "learning_rate": 0.0002997405702928922,
      "loss_nats": 2.066124677658081,
      "step": 550,
      "unclipped_grad_norm": 1.0647740364074707
    },
    {
      "bits_per_byte": 2.990274733302516,
      "learning_rate": 0.0002997048396745345,
      "loss_nats": 2.0727005004882812,
      "step": 560,
      "unclipped_grad_norm": 1.5665196180343628
    },
    {
      "bits_per_byte": 2.920768976852217,
      "learning_rate": 0.0002996668073221302,
      "loss_nats": 2.0245227813720703,
      "step": 570,
      "unclipped_grad_norm": 1.1736958026885986
    },
    {
      "bits_per_byte": 2.9347649249794325,
      "learning_rate": 0.000299626473885547,
      "loss_nats": 2.034224033355713,
      "step": 580,
      "unclipped_grad_norm": 1.3881648778915405
    },
    {
      "bits_per_byte": 2.871172963961702,
      "learning_rate": 0.00029958384005397226,
      "loss_nats": 1.9901454448699951,
      "step": 590,
      "unclipped_grad_norm": 0.9850356578826904
    },
    {
      "bits_per_byte": 2.835541942321247,
      "learning_rate": 0.0002995389065559004,
      "loss_nats": 1.9654479026794434,
      "step": 600,
      "unclipped_grad_norm": 1.141566514968872
    },
    {
      "bits_per_byte": 2.8507572474430796,
      "learning_rate": 0.000299491674159121,
      "loss_nats": 1.975994348526001,
      "step": 610,
      "unclipped_grad_norm": 1.0074148178100586
    },
    {
      "bits_per_byte": 2.8309881856917016,
      "learning_rate": 0.00029944214367070535,
      "loss_nats": 1.9622914791107178,
      "step": 620,
      "unclipped_grad_norm": 1.614481806755066
    },
    {
      "bits_per_byte": 2.744016559150379,
      "learning_rate": 0.00029939031593699284,
      "loss_nats": 1.9020073413848877,
      "step": 630,
      "unclipped_grad_norm": 1.23131263256073
    },
    {
      "bits_per_byte": 2.7596193411846057,
      "learning_rate": 0.00029933619184357654,
      "loss_nats": 1.9128223657608032,
      "step": 640,
      "unclipped_grad_norm": 1.2868338823318481
    },
    {
      "bits_per_byte": 2.7369749014934524,
      "learning_rate": 0.00029927977231528774,
      "loss_nats": 1.8971264362335205,
      "step": 650,
      "unclipped_grad_norm": 1.4851713180541992
    },
    {
      "bits_per_byte": 2.7511890956046128,
      "learning_rate": 0.00029922105831618057,
      "loss_nats": 1.906978964805603,
      "step": 660,
      "unclipped_grad_norm": 0.9212717413902283
    },
    {
      "bits_per_byte": 2.7285928110557007,
      "learning_rate": 0.0002991600508495153,
      "loss_nats": 1.8913164138793945,
      "step": 670,
      "unclipped_grad_norm": 1.196967601776123
    },
    {
      "bits_per_byte": 2.7059170705220903,
      "learning_rate": 0.00029909675095774114,
      "loss_nats": 1.8755987882614136,
      "step": 680,
      "unclipped_grad_norm": 1.0040963888168335
    },
    {
      "bits_per_byte": 2.678678114313137,
      "learning_rate": 0.0002990311597224787,
      "loss_nats": 1.8567181825637817,
      "step": 690,
      "unclipped_grad_norm": 1.19021475315094
    },
    {
      "bits_per_byte": 2.6497494285594567,
      "learning_rate": 0.00029896327826450103,
      "loss_nats": 1.8366663455963135,
      "step": 700,
      "unclipped_grad_norm": 1.2448054552078247
    },
    {
      "bits_per_byte": 2.6450725323518935,
      "learning_rate": 0.0002988931077437152,
      "loss_nats": 1.8334245681762695,
      "step": 710,
      "unclipped_grad_norm": 1.2336615324020386
    },
    {
      "bits_per_byte": 2.640725326886034,
      "learning_rate": 0.00029882064935914164,
      "loss_nats": 1.8304113149642944,
      "step": 720,
      "unclipped_grad_norm": 1.0797665119171143
    },
    {
      "bits_per_byte": 2.6141132474394757,
      "learning_rate": 0.0002987459043488943,
      "loss_nats": 1.8119652271270752,
      "step": 730,
      "unclipped_grad_norm": 1.052964448928833
    },
    {
      "bits_per_byte": 2.5497685183589187,
      "learning_rate": 0.0002986688739901591,
      "loss_nats": 1.7673648595809937,
      "step": 740,
      "unclipped_grad_norm": 1.2775253057479858
    },
    {
      "bits_per_byte": 2.5706501358787284,
      "learning_rate": 0.0002985895595991723,
      "loss_nats": 1.7818388938903809,
      "step": 750,
      "unclipped_grad_norm": 1.240071415901184
    },
    {
      "bits_per_byte": 2.470865801848127,
      "learning_rate": 0.00029850796253119824,
      "loss_nats": 1.7126736640930176,
      "step": 760,
      "unclipped_grad_norm": 1.5425997972488403
    },
    {
      "bits_per_byte": 2.521409439144896,
      "learning_rate": 0.00029842408418050544,
      "loss_nats": 1.7477078437805176,
      "step": 770,
      "unclipped_grad_norm": 1.3344701528549194
    },
    {
      "bits_per_byte": 2.472837410957618,
      "learning_rate": 0.0002983379259803436,
      "loss_nats": 1.7140402793884277,
      "step": 780,
      "unclipped_grad_norm": 1.1049633026123047
    },
    {
      "bits_per_byte": 2.473506767434776,
      "learning_rate": 0.0002982494894029186,
      "loss_nats": 1.7145042419433594,
      "step": 790,
      "unclipped_grad_norm": 1.1233389377593994
    },
    {
      "bits_per_byte": 2.401472866016794,
      "learning_rate": 0.0002981587759593675,
      "loss_nats": 1.664574146270752,
      "step": 800,
      "unclipped_grad_norm": 1.229269027709961
    },
    {
      "bits_per_byte": 2.4415747486490784,
      "learning_rate": 0.00029806578719973283,
      "loss_nats": 1.6923706531524658,
      "step": 810,
      "unclipped_grad_norm": 1.1753153800964355
    },
    {
      "bits_per_byte": 2.4350285730092875,
      "learning_rate": 0.00029797052471293596,
      "loss_nats": 1.6878331899642944,
      "step": 820,
      "unclipped_grad_norm": 1.128970742225647
    },
    {
      "bits_per_byte": 2.4442745323023227,
      "learning_rate": 0.0002978729901267499,
      "loss_nats": 1.694242000579834,
      "step": 830,
      "unclipped_grad_norm": 1.1320509910583496
    },
    {
      "bits_per_byte": 2.411402456347009,
      "learning_rate": 0.0002977731851077716,
      "loss_nats": 1.6714568138122559,
      "step": 840,
      "unclipped_grad_norm": 1.2860697507858276
    },
    {
      "bits_per_byte": 2.3597125546822393,
      "learning_rate": 0.0002976711113613935,
      "loss_nats": 1.6356281042099,
      "step": 850,
      "unclipped_grad_norm": 1.1755540370941162
    },
    {
      "bits_per_byte": 2.4397532803937905,
      "learning_rate": 0.0002975667706317743,
      "loss_nats": 1.6911081075668335,
      "step": 860,
      "unclipped_grad_norm": 1.4042041301727295
    },
    {
      "bits_per_byte": 2.4029735866282227,
      "learning_rate": 0.00029746016470180915,
      "loss_nats": 1.665614366531372,
      "step": 870,
      "unclipped_grad_norm": 1.2927662134170532
    },
    {
      "bits_per_byte": 2.3685887512759196,
      "learning_rate": 0.00029735129539309926,
      "loss_nats": 1.6417806148529053,
      "step": 880,
      "unclipped_grad_norm": 1.1862192153930664
    },
    {
      "bits_per_byte": 2.3211230874816424,
      "learning_rate": 0.0002972401645659207,
      "loss_nats": 1.6088799238204956,
      "step": 890,
      "unclipped_grad_norm": 1.135436773300171
    },
    {
      "bits_per_byte": 2.364436574136139,
      "learning_rate": 0.00029712677411919283,
      "loss_nats": 1.6389025449752808,
      "step": 900,
      "unclipped_grad_norm": 1.3485081195831299
    },
    {
      "bits_per_byte": 2.347153177637681,
      "learning_rate": 0.0002970111259904454,
      "loss_nats": 1.626922607421875,
      "step": 910,
      "unclipped_grad_norm": 1.1310322284698486
    },
    {
      "bits_per_byte": 2.338854326802947,
      "learning_rate": 0.0002968932221557859,
      "loss_nats": 1.6211702823638916,
      "step": 920,
      "unclipped_grad_norm": 1.1499736309051514
    },
    {
      "bits_per_byte": 2.28766523861749,
      "learning_rate": 0.0002967730646298655,
      "loss_nats": 1.5856887102127075,
      "step": 930,
      "unclipped_grad_norm": 1.0895946025848389
    },
    {
      "bits_per_byte": 2.3220005429663444,
      "learning_rate": 0.0002966506554658449,
      "loss_nats": 1.6094881296157837,
      "step": 940,
      "unclipped_grad_norm": 1.0576175451278687
    },
    {
      "bits_per_byte": 2.3596927766773903,
      "learning_rate": 0.00029652599675535876,
      "loss_nats": 1.6356143951416016,
      "step": 950,
      "unclipped_grad_norm": 1.078567385673523
    },
    {
      "bits_per_byte": 2.2833796029406397,
      "learning_rate": 0.0002963990906284808,
      "loss_nats": 1.5827181339263916,
      "step": 960,
      "unclipped_grad_norm": 1.1438634395599365
    },
    {
      "bits_per_byte": 2.2449070839426293,
      "learning_rate": 0.0002962699392536863,
      "loss_nats": 1.5560510158538818,
      "step": 970,
      "unclipped_grad_norm": 1.2200326919555664
    },
    {
      "bits_per_byte": 2.3829726922634817,
      "learning_rate": 0.0002961385448378161,
      "loss_nats": 1.6517508029937744,
      "step": 980,
      "unclipped_grad_norm": 1.0979547500610352
    },
    {
      "bits_per_byte": 2.2178129371257023,
      "learning_rate": 0.00029600490962603817,
      "loss_nats": 1.5372707843780518,
      "step": 990,
      "unclipped_grad_norm": 0.9582167863845825
    },
    {
      "bits_per_byte": 2.2343387500471095,
      "learning_rate": 0.00029586903590180956,
      "loss_nats": 1.5487256050109863,
      "step": 1000,
      "unclipped_grad_norm": 1.1553164720535278
    },
    {
      "bits_per_byte": 2.2444941535979077,
      "learning_rate": 0.00029573092598683735,
      "loss_nats": 1.5557647943496704,
      "step": 1010,
      "unclipped_grad_norm": 1.5419076681137085
    },
    {
      "bits_per_byte": 2.204213924956678,
      "learning_rate": 0.00029559058224103895,
      "loss_nats": 1.5278446674346924,
      "step": 1020,
      "unclipped_grad_norm": 1.2040239572525024
    },
    {
      "bits_per_byte": 2.2117641353122064,
      "learning_rate": 0.0002954480070625017,
      "loss_nats": 1.5330780744552612,
      "step": 1030,
      "unclipped_grad_norm": 1.308585286140442
    },
    {
      "bits_per_byte": 2.2977618241016926,
      "learning_rate": 0.00029530320288744205,
      "loss_nats": 1.5926871299743652,
      "step": 1040,
      "unclipped_grad_norm": 1.4551756381988525
    },
    {
      "bits_per_byte": 2.2190181915429488,
      "learning_rate": 0.0002951561721901638,
      "loss_nats": 1.5381062030792236,
      "step": 1050,
      "unclipped_grad_norm": 1.5392372608184814
    },
    {
      "bits_per_byte": 2.226578548874878,
      "learning_rate": 0.00029500691748301594,
      "loss_nats": 1.543346643447876,
      "step": 1060,
      "unclipped_grad_norm": 1.1143345832824707
    },
    {
      "bits_per_byte": 2.276697561006667,
      "learning_rate": 0.00029485544131634963,
      "loss_nats": 1.578086495399475,
      "step": 1070,
      "unclipped_grad_norm": 1.1605373620986938
    },
    {
      "bits_per_byte": 2.2501855754629103,
      "learning_rate": 0.0002947017462784745,
      "loss_nats": 1.5597097873687744,
      "step": 1080,
      "unclipped_grad_norm": 1.0904817581176758
    },
    {
      "bits_per_byte": 2.178866605924565,
      "learning_rate": 0.00029454583499561476,
      "loss_nats": 1.5102752447128296,
      "step": 1090,
      "unclipped_grad_norm": 1.1060513257980347
    },
    {
      "bits_per_byte": 2.2050501046051734,
      "learning_rate": 0.00029438771013186413,
      "loss_nats": 1.5284242630004883,
      "step": 1100,
      "unclipped_grad_norm": 1.2513083219528198
    },
    {
      "bits_per_byte": 2.140159674573429,
      "learning_rate": 0.0002942273743891401,
      "loss_nats": 1.483445644378662,
      "step": 1110,
      "unclipped_grad_norm": 1.0150246620178223
    },
    {
      "bits_per_byte": 2.197250347423243,
      "learning_rate": 0.00029406483050713824,
      "loss_nats": 1.5230178833007812,
      "step": 1120,
      "unclipped_grad_norm": 1.0046241283416748
    },
    {
      "bits_per_byte": 2.212073188135807,
      "learning_rate": 0.00029390008126328504,
      "loss_nats": 1.533292293548584,
      "step": 1130,
      "unclipped_grad_norm": 0.9860795736312866
    },
    {
      "bits_per_byte": 2.1513696757393173,
      "learning_rate": 0.00029373312947269045,
      "loss_nats": 1.4912158250808716,
      "step": 1140,
      "unclipped_grad_norm": 1.0693174600601196
    },
    {
      "bits_per_byte": 2.217324850362554,
      "learning_rate": 0.00029356397798809995,
      "loss_nats": 1.5369324684143066,
      "step": 1150,
      "unclipped_grad_norm": 1.2721859216690063
    },
    {
      "bits_per_byte": 2.1164305403032837,
      "learning_rate": 0.0002933926296998457,
      "loss_nats": 1.4669978618621826,
      "step": 1160,
      "unclipped_grad_norm": 1.2611263990402222
    },
    {
      "bits_per_byte": 2.1545339845325424,
      "learning_rate": 0.0002932190875357972,
      "loss_nats": 1.4934091567993164,
      "step": 1170,
      "unclipped_grad_norm": 1.396773099899292
    },
    {
      "bits_per_byte": 2.157210722510571,
      "learning_rate": 0.00029304335446131126,
      "loss_nats": 1.4952645301818848,
      "step": 1180,
      "unclipped_grad_norm": 1.0693202018737793
    },
    {
      "bits_per_byte": 2.146851003570548,
      "learning_rate": 0.00029286543347918116,
      "loss_nats": 1.4880837202072144,
      "step": 1190,
      "unclipped_grad_norm": 1.1468050479888916
    },
    {
      "bits_per_byte": 2.1677991783762414,
      "learning_rate": 0.0002926853276295856,
      "loss_nats": 1.5026038885116577,
      "step": 1200,
      "unclipped_grad_norm": 1.1243788003921509
    },
    {
      "bits_per_byte": 2.171953075342531,
      "learning_rate": 0.0002925030399900368,
      "loss_nats": 1.5054831504821777,
      "step": 1210,
      "unclipped_grad_norm": 1.0842303037643433
    },
    {
      "bits_per_byte": 2.1393570315418518,
      "learning_rate": 0.00029231857367532726,
      "loss_nats": 1.4828892946243286,
      "step": 1220,
      "unclipped_grad_norm": 1.0247055292129517
    },
    {
      "bits_per_byte": 2.133679196306267,
      "learning_rate": 0.0002921319318374776,
      "loss_nats": 1.4789537191390991,
      "step": 1230,
      "unclipped_grad_norm": 1.0191700458526611
    },
    {
      "bits_per_byte": 2.0961321159525497,
      "learning_rate": 0.00029194311766568174,
      "loss_nats": 1.452928066253662,
      "step": 1240,
      "unclipped_grad_norm": 1.0066498517990112
    },
    {
      "bits_per_byte": 2.0423632880041502,
      "learning_rate": 0.000291752134386253,
      "loss_nats": 1.4156583547592163,
      "step": 1250,
      "unclipped_grad_norm": 1.1115199327468872
    },
    {
      "bits_per_byte": 2.102061045858394,
      "learning_rate": 0.00029155898526256856,
      "loss_nats": 1.4570376873016357,
      "step": 1260,
      "unclipped_grad_norm": 1.1423614025115967
    },
    {
      "bits_per_byte": 2.1196135952054522,
      "learning_rate": 0.0002913636735950141,
      "loss_nats": 1.4692041873931885,
      "step": 1270,
      "unclipped_grad_norm": 0.9267144203186035
    },
    {
      "bits_per_byte": 2.1628902775726644,
      "learning_rate": 0.0002911662027209272,
      "loss_nats": 1.4992012977600098,
      "step": 1280,
      "unclipped_grad_norm": 1.1472139358520508
    },
    {
      "bits_per_byte": 2.104273774644395,
      "learning_rate": 0.00029096657601454013,
      "loss_nats": 1.458571434020996,
      "step": 1290,
      "unclipped_grad_norm": 1.0092620849609375
    },
    {
      "bits_per_byte": 2.0607607881148975,
      "learning_rate": 0.00029076479688692266,
      "loss_nats": 1.428410530090332,
      "step": 1300,
      "unclipped_grad_norm": 1.0175256729125977
    },
    {
      "bits_per_byte": 2.090551794880004,
      "learning_rate": 0.00029056086878592323,
      "loss_nats": 1.449060082435608,
      "step": 1310,
      "unclipped_grad_norm": 1.0536231994628906
    },
    {
      "bits_per_byte": 2.0678022737891735,
      "learning_rate": 0.00029035479519611045,
      "loss_nats": 1.4332913160324097,
      "step": 1320,
      "unclipped_grad_norm": 1.0326532125473022
    },
    {
      "bits_per_byte": 2.079288479092348,
      "learning_rate": 0.0002901465796387134,
      "loss_nats": 1.4412529468536377,
      "step": 1330,
      "unclipped_grad_norm": 1.0093759298324585
    },
    {
      "bits_per_byte": 2.077430378532426,
      "learning_rate": 0.00028993622567156154,
      "loss_nats": 1.439965009689331,
      "step": 1340,
      "unclipped_grad_norm": 0.9901224970817566
    },
    {
      "bits_per_byte": 2.085392143371469,
      "learning_rate": 0.0002897237368890237,
      "loss_nats": 1.445483684539795,
      "step": 1350,
      "unclipped_grad_norm": 0.9441472291946411
    },
    {
      "bits_per_byte": 2.0594795173659697,
      "learning_rate": 0.0002895091169219469,
      "loss_nats": 1.4275224208831787,
      "step": 1360,
      "unclipped_grad_norm": 1.1019998788833618
    },
    {
      "bits_per_byte": 2.0796826633281253,
      "learning_rate": 0.00028929236943759425,
      "loss_nats": 1.441526174545288,
      "step": 1370,
      "unclipped_grad_norm": 1.3928191661834717
    },
    {
      "bits_per_byte": 2.0298482824835147,
      "learning_rate": 0.00028907349813958225,
      "loss_nats": 1.4069836139678955,
      "step": 1380,
      "unclipped_grad_norm": 1.0244042873382568
    },
    {
      "bits_per_byte": 2.0804338835470944,
      "learning_rate": 0.0002888525067678175,
      "loss_nats": 1.442046880722046,
      "step": 1390,
      "unclipped_grad_norm": 1.095209002494812
    },
    {
      "bits_per_byte": 2.195902519388431,
      "learning_rate": 0.00028862939909843273,
      "loss_nats": 1.5220836400985718,
      "step": 1400,
      "unclipped_grad_norm": 1.1566489934921265
    },
    {
      "bits_per_byte": 2.0701701309262526,
      "learning_rate": 0.0002884041789437225,
      "loss_nats": 1.4349325895309448,
      "step": 1410,
      "unclipped_grad_norm": 0.889271080493927
    },
    {
      "bits_per_byte": 2.060555784795069,
      "learning_rate": 0.0002881768501520778,
      "loss_nats": 1.4282684326171875,
      "step": 1420,
      "unclipped_grad_norm": 0.933581531047821
    },
    {
      "bits_per_byte": 2.0393702739311848,
      "learning_rate": 0.0002879474166079205,
      "loss_nats": 1.413583755493164,
      "step": 1430,
      "unclipped_grad_norm": 1.1070539951324463
    },
    {
      "bits_per_byte": 2.055492615553668,
      "learning_rate": 0.00028771588223163694,
      "loss_nats": 1.4247589111328125,
      "step": 1440,
      "unclipped_grad_norm": 1.024774193763733
    },
    {
      "bits_per_byte": 2.0178352943207476,
      "learning_rate": 0.00028748225097951054,
      "loss_nats": 1.3986568450927734,
      "step": 1450,
      "unclipped_grad_norm": 0.9543858766555786
    },
    {
      "bits_per_byte": 2.144940448302113,
      "learning_rate": 0.000287246526843655,
      "loss_nats": 1.4867594242095947,
      "step": 1460,
      "unclipped_grad_norm": 0.9325331449508667
    },
    {
      "bits_per_byte": 2.1008117638825268,
      "learning_rate": 0.00028700871385194533,
      "loss_nats": 1.4561717510223389,
      "step": 1470,
      "unclipped_grad_norm": 0.8672807216644287
    },
    {
      "bits_per_byte": 2.0709982273901573,
      "learning_rate": 0.00028676881606794945,
      "loss_nats": 1.4355065822601318,
      "step": 1480,
      "unclipped_grad_norm": 0.9597178101539612
    },
    {
      "bits_per_byte": 2.0747749664031043,
      "learning_rate": 0.00028652683759085854,
      "loss_nats": 1.438124418258667,
      "step": 1490,
      "unclipped_grad_norm": 1.0223972797393799
    },
    {
      "bits_per_byte": 1.9616643846877473,
      "learning_rate": 0.0002862827825554171,
      "loss_nats": 1.3597221374511719,
      "step": 1500,
      "unclipped_grad_norm": 0.9415771961212158
    },
    {
      "bits_per_byte": 1.978057942950641,
      "learning_rate": 0.00028603665513185245,
      "loss_nats": 1.371085286140442,
      "step": 1510,
      "unclipped_grad_norm": 1.0318301916122437
    },
    {
      "bits_per_byte": 2.0205063568712975,
      "learning_rate": 0.000285788459525803,
      "loss_nats": 1.4005082845687866,
      "step": 1520,
      "unclipped_grad_norm": 0.9729053378105164
    },
    {
      "bits_per_byte": 2.0668845743641695,
      "learning_rate": 0.00028553819997824707,
      "loss_nats": 1.4326552152633667,
      "step": 1530,
      "unclipped_grad_norm": 0.9350422620773315
    },
    {
      "bits_per_byte": 2.045209256910628,
      "learning_rate": 0.00028528588076542964,
      "loss_nats": 1.4176310300827026,
      "step": 1540,
      "unclipped_grad_norm": 0.9831364154815674
    },
    {
      "bits_per_byte": 2.101370363532529,
      "learning_rate": 0.00028503150619878996,
      "loss_nats": 1.4565589427947998,
      "step": 1550,
      "unclipped_grad_norm": 1.1195578575134277
    },
    {
      "bits_per_byte": 1.9633480948396942,
      "learning_rate": 0.0002847750806248876,
      "loss_nats": 1.360889196395874,
      "step": 1560,
      "unclipped_grad_norm": 1.038403034210205
    },
    {
      "bits_per_byte": 1.9669683296403562,
      "learning_rate": 0.000284516608425328,
      "loss_nats": 1.363398551940918,
      "step": 1570,
      "unclipped_grad_norm": 0.8551042675971985
    },
    {
      "bits_per_byte": 1.997066325437253,
      "learning_rate": 0.000284256094016688,
      "loss_nats": 1.384260892868042,
      "step": 1580,
      "unclipped_grad_norm": 0.9270023107528687
    },
    {
      "bits_per_byte": 2.0065728384463455,
      "learning_rate": 0.0002839935418504399,
      "loss_nats": 1.390850305557251,
      "step": 1590,
      "unclipped_grad_norm": 0.9501463174819946
    },
    {
      "bits_per_byte": 1.9863786355820345,
      "learning_rate": 0.000283728956412876,
      "loss_nats": 1.3768527507781982,
      "step": 1600,
      "unclipped_grad_norm": 0.8800983428955078
    },
    {
      "bits_per_byte": 1.9619350853802053,
      "learning_rate": 0.00028346234222503136,
      "loss_nats": 1.3599097728729248,
      "step": 1610,
      "unclipped_grad_norm": 0.911471962928772
    },
    {
      "bits_per_byte": 1.9591475905750209,
      "learning_rate": 0.00028319370384260675,
      "loss_nats": 1.3579776287078857,
      "step": 1620,
      "unclipped_grad_norm": 0.9253746271133423
    },
    {
      "bits_per_byte": 1.9975408255709834,
      "learning_rate": 0.00028292304585589105,
      "loss_nats": 1.3845897912979126,
      "step": 1630,
      "unclipped_grad_norm": 0.932426393032074
    },
    {
      "bits_per_byte": 1.9590067367839643,
      "learning_rate": 0.0002826503728896824,
      "loss_nats": 1.3578799962997437,
      "step": 1640,
      "unclipped_grad_norm": 0.9996587634086609
    },
    {
      "bits_per_byte": 1.9568615971797525,
      "learning_rate": 0.0002823756896032095,
      "loss_nats": 1.3563930988311768,
      "step": 1650,
      "unclipped_grad_norm": 0.9020648002624512
    },
    {
      "bits_per_byte": 1.9902011219801112,
      "learning_rate": 0.0002820990006900518,
      "loss_nats": 1.379502296447754,
      "step": 1660,
      "unclipped_grad_norm": 0.9605674743652344
    },
    {
      "bits_per_byte": 1.9362635790512155,
      "learning_rate": 0.00028182031087805947,
      "loss_nats": 1.3421156406402588,
      "step": 1670,
      "unclipped_grad_norm": 0.9840894937515259
    },
    {
      "bits_per_byte": 1.9741790462430813,
      "learning_rate": 0.0002815396249292725,
      "loss_nats": 1.3683966398239136,
      "step": 1680,
      "unclipped_grad_norm": 0.9542845487594604
    },
    {
      "bits_per_byte": 1.975246198591685,
      "learning_rate": 0.0002812569476398391,
      "loss_nats": 1.3691363334655762,
      "step": 1690,
      "unclipped_grad_norm": 0.8774324655532837
    },
    {
      "bits_per_byte": 1.936552509904665,
      "learning_rate": 0.00028097228383993446,
      "loss_nats": 1.342315912246704,
      "step": 1700,
      "unclipped_grad_norm": 0.9619165658950806
    },
    {
      "bits_per_byte": 1.9815170300074447,
      "learning_rate": 0.0002806856383936774,
      "loss_nats": 1.3734829425811768,
      "step": 1710,
      "unclipped_grad_norm": 1.017353892326355
    },
    {
      "bits_per_byte": 1.9011681113855474,
      "learning_rate": 0.00028039701619904774,
      "loss_nats": 1.3177893161773682,
      "step": 1720,
      "unclipped_grad_norm": 0.890689492225647
    },
    {
      "bits_per_byte": 1.9742225578537496,
      "learning_rate": 0.0002801064221878024,
      "loss_nats": 1.36842679977417,
      "step": 1730,
      "unclipped_grad_norm": 0.9096551537513733
    },
    {
      "bits_per_byte": 1.9832260216090685,
      "learning_rate": 0.00027981386132539134,
      "loss_nats": 1.3746675252914429,
      "step": 1740,
      "unclipped_grad_norm": 0.9014292359352112
    },
    {
      "bits_per_byte": 1.9548246346629332,
      "learning_rate": 0.0002795193386108724,
      "loss_nats": 1.3549811840057373,
      "step": 1750,
      "unclipped_grad_norm": 0.8635335564613342
    },
    {
      "bits_per_byte": 1.9935654465962895,
      "learning_rate": 0.00027922285907682625,
      "loss_nats": 1.3818342685699463,
      "step": 1760,
      "unclipped_grad_norm": 0.9282572865486145
    },
    {
      "bits_per_byte": 1.9777825987266096,
      "learning_rate": 0.00027892442778927005,
      "loss_nats": 1.370894432067871,
      "step": 1770,
      "unclipped_grad_norm": 0.9273641705513
    },
    {
      "bits_per_byte": 2.0529696300655105,
      "learning_rate": 0.00027862404984757116,
      "loss_nats": 1.4230101108551025,
      "step": 1780,
      "unclipped_grad_norm": 0.9726579189300537
    },
    {
      "bits_per_byte": 1.9025646105105534,
      "learning_rate": 0.0002783217303843597,
      "loss_nats": 1.3187572956085205,
      "step": 1790,
      "unclipped_grad_norm": 0.9649468660354614
    },
    {
      "bits_per_byte": 2.051186341958714,
      "learning_rate": 0.00027801747456544134,
      "loss_nats": 1.4217740297317505,
      "step": 1800,
      "unclipped_grad_norm": 1.1441059112548828
    },
    {
      "bits_per_byte": 1.9169516471858308,
      "learning_rate": 0.00027771128758970843,
      "loss_nats": 1.3287296295166016,
      "step": 1810,
      "unclipped_grad_norm": 1.0408838987350464
    },
    {
      "bits_per_byte": 1.9713771048952242,
      "learning_rate": 0.00027740317468905165,
      "loss_nats": 1.3664544820785522,
      "step": 1820,
      "unclipped_grad_norm": 0.9737800359725952
    },
    {
      "bits_per_byte": 1.9989835880290714,
      "learning_rate": 0.0002770931411282704,
      "loss_nats": 1.385589838027954,
      "step": 1830,
      "unclipped_grad_norm": 0.8691428899765015
    },
    {
      "bits_per_byte": 1.9804913254776988,
      "learning_rate": 0.0002767811922049827,
      "loss_nats": 1.372771978378296,
      "step": 1840,
      "unclipped_grad_norm": 0.915309727191925
    },
    {
      "bits_per_byte": 1.918886795973339,
      "learning_rate": 0.00027646733324953503,
      "loss_nats": 1.330070972442627,
      "step": 1850,
      "unclipped_grad_norm": 0.9071885347366333
    },
    {
      "bits_per_byte": 1.9458363053808903,
      "learning_rate": 0.00027615156962491097,
      "loss_nats": 1.3487509489059448,
      "step": 1860,
      "unclipped_grad_norm": 0.8704239726066589
    },
    {
      "bits_per_byte": 2.024965007094916,
      "learning_rate": 0.0002758339067266396,
      "loss_nats": 1.4035987854003906,
      "step": 1870,
      "unclipped_grad_norm": 0.9987708330154419
    },
    {
      "bits_per_byte": 1.9165194547842126,
      "learning_rate": 0.0002755143499827035,
      "loss_nats": 1.3284300565719604,
      "step": 1880,
      "unclipped_grad_norm": 0.868863582611084
    },
    {
      "bits_per_byte": 1.9462939512148363,
      "learning_rate": 0.0002751929048534456,
      "loss_nats": 1.3490681648254395,
      "step": 1890,
      "unclipped_grad_norm": 0.8501539826393127
    },
    {
      "bits_per_byte": 1.9456569274760405,
      "learning_rate": 0.0002748695768314763,
      "loss_nats": 1.3486266136169434,
      "step": 1900,
      "unclipped_grad_norm": 0.8660019040107727
    },
    {
      "bits_per_byte": 1.9970934986960893,
      "learning_rate": 0.00027454437144157947,
      "loss_nats": 1.384279727935791,
      "step": 1910,
      "unclipped_grad_norm": 0.8706849217414856
    },
    {
      "bits_per_byte": 1.9090278905126286,
      "learning_rate": 0.00027421729424061787,
      "loss_nats": 1.3232372999191284,
      "step": 1920,
      "unclipped_grad_norm": 0.9953081607818604
    },
    {
      "bits_per_byte": 1.9556608143114282,
      "learning_rate": 0.0002738883508174385,
      "loss_nats": 1.3555607795715332,
      "step": 1930,
      "unclipped_grad_norm": 0.8388954401016235
    },
    {
      "bits_per_byte": 1.9678466450383125,
      "learning_rate": 0.0002735575467927768,
      "loss_nats": 1.3640073537826538,
      "step": 1940,
      "unclipped_grad_norm": 0.9170766472816467
    },
    {
      "bits_per_byte": 1.9103982482747035,
      "learning_rate": 0.0002732248878191609,
      "loss_nats": 1.324187159538269,
      "step": 1950,
      "unclipped_grad_norm": 0.942866861820221
    },
    {
      "bits_per_byte": 1.9025671902503163,
      "learning_rate": 0.0002728903795808148,
      "loss_nats": 1.3187590837478638,
      "step": 1960,
      "unclipped_grad_norm": 0.8135740756988525
    },
    {
      "bits_per_byte": 1.9075963069268465,
      "learning_rate": 0.0002725540277935614,
      "loss_nats": 1.3222450017929077,
      "step": 1970,
      "unclipped_grad_norm": 0.9107059836387634
    },
    {
      "bits_per_byte": 1.9379792779762228,
      "learning_rate": 0.00027221583820472456,
      "loss_nats": 1.3433048725128174,
      "step": 1980,
      "unclipped_grad_norm": 0.9617857336997986
    },
    {
      "bits_per_byte": 1.9109520324104816,
      "learning_rate": 0.0002718758165930314,
      "loss_nats": 1.3245710134506226,
      "step": 1990,
      "unclipped_grad_norm": 0.9190393090248108
    },
    {
      "bits_per_byte": 1.9811283492164946,
      "learning_rate": 0.0002715339687685131,
      "loss_nats": 1.373213529586792,
      "step": 2000,
      "unclipped_grad_norm": 0.878118097782135
    },
    {
      "bits_per_byte": 1.9843879363982975,
      "learning_rate": 0.00027119030057240576,
      "loss_nats": 1.375472903251648,
      "step": 2010,
      "unclipped_grad_norm": 1.0113674402236938
    },
    {
      "bits_per_byte": 1.8836764438968945,
      "learning_rate": 0.0002708448178770506,
      "loss_nats": 1.3056650161743164,
      "step": 2020,
      "unclipped_grad_norm": 0.8797336220741272
    },
    {
      "bits_per_byte": 1.9153360421636256,
      "learning_rate": 0.00027049752658579376,
      "loss_nats": 1.3276097774505615,
      "step": 2030,
      "unclipped_grad_norm": 0.8666315674781799
    },
    {
      "bits_per_byte": 1.906092662610353,
      "learning_rate": 0.00027014843263288514,
      "loss_nats": 1.3212027549743652,
      "step": 2040,
      "unclipped_grad_norm": 0.8591557145118713
    },
    {
      "bits_per_byte": 1.8798058023565762,
      "learning_rate": 0.00026979754198337724,
      "loss_nats": 1.3029820919036865,
      "step": 2050,
      "unclipped_grad_norm": 0.9546817541122437
    },
    {
      "bits_per_byte": 1.8574308594793247,
      "learning_rate": 0.00026944486063302305,
      "loss_nats": 1.2874729633331299,
      "step": 2060,
      "unclipped_grad_norm": 0.8160566687583923
    },
    {
      "bits_per_byte": 1.9859665651505671,
      "learning_rate": 0.0002690903946081738,
      "loss_nats": 1.3765671253204346,
      "step": 2070,
      "unclipped_grad_norm": 0.8717494606971741
    },
    {
      "bits_per_byte": 1.952941940583938,
      "learning_rate": 0.00026873414996567596,
      "loss_nats": 1.353676199913025,
      "step": 2080,
      "unclipped_grad_norm": 0.9155706763267517
    },
    {
      "bits_per_byte": 1.8808098370723132,
      "learning_rate": 0.00026837613279276736,
      "loss_nats": 1.303678035736084,
      "step": 2090,
      "unclipped_grad_norm": 0.8717590570449829
    },
    {
      "bits_per_byte": 1.8310299747280259,
      "learning_rate": 0.0002680163492069739,
      "loss_nats": 1.269173264503479,
      "step": 2100,
      "unclipped_grad_norm": 0.9355071783065796
    },
    {
      "bits_per_byte": 1.868735623085839,
      "learning_rate": 0.00026765480535600414,
      "loss_nats": 1.2953088283538818,
      "step": 2110,
      "unclipped_grad_norm": 0.839803159236908
    },
    {
      "bits_per_byte": 1.8925072370707472,
      "learning_rate": 0.0002672915074176452,
      "loss_nats": 1.3117860555648804,
      "step": 2120,
      "unclipped_grad_norm": 1.064001202583313
    },
    {
      "bits_per_byte": 1.9234560310414621,
      "learning_rate": 0.00026692646159965624,
      "loss_nats": 1.333238124847412,
      "step": 2130,
      "unclipped_grad_norm": 0.9675858020782471
    },
    {
      "bits_per_byte": 1.9467920129717349,
      "learning_rate": 0.0002665596741396633,
      "loss_nats": 1.3494133949279785,
      "step": 2140,
      "unclipped_grad_norm": 0.9227033257484436
    },
    {
      "bits_per_byte": 1.8425244351984418,
      "learning_rate": 0.000266191151305052,
      "loss_nats": 1.2771406173706055,
      "step": 2150,
      "unclipped_grad_norm": 0.9172095060348511
    },
    {
      "bits_per_byte": 1.9385437250363546,
      "learning_rate": 0.00026582089939286075,
      "loss_nats": 1.343696117401123,
      "step": 2160,
      "unclipped_grad_norm": 0.9335131049156189
    },
    {
      "bits_per_byte": 1.8590810330143535,
      "learning_rate": 0.0002654489247296732,
      "loss_nats": 1.2886167764663696,
      "step": 2170,
      "unclipped_grad_norm": 0.8850465416908264
    },
    {
      "bits_per_byte": 1.9348510855396792,
      "learning_rate": 0.00026507523367151007,
      "loss_nats": 1.3411365747451782,
      "step": 2180,
      "unclipped_grad_norm": 0.8827643990516663
    },
    {
      "bits_per_byte": 1.9013160164652894,
      "learning_rate": 0.00026469983260372037,
      "loss_nats": 1.3178918361663818,
      "step": 2190,
      "unclipped_grad_norm": 0.9066281318664551
    },
    {
      "bits_per_byte": 1.8761559865399657,
      "learning_rate": 0.0002643227279408727,
      "loss_nats": 1.3004522323608398,
      "step": 2200,
      "unclipped_grad_norm": 0.844115674495697
    },
    {
      "bits_per_byte": 1.8719651133037407,
      "learning_rate": 0.0002639439261266453,
      "loss_nats": 1.2975473403930664,
      "step": 2210,
      "unclipped_grad_norm": 0.9099118709564209
    },
    {
      "bits_per_byte": 1.8935372411667646,
      "learning_rate": 0.000263563433633716,
      "loss_nats": 1.3125,
      "step": 2220,
      "unclipped_grad_norm": 1.0520122051239014
    },
    {
      "bits_per_byte": 1.8237762624625853,
      "learning_rate": 0.00026318125696365156,
      "loss_nats": 1.2641453742980957,
      "step": 2230,
      "unclipped_grad_norm": 0.8068414330482483
    },
    {
      "bits_per_byte": 1.9063850331168197,
      "learning_rate": 0.000262797402646797,
      "loss_nats": 1.3214054107666016,
      "step": 2240,
      "unclipped_grad_norm": 0.8128858804702759
    },
    {
      "bits_per_byte": 1.9127232817317177,
      "learning_rate": 0.00026241187724216334,
      "loss_nats": 1.325798749923706,
      "step": 2250,
      "unclipped_grad_norm": 0.8365156650543213
    },
    {
      "bits_per_byte": 1.965086839439917,
      "learning_rate": 0.0002620246873373162,
      "loss_nats": 1.3620944023132324,
      "step": 2260,
      "unclipped_grad_norm": 0.9489718675613403
    },
    {
      "bits_per_byte": 1.8335648270190927,
      "learning_rate": 0.00026163583954826264,
      "loss_nats": 1.270930290222168,
      "step": 2270,
      "unclipped_grad_norm": 0.8451220989227295
    },
    {
      "bits_per_byte": 1.8897092513238603,
      "learning_rate": 0.0002612453405193386,
      "loss_nats": 1.3098466396331787,
      "step": 2280,
      "unclipped_grad_norm": 0.8222805261611938
    },
    {
      "bits_per_byte": 1.8508822760824215,
      "learning_rate": 0.000260853196923095,
      "loss_nats": 1.2829338312149048,
      "step": 2290,
      "unclipped_grad_norm": 0.8723782896995544
    },
    {
      "bits_per_byte": 1.839125714052091,
      "learning_rate": 0.0002604594154601839,
      "loss_nats": 1.274784803390503,
      "step": 2300,
      "unclipped_grad_norm": 0.8368175625801086
    },
    {
      "bits_per_byte": 1.846592168856649,
      "learning_rate": 0.0002600640028592441,
      "loss_nats": 1.2799601554870605,
      "step": 2310,
      "unclipped_grad_norm": 0.8264262080192566
    },
    {
      "bits_per_byte": 1.893433707610945,
      "learning_rate": 0.0002596669658767859,
      "loss_nats": 1.3124282360076904,
      "step": 2320,
      "unclipped_grad_norm": 0.8465461134910583
    },
    {
      "bits_per_byte": 1.8498577754312318,
      "learning_rate": 0.000259268311297076,
      "loss_nats": 1.2822237014770508,
      "step": 2330,
      "unclipped_grad_norm": 0.8474167585372925
    },
    {
      "bits_per_byte": 1.8556085313107824,
      "learning_rate": 0.00025886804593202114,
      "loss_nats": 1.2862098217010498,
      "step": 2340,
      "unclipped_grad_norm": 0.8308321237564087
    },
    {
      "bits_per_byte": 1.9187394068415498,
      "learning_rate": 0.00025846617662105193,
      "loss_nats": 1.329968810081482,
      "step": 2350,
      "unclipped_grad_norm": 0.8866831660270691
    },
    {
      "bits_per_byte": 1.8564204614055058,
      "learning_rate": 0.00025806271023100637,
      "loss_nats": 1.286772608757019,
      "step": 2360,
      "unclipped_grad_norm": 0.8822014927864075
    },
    {
      "bits_per_byte": 1.767254508221825,
      "learning_rate": 0.00025765765365601165,
      "loss_nats": 1.2249674797058105,
      "step": 2370,
      "unclipped_grad_norm": 0.8126892447471619
    },
    {
      "bits_per_byte": 1.8880877988915257,
      "learning_rate": 0.00025725101381736707,
      "loss_nats": 1.308722734451294,
      "step": 2380,
      "unclipped_grad_norm": 0.8927111625671387
    },
    {
      "bits_per_byte": 1.8776909316989163,
      "learning_rate": 0.00025684279766342567,
      "loss_nats": 1.3015161752700806,
      "step": 2390,
      "unclipped_grad_norm": 0.864246666431427
    },
    {
      "bits_per_byte": 1.9130975159799952,
      "learning_rate": 0.00025643301216947504,
      "loss_nats": 1.3260581493377686,
      "step": 2400,
      "unclipped_grad_norm": 0.9290322065353394
    },
    {
      "bits_per_byte": 1.848231851449975,
      "learning_rate": 0.00025602166433761866,
      "loss_nats": 1.2810966968536377,
      "step": 2410,
      "unclipped_grad_norm": 0.9126319885253906
    },
    {
      "bits_per_byte": 1.8675582298580322,
      "learning_rate": 0.0002556087611966561,
      "loss_nats": 1.2944927215576172,
      "step": 2420,
      "unclipped_grad_norm": 0.839264452457428
    },
    {
      "bits_per_byte": 1.8763154144573155,
      "learning_rate": 0.00025519430980196277,
      "loss_nats": 1.3005627393722534,
      "step": 2430,
      "unclipped_grad_norm": 0.8645790219306946
    },
    {
      "bits_per_byte": 1.8291109923096986,
      "learning_rate": 0.0002547783172353695,
      "loss_nats": 1.2678431272506714,
      "step": 2440,
      "unclipped_grad_norm": 0.8184406161308289
    },
    {
      "bits_per_byte": 1.9499289765234726,
      "learning_rate": 0.0002543607906050415,
      "loss_nats": 1.3515877723693848,
      "step": 2450,
      "unclipped_grad_norm": 0.9486745595932007
    },
    {
      "bits_per_byte": 1.82168426549749,
      "learning_rate": 0.0002539417370453568,
      "loss_nats": 1.2626953125,
      "step": 2460,
      "unclipped_grad_norm": 0.8475955128669739
    },
    {
      "bits_per_byte": 1.8165447359591058,
      "learning_rate": 0.00025352116371678464,
      "loss_nats": 1.2591328620910645,
      "step": 2470,
      "unclipped_grad_norm": 0.8975434899330139
    },
    {
      "bits_per_byte": 1.9011218480524652,
      "learning_rate": 0.0002530990778057627,
      "loss_nats": 1.317757248878479,
      "step": 2480,
      "unclipped_grad_norm": 0.9190422296524048
    },
    {
      "bits_per_byte": 1.8940924011637497,
      "learning_rate": 0.00025267548652457475,
      "loss_nats": 1.31288480758667,
      "step": 2490,
      "unclipped_grad_norm": 0.8232344388961792
    },
    {
      "bits_per_byte": 1.8466803959565414,
      "learning_rate": 0.00025225039711122696,
      "loss_nats": 1.2800213098526,
      "step": 2500,
      "unclipped_grad_norm": 0.7958891987800598
    },
    {
      "bits_per_byte": 1.815657993411257,
      "learning_rate": 0.00025182381682932464,
      "loss_nats": 1.2585182189941406,
      "step": 2510,
      "unclipped_grad_norm": 0.9414010643959045
    },
    {
      "bits_per_byte": 1.8382814512190053,
      "learning_rate": 0.0002513957529679477,
      "loss_nats": 1.2741996049880981,
      "step": 2520,
      "unclipped_grad_norm": 0.8523510694503784
    },
    {
      "bits_per_byte": 1.8459320994426374,
      "learning_rate": 0.00025096621284152656,
      "loss_nats": 1.2795026302337646,
      "step": 2530,
      "unclipped_grad_norm": 0.7949919700622559
    },
    {
      "bits_per_byte": 1.8449038151731287,
      "learning_rate": 0.0002505352037897168,
      "loss_nats": 1.2787898778915405,
      "step": 2540,
      "unclipped_grad_norm": 0.8316088318824768
    },
    {
      "bits_per_byte": 1.8031964744950397,
      "learning_rate": 0.0002501027331772739,
      "loss_nats": 1.2498805522918701,
      "step": 2550,
      "unclipped_grad_norm": 1.0299166440963745
    },
    {
      "bits_per_byte": 1.7815225329027051,
      "learning_rate": 0.00024966880839392746,
      "loss_nats": 1.2348573207855225,
      "step": 2560,
      "unclipped_grad_norm": 0.8839435577392578
    },
    {
      "bits_per_byte": 1.8260208080389957,
      "learning_rate": 0.0002492334368542548,
      "loss_nats": 1.265701174736023,
      "step": 2570,
      "unclipped_grad_norm": 0.9318905472755432
    },
    {
      "bits_per_byte": 1.7994856048373729,
      "learning_rate": 0.00024879662599755416,
      "loss_nats": 1.247308373451233,
      "step": 2580,
      "unclipped_grad_norm": 0.8335656523704529
    },
    {
      "bits_per_byte": 1.796869232769797,
      "learning_rate": 0.00024835838328771803,
      "loss_nats": 1.2454948425292969,
      "step": 2590,
      "unclipped_grad_norm": 0.8446885347366333
    },
    {
      "bits_per_byte": 1.824228920799656,
      "learning_rate": 0.0002479187162131051,
      "loss_nats": 1.2644591331481934,
      "step": 2600,
      "unclipped_grad_norm": 0.8396548628807068
    },
    {
      "bits_per_byte": 1.8444822856958638,
      "learning_rate": 0.00024747763228641264,
      "loss_nats": 1.2784976959228516,
      "step": 2610,
      "unclipped_grad_norm": 0.8530864119529724
    },
    {
      "bits_per_byte": 1.7628988756060733,
      "learning_rate": 0.00024703513904454795,
      "loss_nats": 1.2219483852386475,
      "step": 2620,
      "unclipped_grad_norm": 0.8868519067764282
    },
    {
      "bits_per_byte": 1.7732109553518074,
      "learning_rate": 0.00024659124404849956,
      "loss_nats": 1.2290961742401123,
      "step": 2630,
      "unclipped_grad_norm": 0.7921182513237
    },
    {
      "bits_per_byte": 1.7536496486426716,
      "learning_rate": 0.00024614595488320833,
      "loss_nats": 1.2155373096466064,
      "step": 2640,
      "unclipped_grad_norm": 0.8794499039649963
    },
    {
      "bits_per_byte": 1.832483744075769,
      "learning_rate": 0.00024569927915743745,
      "loss_nats": 1.2701809406280518,
      "step": 2650,
      "unclipped_grad_norm": 0.8381353616714478
    },
    {
      "bits_per_byte": 1.733659417202284,
      "learning_rate": 0.0002452512245036427,
      "loss_nats": 1.201681137084961,
      "step": 2660,
      "unclipped_grad_norm": 0.7765254974365234
    },
    {
      "bits_per_byte": 1.8051608603331948,
      "learning_rate": 0.00024480179857784177,
      "loss_nats": 1.2512421607971191,
      "step": 2670,
      "unclipped_grad_norm": 0.8680312633514404
    },
    {
      "bits_per_byte": 1.923795868759567,
      "learning_rate": 0.00024435100905948387,
      "loss_nats": 1.3334736824035645,
      "step": 2680,
      "unclipped_grad_norm": 0.9085569977760315
    },
    {
      "bits_per_byte": 1.8284822237381442,
      "learning_rate": 0.00024389886365131793,
      "loss_nats": 1.2674072980880737,
      "step": 2690,
      "unclipped_grad_norm": 0.8529220819473267
    },
    {
      "bits_per_byte": 1.7587946816258833,
      "learning_rate": 0.00024344537007926163,
      "loss_nats": 1.2191035747528076,
      "step": 2700,
      "unclipped_grad_norm": 0.8387420773506165
    },
    {
      "bits_per_byte": 1.8291632750355609,
      "learning_rate": 0.00024299053609226878,
      "loss_nats": 1.2678793668746948,
      "step": 2710,
      "unclipped_grad_norm": 0.7956082820892334
    },
    {
      "bits_per_byte": 1.872990129902883,
      "learning_rate": 0.00024253436946219733,
      "loss_nats": 1.298257827758789,
      "step": 2720,
      "unclipped_grad_norm": 0.7902501821517944
    },
    {
      "bits_per_byte": 1.7687435340129951,
      "learning_rate": 0.0002420768779836764,
      "loss_nats": 1.2259995937347412,
      "step": 2730,
      "unclipped_grad_norm": 0.8540245294570923
    },
    {
      "bits_per_byte": 1.8104493268472261,
      "learning_rate": 0.00024161806947397305,
      "loss_nats": 1.2549078464508057,
      "step": 2740,
      "unclipped_grad_norm": 0.8416579961776733
    },
    {
      "bits_per_byte": 1.7908353934469263,
      "learning_rate": 0.00024115795177285886,
      "loss_nats": 1.2413125038146973,
      "step": 2750,
      "unclipped_grad_norm": 0.7708072662353516
    },
    {
      "bits_per_byte": 1.9183405790741987,
      "learning_rate": 0.0002406965327424758,
      "loss_nats": 1.3296923637390137,
      "step": 2760,
      "unclipped_grad_norm": 0.9251072406768799
    },
    {
      "bits_per_byte": 1.8352038216818154,
      "learning_rate": 0.00024023382026720197,
      "loss_nats": 1.272066354751587,
      "step": 2770,
      "unclipped_grad_norm": 0.8847647905349731
    },
    {
      "bits_per_byte": 1.8372603902208329,
      "learning_rate": 0.00023976982225351698,
      "loss_nats": 1.2734918594360352,
      "step": 2780,
      "unclipped_grad_norm": 0.8566639423370361
    },
    {
      "bits_per_byte": 1.8040237110456898,
      "learning_rate": 0.0002393045466298666,
      "loss_nats": 1.2504539489746094,
      "step": 2790,
      "unclipped_grad_norm": 0.8849772214889526
    },
    {
      "bits_per_byte": 1.8313404034128333,
      "learning_rate": 0.00023883800134652763,
      "loss_nats": 1.2693884372711182,
      "step": 2800,
      "unclipped_grad_norm": 0.8829222321510315
    },
    {
      "bits_per_byte": 1.7673478948012433,
      "learning_rate": 0.0002383701943754715,
      "loss_nats": 1.2250322103500366,
      "step": 2810,
      "unclipped_grad_norm": 0.7698772549629211
    },
    {
      "bits_per_byte": 1.8410065163219265,
      "learning_rate": 0.00023790113371022884,
      "loss_nats": 1.2760884761810303,
      "step": 2820,
      "unclipped_grad_norm": 0.875986635684967
    },
    {
      "bits_per_byte": 1.7626952481474518,
      "learning_rate": 0.0002374308273657521,
      "loss_nats": 1.2218072414398193,
      "step": 2830,
      "unclipped_grad_norm": 0.815294623374939
    },
    {
      "bits_per_byte": 1.808952733819418,
      "learning_rate": 0.00023695928337827911,
      "loss_nats": 1.2538704872131348,
      "step": 2840,
      "unclipped_grad_norm": 0.7992775440216064
    },
    {
      "bits_per_byte": 1.7569166310784614,
      "learning_rate": 0.00023648650980519564,
      "loss_nats": 1.217801809310913,
      "step": 2850,
      "unclipped_grad_norm": 0.8472921848297119
    },
    {
      "bits_per_byte": 1.792434144169347,
      "learning_rate": 0.00023601251472489757,
      "loss_nats": 1.2424206733703613,
      "step": 2860,
      "unclipped_grad_norm": 0.8268847465515137
    },
    {
      "bits_per_byte": 1.793870887234655,
      "learning_rate": 0.00023553730623665305,
      "loss_nats": 1.2434165477752686,
      "step": 2870,
      "unclipped_grad_norm": 0.8292641639709473
    },
    {
      "bits_per_byte": 1.79901058875569,
      "learning_rate": 0.00023506089246046403,
      "loss_nats": 1.2469791173934937,
      "step": 2880,
      "unclipped_grad_norm": 0.8117106556892395
    },
    {
      "bits_per_byte": 1.7566032786885895,
      "learning_rate": 0.00023458328153692736,
      "loss_nats": 1.2175846099853516,
      "step": 2890,
      "unclipped_grad_norm": 0.8036413788795471
    },
    {
      "bits_per_byte": 1.7856263829175936,
      "learning_rate": 0.0002341044816270961,
      "loss_nats": 1.2377018928527832,
      "step": 2900,
      "unclipped_grad_norm": 0.7987298369407654
    },
    {
      "bits_per_byte": 1.7581136303284666,
      "learning_rate": 0.00023362450091233953,
      "loss_nats": 1.2186315059661865,
      "step": 2910,
      "unclipped_grad_norm": 0.8029706478118896
    },
    {
      "bits_per_byte": 1.8051701473963413,
      "learning_rate": 0.00023314334759420388,
      "loss_nats": 1.2512485980987549,
      "step": 2920,
      "unclipped_grad_norm": 0.7970700263977051
    },
    {
      "bits_per_byte": 1.756548588205615,
      "learning_rate": 0.0002326610298942717,
      "loss_nats": 1.2175467014312744,
      "step": 2930,
      "unclipped_grad_norm": 0.8035452961921692
    },
    {
      "bits_per_byte": 1.7718779178249697,
      "learning_rate": 0.00023217755605402175,
      "loss_nats": 1.2281721830368042,
      "step": 2940,
      "unclipped_grad_norm": 0.8500884771347046
    },
    {
      "bits_per_byte": 1.8224836408587004,
      "learning_rate": 0.00023169293433468802,
      "loss_nats": 1.263249397277832,
      "step": 2950,
      "unclipped_grad_norm": 0.8720842599868774
    },
    {
      "bits_per_byte": 1.8354507887684544,
      "learning_rate": 0.00023120717301711857,
      "loss_nats": 1.2722375392913818,
      "step": 2960,
      "unclipped_grad_norm": 0.8293792009353638
    },
    {
      "bits_per_byte": 1.8121727649915225,
      "learning_rate": 0.00023072028040163408,
      "loss_nats": 1.256102442741394,
      "step": 2970,
      "unclipped_grad_norm": 0.8630385994911194
    },
    {
      "bits_per_byte": 1.8230131754407068,
      "learning_rate": 0.00023023226480788593,
      "loss_nats": 1.2636164426803589,
      "step": 2980,
      "unclipped_grad_norm": 0.7837274670600891
    },
    {
      "bits_per_byte": 1.755593396562723,
      "learning_rate": 0.00022974313457471416,
      "loss_nats": 1.2168846130371094,
      "step": 2990,
      "unclipped_grad_norm": 0.8064510226249695
    },
    {
      "bits_per_byte": 1.8344553531852605,
      "learning_rate": 0.0002292528980600049,
      "loss_nats": 1.271547555923462,
      "step": 3000,
      "unclipped_grad_norm": 0.793539822101593
    },
    {
      "bits_per_byte": 1.7345470196633868,
      "learning_rate": 0.00022876156364054758,
      "loss_nats": 1.2022963762283325,
      "step": 3010,
      "unclipped_grad_norm": 0.8690526485443115
    },
    {
      "bits_per_byte": 1.8061236192127248,
      "learning_rate": 0.0002282691397118918,
      "loss_nats": 1.2519094944000244,
      "step": 3020,
      "unclipped_grad_norm": 0.9625511169433594
    },
    {
      "bits_per_byte": 1.7339495519342893,
      "learning_rate": 0.00022777563468820376,
      "loss_nats": 1.201882243156433,
      "step": 3030,
      "unclipped_grad_norm": 0.7808479070663452
    },
    {
      "bits_per_byte": 1.725635050678327,
      "learning_rate": 0.00022728105700212282,
      "loss_nats": 1.1961190700531006,
      "step": 3040,
      "unclipped_grad_norm": 0.8198943138122559
    },
    {
      "bits_per_byte": 1.8739061095013785,
      "learning_rate": 0.00022678541510461698,
      "loss_nats": 1.2988927364349365,
      "step": 3050,
      "unclipped_grad_norm": 0.8317142724990845
    },
    {
      "bits_per_byte": 1.8166181725510242,
      "learning_rate": 0.00022628871746483876,
      "loss_nats": 1.2591837644577026,
      "step": 3060,
      "unclipped_grad_norm": 0.8308979868888855
    },
    {
      "bits_per_byte": 1.7248370511783235,
      "learning_rate": 0.0002257909725699804,
      "loss_nats": 1.195565938949585,
      "step": 3070,
      "unclipped_grad_norm": 0.8185245394706726
    },
    {
      "bits_per_byte": 1.8189339189448919,
      "learning_rate": 0.00022529218892512886,
      "loss_nats": 1.260788917541504,
      "step": 3080,
      "unclipped_grad_norm": 0.8484242558479309
    },
    {
      "bits_per_byte": 1.7918220579149262,
      "learning_rate": 0.00022479237505312034,
      "loss_nats": 1.24199640750885,
      "step": 3090,
      "unclipped_grad_norm": 0.8042747974395752
    },
    {
      "bits_per_byte": 1.7567866121944091,
      "learning_rate": 0.00022429153949439496,
      "loss_nats": 1.2177116870880127,
      "step": 3100,
      "unclipped_grad_norm": 0.8309313058853149
    },
    {
      "bits_per_byte": 1.9063415215061514,
      "learning_rate": 0.00022378969080685053,
      "loss_nats": 1.3213752508163452,
      "step": 3110,
      "unclipped_grad_norm": 0.8743271231651306
    },
    {
      "bits_per_byte": 1.781927724028138,
      "learning_rate": 0.0002232868375656965,
      "loss_nats": 1.235138177871704,
      "step": 3120,
      "unclipped_grad_norm": 0.8567306995391846
    },
    {
      "bits_per_byte": 1.731061619261001,
      "learning_rate": 0.00022278298836330733,
      "loss_nats": 1.1998804807662964,
      "step": 3130,
      "unclipped_grad_norm": 0.8699595332145691
    },
    {
      "bits_per_byte": 1.7586823769548698,
      "learning_rate": 0.00022227815180907572,
      "loss_nats": 1.219025731086731,
      "step": 3140,
      "unclipped_grad_norm": 0.7885372042655945
    },
    {
      "bits_per_byte": 1.744890228268927,
      "learning_rate": 0.00022177233652926546,
      "loss_nats": 1.209465742111206,
      "step": 3150,
      "unclipped_grad_norm": 0.7910884022712708
    },
    {
      "bits_per_byte": 1.7638991267034914,
      "learning_rate": 0.00022126555116686416,
      "loss_nats": 1.2226417064666748,
      "step": 3160,
      "unclipped_grad_norm": 0.8217395544052124
    },
    {
      "bits_per_byte": 1.816583260072899,
      "learning_rate": 0.00022075780438143532,
      "loss_nats": 1.2591595649719238,
      "step": 3170,
      "unclipped_grad_norm": 0.8992500305175781
    },
    {
      "bits_per_byte": 1.8054272614593814,
      "learning_rate": 0.00022024910484897065,
      "loss_nats": 1.2514268159866333,
      "step": 3180,
      "unclipped_grad_norm": 0.8302581906318665
    },
    {
      "bits_per_byte": 1.7809063190646637,
      "learning_rate": 0.00021973946126174158,
      "loss_nats": 1.234430193901062,
      "step": 3190,
      "unclipped_grad_norm": 0.8399643898010254
    },
    {
      "bits_per_byte": 1.7853408917171614,
      "learning_rate": 0.00021922888232815086,
      "loss_nats": 1.237504005432129,
      "step": 3200,
      "unclipped_grad_norm": 0.8274973630905151
    },
    {
      "bits_per_byte": 1.7397025435883011,
      "learning_rate": 0.0002187173767725837,
      "loss_nats": 1.2058699131011963,
      "step": 3210,
      "unclipped_grad_norm": 0.8199328184127808
    },
    {
      "bits_per_byte": 1.7292408389363165,
      "learning_rate": 0.0002182049533352588,
      "loss_nats": 1.1986184120178223,
      "step": 3220,
      "unclipped_grad_norm": 0.81409752368927
    },
    {
      "bits_per_byte": 1.7790022991369616,
      "learning_rate": 0.00021769162077207884,
      "loss_nats": 1.2331104278564453,
      "step": 3230,
      "unclipped_grad_norm": 0.816062867641449
    },
    {
      "bits_per_byte": 1.7934072220079287,
      "learning_rate": 0.00021717738785448102,
      "loss_nats": 1.2430951595306396,
      "step": 3240,
      "unclipped_grad_norm": 0.9001877307891846
    },
    {
      "bits_per_byte": 1.765244718963842,
      "learning_rate": 0.00021666226336928708,
      "loss_nats": 1.2235743999481201,
      "step": 3250,
      "unclipped_grad_norm": 0.8105912804603577
    },
    {
      "bits_per_byte": 1.7706184888727015,
      "learning_rate": 0.0002161462561185532,
      "loss_nats": 1.2272992134094238,
      "step": 3260,
      "unclipped_grad_norm": 0.8049122095108032
    },
    {
      "bits_per_byte": 1.8044983831620713,
      "learning_rate": 0.00021562937491941957,
      "loss_nats": 1.2507829666137695,
      "step": 3270,
      "unclipped_grad_norm": 0.8440065979957581
    },
    {
      "bits_per_byte": 1.793506112032175,
      "learning_rate": 0.00021511162860395975,
      "loss_nats": 1.2431637048721313,
      "step": 3280,
      "unclipped_grad_norm": 0.8675442337989807
    },
    {
      "bits_per_byte": 1.7443828794488816,
      "learning_rate": 0.0002145930260190298,
      "loss_nats": 1.2091140747070312,
      "step": 3290,
      "unclipped_grad_norm": 0.8013374209403992
    },
    {
      "bits_per_byte": 1.7796785349201543,
      "learning_rate": 0.00021407357602611698,
      "loss_nats": 1.233579158782959,
      "step": 3300,
      "unclipped_grad_norm": 0.8424150943756104
    },
    {
      "bits_per_byte": 1.7358951916635004,
      "learning_rate": 0.0002135532875011885,
      "loss_nats": 1.203230857849121,
      "step": 3310,
      "unclipped_grad_norm": 0.7571165561676025
    },
    {
      "bits_per_byte": 1.7447595214542713,
      "learning_rate": 0.0002130321693345397,
      "loss_nats": 1.2093751430511475,
      "step": 3320,
      "unclipped_grad_norm": 0.7720819115638733
    },
    {
      "bits_per_byte": 1.7607893364105902,
      "learning_rate": 0.00021251023043064224,
      "loss_nats": 1.2204861640930176,
      "step": 3330,
      "unclipped_grad_norm": 0.8520592451095581
    },
    {
      "bits_per_byte": 1.6976324914995427,
      "learning_rate": 0.00021198747970799188,
      "loss_nats": 1.1767091751098633,
      "step": 3340,
      "unclipped_grad_norm": 0.80418860912323
    },
    {
      "bits_per_byte": 1.7457931371859567,
      "learning_rate": 0.00021146392609895618,
      "loss_nats": 1.2100915908813477,
      "step": 3350,
      "unclipped_grad_norm": 0.8466405868530273
    },
    {
      "bits_per_byte": 1.7522414546974068,
      "learning_rate": 0.00021093957854962178,
      "loss_nats": 1.2145612239837646,
      "step": 3360,
      "unclipped_grad_norm": 0.7725750803947449
    },
    {
      "bits_per_byte": 1.7121540186077948,
      "learning_rate": 0.00021041444601964157,
      "loss_nats": 1.186774730682373,
      "step": 3370,
      "unclipped_grad_norm": 0.779502809047699
    },
    {
      "bits_per_byte": 1.7592602386617688,
      "learning_rate": 0.00020988853748208164,
      "loss_nats": 1.2194262742996216,
      "step": 3380,
      "unclipped_grad_norm": 0.8456810712814331
    },
    {
      "bits_per_byte": 1.7687302913488787,
      "learning_rate": 0.00020936186192326782,
      "loss_nats": 1.2259904146194458,
      "step": 3390,
      "unclipped_grad_norm": 0.8990693688392639
    },
    {
      "bits_per_byte": 1.7458971866897286,
      "learning_rate": 0.00020883442834263225,
      "loss_nats": 1.2101637125015259,
      "step": 3400,
      "unclipped_grad_norm": 0.8017266392707825
    },
    {
      "bits_per_byte": 1.738469084016313,
      "learning_rate": 0.0002083062457525596,
      "loss_nats": 1.205014944076538,
      "step": 3410,
      "unclipped_grad_norm": 0.8378265500068665
    },
    {
      "bits_per_byte": 1.7225225086630118,
      "learning_rate": 0.00020777732317823306,
      "loss_nats": 1.1939616203308105,
      "step": 3420,
      "unclipped_grad_norm": 0.8268731832504272
    },
    {
      "bits_per_byte": 1.7602646173428078,
      "learning_rate": 0.00020724766965748005,
      "loss_nats": 1.2201224565505981,
      "step": 3430,
      "unclipped_grad_norm": 0.8129444122314453
    },
    {
      "bits_per_byte": 1.768388733804265,
      "learning_rate": 0.00020671729424061788,
      "loss_nats": 1.225753664970398,
      "step": 3440,
      "unclipped_grad_norm": 0.8220692873001099
    },
    {
      "bits_per_byte": 1.8502017407329574,
      "learning_rate": 0.00020618620599029906,
      "loss_nats": 1.2824621200561523,
      "step": 3450,
      "unclipped_grad_norm": 0.8645455837249756
    },
    {
      "bits_per_byte": 1.8368216624784819,
      "learning_rate": 0.00020565441398135649,
      "loss_nats": 1.2731877565383911,
      "step": 3460,
      "unclipped_grad_norm": 0.8615949153900146
    },
    {
      "bits_per_byte": 1.785747458703801,
      "learning_rate": 0.00020512192730064827,
      "loss_nats": 1.237785816192627,
      "step": 3470,
      "unclipped_grad_norm": 0.8510363101959229
    },
    {
      "bits_per_byte": 1.7659735814381987,
      "learning_rate": 0.00020458875504690273,
      "loss_nats": 1.2240796089172363,
      "step": 3480,
      "unclipped_grad_norm": 0.8455166816711426
    },
    {
      "bits_per_byte": 1.6893135186946577,
      "learning_rate": 0.00020405490633056245,
      "loss_nats": 1.1709429025650024,
      "step": 3490,
      "unclipped_grad_norm": 0.8278471231460571
    },
    {
      "bits_per_byte": 1.7031111708254283,
      "learning_rate": 0.00020352039027362917,
      "loss_nats": 1.180506706237793,
      "step": 3500,
      "unclipped_grad_norm": 0.814669132232666
    },
    {
      "bits_per_byte": 1.7385170671759038,
      "learning_rate": 0.0002029852160095075,
      "loss_nats": 1.2050482034683228,
      "step": 3510,
      "unclipped_grad_norm": 0.9123345017433167
    },
    {
      "bits_per_byte": 1.700717172325418,
      "learning_rate": 0.000202449392682849,
      "loss_nats": 1.178847312927246,
      "step": 3520,
      "unclipped_grad_norm": 0.7641705870628357
    },
    {
      "bits_per_byte": 1.742950951897798,
      "learning_rate": 0.0002019129294493959,
      "loss_nats": 1.2081215381622314,
      "step": 3530,
      "unclipped_grad_norm": 0.855086624622345
    },
    {
      "bits_per_byte": 1.733061433525234,
      "learning_rate": 0.00020137583547582478,
      "loss_nats": 1.2012666463851929,
      "step": 3540,
      "unclipped_grad_norm": 0.8321052193641663
    },
    {
      "bits_per_byte": 1.7782098030817857,
      "learning_rate": 0.00020083811993958973,
      "loss_nats": 1.2325611114501953,
      "step": 3550,
      "unclipped_grad_norm": 0.8241682052612305
    },
    {
      "bits_per_byte": 1.7088610667917246,
      "learning_rate": 0.00020029979202876563,
      "loss_nats": 1.1844922304153442,
      "step": 3560,
      "unclipped_grad_norm": 0.7756084203720093
    },
    {
      "bits_per_byte": 1.7644671853992913,
      "learning_rate": 0.00019976086094189118,
      "loss_nats": 1.223035454750061,
      "step": 3570,
      "unclipped_grad_norm": 0.8851686716079712
    },
    {
      "bits_per_byte": 1.8350844657221166,
      "learning_rate": 0.00019922133588781175,
      "loss_nats": 1.2719836235046387,
      "step": 3580,
      "unclipped_grad_norm": 0.8648787140846252
    },
    {
      "bits_per_byte": 1.758608252432348,
      "learning_rate": 0.0001986812260855219,
      "loss_nats": 1.2189743518829346,
      "step": 3590,
      "unclipped_grad_norm": 0.8309826254844666
    },
    {
      "bits_per_byte": 1.7417491371335687,
      "learning_rate": 0.00019814054076400787,
      "loss_nats": 1.2072885036468506,
      "step": 3600,
      "unclipped_grad_norm": 0.9055862426757812
    },
    {
      "bits_per_byte": 1.7217079988285255,
      "learning_rate": 0.00019759928916209004,
      "loss_nats": 1.193397045135498,
      "step": 3610,
      "unclipped_grad_norm": 0.8098222613334656
    },
    {
      "bits_per_byte": 1.7173626851718258,
      "learning_rate": 0.00019705748052826493,
      "loss_nats": 1.190385103225708,
      "step": 3620,
      "unclipped_grad_norm": 0.8190554976463318
    },
    {
      "bits_per_byte": 1.7436209963055593,
      "learning_rate": 0.00019651512412054723,
      "loss_nats": 1.2085859775543213,
      "step": 3630,
      "unclipped_grad_norm": 0.8753142356872559
    },
    {
      "bits_per_byte": 1.6961333187319716,
      "learning_rate": 0.00019597222920631144,
      "loss_nats": 1.1756700277328491,
      "step": 3640,
      "unclipped_grad_norm": 0.841607391834259
    },
    {
      "bits_per_byte": 1.7716670670950119,
      "learning_rate": 0.0001954288050621338,
      "loss_nats": 1.228026032447815,
      "step": 3650,
      "unclipped_grad_norm": 0.8459575772285461
    },
    {
      "bits_per_byte": 1.6801827877777005,
      "learning_rate": 0.00019488486097363363,
      "loss_nats": 1.164613962173462,
      "step": 3660,
      "unclipped_grad_norm": 0.8714450001716614
    },
    {
      "bits_per_byte": 1.6935544388822839,
      "learning_rate": 0.00019434040623531452,
      "loss_nats": 1.1738824844360352,
      "step": 3670,
      "unclipped_grad_norm": 0.7989607453346252
    },
    {
      "bits_per_byte": 1.7486903569223915,
      "learning_rate": 0.0001937954501504058,
      "loss_nats": 1.2120997905731201,
      "step": 3680,
      "unclipped_grad_norm": 0.8391386270523071
    },
    {
      "bits_per_byte": 1.7119070515211559,
      "learning_rate": 0.00019325000203070333,
      "loss_nats": 1.1866035461425781,
      "step": 3690,
      "unclipped_grad_norm": 0.8321404457092285
    },
    {
      "bits_per_byte": 1.6629633688253633,
      "learning_rate": 0.0001927040711964106,
      "loss_nats": 1.152678370475769,
      "step": 3700,
      "unclipped_grad_norm": 0.8227249383926392
    },
    {
      "bits_per_byte": 1.726790602109474,
      "learning_rate": 0.00019215766697597923,
      "loss_nats": 1.1969200372695923,
      "step": 3710,
      "unclipped_grad_norm": 0.8174790143966675
    },
    {
      "bits_per_byte": 1.7194202856067484,
      "learning_rate": 0.00019161079870594978,
      "loss_nats": 1.1918113231658936,
      "step": 3720,
      "unclipped_grad_norm": 0.822218120098114
    },
    {
      "bits_per_byte": 1.767769424278508,
      "learning_rate": 0.0001910634757307922,
      "loss_nats": 1.2253243923187256,
      "step": 3730,
      "unclipped_grad_norm": 0.8156639933586121
    },
    {
      "bits_per_byte": 1.7041993050574373,
      "learning_rate": 0.0001905157074027459,
      "loss_nats": 1.1812609434127808,
      "step": 3740,
      "unclipped_grad_norm": 0.7992775440216064
    },
    {
      "bits_per_byte": 1.6957810982630046,
      "learning_rate": 0.00018996750308166037,
      "loss_nats": 1.1754258871078491,
      "step": 3750,
      "unclipped_grad_norm": 0.8203845620155334
    },
    {
      "bits_per_byte": 1.6945959658159089,
      "learning_rate": 0.00018941887213483482,
      "loss_nats": 1.1746044158935547,
      "step": 3760,
      "unclipped_grad_norm": 0.8056831359863281
    },
    {
      "bits_per_byte": 1.7464552703917786,
      "learning_rate": 0.00018886982393685845,
      "loss_nats": 1.2105505466461182,
      "step": 3770,
      "unclipped_grad_norm": 0.9159992337226868
    },
    {
      "bits_per_byte": 1.6766162115641075,
      "learning_rate": 0.00018832036786945008,
      "loss_nats": 1.1621417999267578,
      "step": 3780,
      "unclipped_grad_norm": 0.7913095951080322
    },
    {
      "bits_per_byte": 1.7293030966559289,
      "learning_rate": 0.0001877705133212979,
      "loss_nats": 1.1986615657806396,
      "step": 3790,
      "unclipped_grad_norm": 0.8222495913505554
    },
    {
      "bits_per_byte": 1.6807076788281337,
      "learning_rate": 0.00018722026968789906,
      "loss_nats": 1.164977788925171,
      "step": 3800,
      "unclipped_grad_norm": 0.829034686088562
    },
    {
      "bits_per_byte": 1.6730726810257301,
      "learning_rate": 0.00018666964637139907,
      "loss_nats": 1.1596856117248535,
      "step": 3810,
      "unclipped_grad_norm": 0.8109799027442932
    },
    {
      "bits_per_byte": 1.7832320404522815,
      "learning_rate": 0.00018611865278043115,
      "loss_nats": 1.2360422611236572,
      "step": 3820,
      "unclipped_grad_norm": 0.8464360237121582
    },
    {
      "bits_per_byte": 1.7013447370184165,
      "learning_rate": 0.00018556729832995555,
      "loss_nats": 1.179282307624817,
      "step": 3830,
      "unclipped_grad_norm": 0.849337637424469
    },
    {
      "bits_per_byte": 1.7095980124506718,
      "learning_rate": 0.00018501559244109864,
      "loss_nats": 1.1850030422210693,
      "step": 3840,
      "unclipped_grad_norm": 0.8549852967262268
    },
    {
      "bits_per_byte": 1.7414235739754853,
      "learning_rate": 0.00018446354454099178,
      "loss_nats": 1.207062840461731,
      "step": 3850,
      "unclipped_grad_norm": 0.8230528235435486
    },
    {
      "bits_per_byte": 1.6685270075807752,
      "learning_rate": 0.00018391116406261048,
      "loss_nats": 1.1565347909927368,
      "step": 3860,
      "unclipped_grad_norm": 0.8480414748191833
    },
    {
      "bits_per_byte": 1.7316664822440857,
      "learning_rate": 0.00018335846044461305,
      "loss_nats": 1.2002997398376465,
      "step": 3870,
      "unclipped_grad_norm": 0.8396956324577332
    },
    {
      "bits_per_byte": 1.7011863409969716,
      "learning_rate": 0.0001828054431311793,
      "loss_nats": 1.1791725158691406,
      "step": 3880,
      "unclipped_grad_norm": 0.8941359519958496
    },
    {
      "bits_per_byte": 1.7358762735719053,
      "learning_rate": 0.00018225212157184934,
      "loss_nats": 1.2032177448272705,
      "step": 3890,
      "unclipped_grad_norm": 0.894798755645752
    },
    {
      "bits_per_byte": 1.723932594417436,
      "learning_rate": 0.00018169850522136192,
      "loss_nats": 1.1949390172958374,
      "step": 3900,
      "unclipped_grad_norm": 0.856995165348053
    },
    {
      "bits_per_byte": 1.7425093004503822,
      "learning_rate": 0.000181144603539493,
      "loss_nats": 1.207815408706665,
      "step": 3910,
      "unclipped_grad_norm": 0.815839409828186
    },
    {
      "bits_per_byte": 1.6972770033602091,
      "learning_rate": 0.00018059042599089412,
      "loss_nats": 1.1764627695083618,
      "step": 3920,
      "unclipped_grad_norm": 0.7732813358306885
    },
    {
      "bits_per_byte": 1.6785904004133616,
      "learning_rate": 0.00018003598204493048,
      "loss_nats": 1.1635102033615112,
      "step": 3930,
      "unclipped_grad_norm": 0.8116664886474609
    },
    {
      "bits_per_byte": 1.6934890854749558,
      "learning_rate": 0.00017948128117551936,
      "loss_nats": 1.1738371849060059,
      "step": 3940,
      "unclipped_grad_norm": 0.8463517427444458
    },
    {
      "bits_per_byte": 1.7793810049341614,
      "learning_rate": 0.00017892633286096816,
      "loss_nats": 1.2333729267120361,
      "step": 3950,
      "unclipped_grad_norm": 0.8989379405975342
    },
    {
      "bits_per_byte": 1.7841349493693113,
      "learning_rate": 0.00017837114658381247,
      "loss_nats": 1.2366681098937988,
      "step": 3960,
      "unclipped_grad_norm": 0.8764044642448425
    },
    {
      "bits_per_byte": 1.7434844420807742,
      "learning_rate": 0.00017781573183065386,
      "loss_nats": 1.208491325378418,
      "step": 3970,
      "unclipped_grad_norm": 0.842854380607605
    },
    {
      "bits_per_byte": 1.7365705675334386,
      "learning_rate": 0.0001772600980919981,
      "loss_nats": 1.203698992729187,
      "step": 3980,
      "unclipped_grad_norm": 0.8656630516052246
    },
    {
      "bits_per_byte": 1.7578855813334224,
      "learning_rate": 0.0001767042548620927,
      "loss_nats": 1.2184734344482422,
      "step": 3990,
      "unclipped_grad_norm": 0.8737640976905823
    },
    {
      "bits_per_byte": 1.7628734221737457,
      "learning_rate": 0.00017614821163876486,
      "loss_nats": 1.221930742263794,
      "step": 4000,
      "unclipped_grad_norm": 0.8737133741378784
    },
    {
      "bits_per_byte": 1.7856131402534772,
      "learning_rate": 0.00017559197792325906,
      "loss_nats": 1.2376927137374878,
      "step": 4010,
      "unclipped_grad_norm": 0.8777877688407898
    },
    {
      "bits_per_byte": 1.6452555191445761,
      "learning_rate": 0.0001750355632200748,
      "loss_nats": 1.140404224395752,
      "step": 4020,
      "unclipped_grad_norm": 0.7845610976219177
    },
    {
      "bits_per_byte": 1.6881839366437907,
      "learning_rate": 0.00017447897703680418,
      "loss_nats": 1.170159935951233,
      "step": 4030,
      "unclipped_grad_norm": 0.8164089322090149
    },
    {
      "bits_per_byte": 1.7054205538612142,
      "learning_rate": 0.0001739222288839694,
      "loss_nats": 1.1821074485778809,
      "step": 4040,
      "unclipped_grad_norm": 0.8447781205177307
    },
    {
      "bits_per_byte": 1.6980753468255143,
      "learning_rate": 0.00017336532827486026,
      "loss_nats": 1.1770161390304565,
      "step": 4050,
      "unclipped_grad_norm": 0.8900085091590881
    },
    {
      "bits_per_byte": 1.715704084486905,
      "learning_rate": 0.00017280828472537163,
      "loss_nats": 1.1892354488372803,
      "step": 4060,
      "unclipped_grad_norm": 0.864895761013031
    },
    {
      "bits_per_byte": 1.6694995694714043,
      "learning_rate": 0.0001722511077538408,
      "loss_nats": 1.1572089195251465,
      "step": 4070,
      "unclipped_grad_norm": 0.8258009552955627
    },
    {
      "bits_per_byte": 1.7115804564671673,
      "learning_rate": 0.00017169380688088497,
      "loss_nats": 1.1863771677017212,
      "step": 4080,
      "unclipped_grad_norm": 0.8420462012290955
    },
    {
      "bits_per_byte": 1.656621852540099,
      "learning_rate": 0.0001711363916292384,
      "loss_nats": 1.148282766342163,
      "step": 4090,
      "unclipped_grad_norm": 0.8618301153182983
    },
    {
      "bits_per_byte": 1.6356337777594054,
      "learning_rate": 0.00017057887152358974,
      "loss_nats": 1.133734941482544,
      "step": 4100,
      "unclipped_grad_norm": 0.7950634360313416
    },
    {
      "bits_per_byte": 1.7423063609223641,
      "learning_rate": 0.0001700212560904194,
      "loss_nats": 1.2076747417449951,
      "step": 4110,
      "unclipped_grad_norm": 0.9093177914619446
    },
    {
      "bits_per_byte": 1.6838860901987291,
      "learning_rate": 0.00016946355485783656,
      "loss_nats": 1.1671808958053589,
      "step": 4120,
      "unclipped_grad_norm": 0.8371159434318542
    },
    {
      "bits_per_byte": 1.6537941137946128,
      "learning_rate": 0.0001689057773554166,
      "loss_nats": 1.1463227272033691,
      "step": 4130,
      "unclipped_grad_norm": 0.8010553121566772
    },
    {
      "bits_per_byte": 1.6821887934173643,
      "learning_rate": 0.00016834793311403806,
      "loss_nats": 1.1660044193267822,
      "step": 4140,
      "unclipped_grad_norm": 0.8702026009559631
    },
    {
      "bits_per_byte": 1.7209915191050311,
      "learning_rate": 0.00016779003166571981,
      "loss_nats": 1.1929004192352295,
      "step": 4150,
      "unclipped_grad_norm": 0.8466006517410278
    },
    {
      "bits_per_byte": 1.6966901985554654,
      "learning_rate": 0.00016723208254345838,
      "loss_nats": 1.1760560274124146,
      "step": 4160,
      "unclipped_grad_norm": 0.869064450263977
    },
    {
      "bits_per_byte": 1.6446666505480219,
      "learning_rate": 0.00016667409528106481,
      "loss_nats": 1.13999605178833,
      "step": 4170,
      "unclipped_grad_norm": 0.8484132289886475
    },
    {
      "bits_per_byte": 1.6167019275524273,
      "learning_rate": 0.00016611607941300184,
      "loss_nats": 1.120612382888794,
      "step": 4180,
      "unclipped_grad_norm": 0.8314529657363892
    },
    {
      "bits_per_byte": 1.7785682149261839,
      "learning_rate": 0.0001655580444742211,
      "loss_nats": 1.2328095436096191,
      "step": 4190,
      "unclipped_grad_norm": 0.8785865902900696
    },
    {
      "bits_per_byte": 1.6748427264684103,
      "learning_rate": 0.00016499999999999997,
      "loss_nats": 1.1609125137329102,
      "step": 4200,
      "unclipped_grad_norm": 0.8246812224388123
    },
    {
      "bits_per_byte": 1.679696764806362,
      "learning_rate": 0.0001644419555257789,
      "loss_nats": 1.1642770767211914,
      "step": 4210,
      "unclipped_grad_norm": 0.846137285232544
    },
    {
      "bits_per_byte": 1.73478487166953,
      "learning_rate": 0.0001638839205869981,
      "loss_nats": 1.2024612426757812,
      "step": 4220,
      "unclipped_grad_norm": 0.9032177925109863
    },
    {
      "bits_per_byte": 1.6973831166557916,
      "learning_rate": 0.0001633259047189352,
      "loss_nats": 1.1765363216400146,
      "step": 4230,
      "unclipped_grad_norm": 0.8501660227775574
    },
    {
      "bits_per_byte": 1.7410484798139536,
      "learning_rate": 0.0001627679174565416,
      "loss_nats": 1.2068028450012207,
      "step": 4240,
      "unclipped_grad_norm": 0.8855612874031067
    },
    {
      "bits_per_byte": 1.756008390699255,
      "learning_rate": 0.00016220996833428016,
      "loss_nats": 1.2171722650527954,
      "step": 4250,
      "unclipped_grad_norm": 0.849215567111969
    },
    {
      "bits_per_byte": 1.7259122867115178,
      "learning_rate": 0.00016165206688596191,
      "loss_nats": 1.1963112354278564,
      "step": 4260,
      "unclipped_grad_norm": 0.8852341175079346
    },
    {
      "bits_per_byte": 1.7235939605778872,
      "learning_rate": 0.00016109422264458335,
      "loss_nats": 1.194704294204712,
      "step": 4270,
      "unclipped_grad_norm": 0.8789628148078918
    },
    {
      "bits_per_byte": 1.751234840241907,
      "learning_rate": 0.00016053644514216336,
      "loss_nats": 1.213863492012024,
      "step": 4280,
      "unclipped_grad_norm": 0.923163115978241
    },
    {
      "bits_per_byte": 1.688365378340451,
      "learning_rate": 0.00015997874390958058,
      "loss_nats": 1.170285701751709,
      "step": 4290,
      "unclipped_grad_norm": 0.8880194425582886
    },
    {
      "bits_per_byte": 1.629311007583085,
      "learning_rate": 0.00015942112847641018,
      "loss_nats": 1.129352331161499,
      "step": 4300,
      "unclipped_grad_norm": 0.8913658261299133
    },
    {
      "bits_per_byte": 1.771773524355896,
      "learning_rate": 0.00015886360837076156,
      "loss_nats": 1.2280998229980469,
      "step": 4310,
      "unclipped_grad_norm": 0.9543031454086304
    },
    {
      "bits_per_byte": 1.6490332900534286,
      "learning_rate": 0.00015830619311911495,
      "loss_nats": 1.1430227756500244,
      "step": 4320,
      "unclipped_grad_norm": 0.8419790267944336
    },
    {
      "bits_per_byte": 1.6371308867351662,
      "learning_rate": 0.00015774889224615915,
      "loss_nats": 1.1347726583480835,
      "step": 4330,
      "unclipped_grad_norm": 0.8086377382278442
    },
    {
      "bits_per_byte": 1.6438610838113803,
      "learning_rate": 0.00015719171527462834,
      "loss_nats": 1.1394376754760742,
      "step": 4340,
      "unclipped_grad_norm": 0.8752440810203552
    },
    {
      "bits_per_byte": 1.5801602577756537,
      "learning_rate": 0.0001566346717251397,
      "loss_nats": 1.0952836275100708,
      "step": 4350,
      "unclipped_grad_norm": 0.8255703449249268
    },
    {
      "bits_per_byte": 1.7363834504092999,
      "learning_rate": 0.00015607777111603056,
      "loss_nats": 1.2035692930221558,
      "step": 4360,
      "unclipped_grad_norm": 0.9419499039649963
    },
    {
      "bits_per_byte": 1.6755530148164737,
      "learning_rate": 0.00015552102296319576,
      "loss_nats": 1.1614048480987549,
      "step": 4370,
      "unclipped_grad_norm": 0.8658137917518616
    },
    {
      "bits_per_byte": 1.644153798283149,
      "learning_rate": 0.00015496443677992516,
      "loss_nats": 1.1396405696868896,
      "step": 4380,
      "unclipped_grad_norm": 0.8454349040985107
    },
    {
      "bits_per_byte": 1.6710379542833722,
      "learning_rate": 0.0001544080220767409,
      "loss_nats": 1.1582752466201782,
      "step": 4390,
      "unclipped_grad_norm": 0.8288133144378662
    },
    {
      "bits_per_byte": 1.6521485837911574,
      "learning_rate": 0.00015385178836123511,
      "loss_nats": 1.1451821327209473,
      "step": 4400,
      "unclipped_grad_norm": 0.8934062719345093
    },
    {
      "bits_per_byte": 1.6486657631285346,
      "learning_rate": 0.00015329574513790726,
      "loss_nats": 1.1427680253982544,
      "step": 4410,
      "unclipped_grad_norm": 0.945034921169281
    },
    {
      "bits_per_byte": 1.6360498037918425,
      "learning_rate": 0.00015273990190800186,
      "loss_nats": 1.1340233087539673,
      "step": 4420,
      "unclipped_grad_norm": 0.8596903681755066
    },
    {
      "bits_per_byte": 1.6605176035474434,
      "learning_rate": 0.0001521842681693461,
      "loss_nats": 1.1509830951690674,
      "step": 4430,
      "unclipped_grad_norm": 0.8684608340263367
    },
    {
      "bits_per_byte": 1.6658086498012377,
      "learning_rate": 0.00015162885341618752,
      "loss_nats": 1.1546505689620972,
      "step": 4440,
      "unclipped_grad_norm": 0.8835514783859253
    },
    {
      "bits_per_byte": 1.6458364765391906,
      "learning_rate": 0.00015107366713903178,
      "loss_nats": 1.1408069133758545,
      "step": 4450,
      "unclipped_grad_norm": 0.8609577417373657
    },
    {
      "bits_per_byte": 1.6156653881156773,
      "learning_rate": 0.0001505187188244806,
      "loss_nats": 1.1198939085006714,
      "step": 4460,
      "unclipped_grad_norm": 0.8464177250862122
    },
    {
      "bits_per_byte": 1.7173920942051233,
      "learning_rate": 0.00014996401795506952,
      "loss_nats": 1.1904054880142212,
      "step": 4470,
      "unclipped_grad_norm": 0.940384030342102
    },
    {
      "bits_per_byte": 1.6389153787205186,
      "learning_rate": 0.00014940957400910585,
      "loss_nats": 1.1360095739364624,
      "step": 4480,
      "unclipped_grad_norm": 0.8790894746780396
    },
    {
      "bits_per_byte": 1.590423322465892,
      "learning_rate": 0.00014885539646050699,
      "loss_nats": 1.1023974418640137,
      "step": 4490,
      "unclipped_grad_norm": 0.8159270286560059
    },
    {
      "bits_per_byte": 1.6820450159212428,
      "learning_rate": 0.00014830149477863805,
      "loss_nats": 1.1659047603607178,
      "step": 4500,
      "unclipped_grad_norm": 0.8648387789726257
    },
    {
      "bits_per_byte": 1.6443777196945721,
      "learning_rate": 0.00014774787842815066,
      "loss_nats": 1.1397957801818848,
      "step": 4510,
      "unclipped_grad_norm": 0.874046266078949
    },
    {
      "bits_per_byte": 1.6401994011918604,
      "learning_rate": 0.00014719455686882066,
      "loss_nats": 1.1368995904922485,
      "step": 4520,
      "unclipped_grad_norm": 0.8307194709777832
    },
    {
      "bits_per_byte": 1.6802708428949422,
      "learning_rate": 0.00014664153955538695,
      "loss_nats": 1.164674997329712,
      "step": 4530,
      "unclipped_grad_norm": 0.8542748689651489
    },
    {
      "bits_per_byte": 1.6174216749462882,
      "learning_rate": 0.00014608883593738946,
      "loss_nats": 1.121111273765564,
      "step": 4540,
      "unclipped_grad_norm": 0.7951624989509583
    },
    {
      "bits_per_byte": 1.6783786897701496,
      "learning_rate": 0.00014553645545900817,
      "loss_nats": 1.1633634567260742,
      "step": 4550,
      "unclipped_grad_norm": 0.9085310697555542
    },
    {
      "bits_per_byte": 1.678087351159588,
      "learning_rate": 0.0001449844075589013,
      "loss_nats": 1.1631615161895752,
      "step": 4560,
      "unclipped_grad_norm": 0.8672650456428528
    },
    {
      "bits_per_byte": 1.6438235915934922,
      "learning_rate": 0.0001444327016700444,
      "loss_nats": 1.1394116878509521,
      "step": 4570,
      "unclipped_grad_norm": 0.8864105343818665
    },
    {
      "bits_per_byte": 1.5942666187647232,
      "learning_rate": 0.0001438813472195688,
      "loss_nats": 1.105061411857605,
      "step": 4580,
      "unclipped_grad_norm": 0.8459784984588623
    },
    {
      "bits_per_byte": 1.7013707063986967,
      "learning_rate": 0.00014333035362860093,
      "loss_nats": 1.179300308227539,
      "step": 4590,
      "unclipped_grad_norm": 0.864471971988678
    },
    {
      "bits_per_byte": 1.685760529110483,
      "learning_rate": 0.00014277973031210088,
      "loss_nats": 1.1684801578521729,
      "step": 4600,
      "unclipped_grad_norm": 0.8711076378822327
    },
    {
      "bits_per_byte": 1.6521852160957913,
      "learning_rate": 0.00014222948667870207,
      "loss_nats": 1.1452075242996216,
      "step": 4610,
      "unclipped_grad_norm": 0.8594344258308411
    },
    {
      "bits_per_byte": 1.5889760884588813,
      "learning_rate": 0.00014167963213054991,
      "loss_nats": 1.1013942956924438,
      "step": 4620,
      "unclipped_grad_norm": 0.8217349052429199
    },
    {
      "bits_per_byte": 1.6772566749559206,
      "learning_rate": 0.00014113017606314152,
      "loss_nats": 1.162585735321045,
      "step": 4630,
      "unclipped_grad_norm": 0.8906145095825195
    },
    {
      "bits_per_byte": 1.6271617403952525,
      "learning_rate": 0.00014058112786516518,
      "loss_nats": 1.127862572669983,
      "step": 4640,
      "unclipped_grad_norm": 0.8258018493652344
    },
    {
      "bits_per_byte": 1.61576359020932,
      "learning_rate": 0.0001400324969183396,
      "loss_nats": 1.1199619770050049,
      "step": 4650,
      "unclipped_grad_norm": 0.9025488495826721
    },
    {
      "bits_per_byte": 1.6536071686531248,
      "learning_rate": 0.00013948429259725406,
      "loss_nats": 1.1461931467056274,
      "step": 4660,
      "unclipped_grad_norm": 0.905563473701477
    },
    {
      "bits_per_byte": 1.6824091031931194,
      "learning_rate": 0.00013893652426920776,
      "loss_nats": 1.1661571264266968,
      "step": 4670,
      "unclipped_grad_norm": 0.8757683038711548
    },
    {
      "bits_per_byte": 1.6350793056930237,
      "learning_rate": 0.00013838920129405016,
      "loss_nats": 1.1333506107330322,
      "step": 4680,
      "unclipped_grad_norm": 0.8878620862960815
    },
    {
      "bits_per_byte": 1.6161242378281793,
      "learning_rate": 0.0001378423330240207,
      "loss_nats": 1.1202119588851929,
      "step": 4690,
      "unclipped_grad_norm": 0.8490695953369141
    },
    {
      "bits_per_byte": 1.7156000349831328,
      "learning_rate": 0.00013729592880358936,
      "loss_nats": 1.189163327217102,
      "step": 4700,
      "unclipped_grad_norm": 0.8805282115936279
    },
    {
      "bits_per_byte": 1.6530434095235966,
      "learning_rate": 0.00013674999796929659,
      "loss_nats": 1.14580237865448,
      "step": 4710,
      "unclipped_grad_norm": 0.9434417486190796
    },
    {
      "bits_per_byte": 1.6037863744379321,
      "learning_rate": 0.00013620454984959417,
      "loss_nats": 1.1116600036621094,
      "step": 4720,
      "unclipped_grad_norm": 0.8885965347290039
    },
    {
      "bits_per_byte": 1.7271340514632472,
      "learning_rate": 0.00013565959376468542,
      "loss_nats": 1.1971580982208252,
      "step": 4730,
      "unclipped_grad_norm": 0.9324085116386414
    },
    {
      "bits_per_byte": 1.6733065774309035,
      "learning_rate": 0.00013511513902636634,
      "loss_nats": 1.1598477363586426,
      "step": 4740,
      "unclipped_grad_norm": 0.8685431480407715
    },
    {
      "bits_per_byte": 1.7120205600707252,
      "learning_rate": 0.0001345711949378662,
      "loss_nats": 1.1866822242736816,
      "step": 4750,
      "unclipped_grad_norm": 0.9381123185157776
    },
    {
      "bits_per_byte": 1.5687653752600876,
      "learning_rate": 0.00013402777079368853,
      "loss_nats": 1.0873852968215942,
      "step": 4760,
      "unclipped_grad_norm": 0.8193861246109009
    },
    {
      "bits_per_byte": 1.681102207029213,
      "learning_rate": 0.0001334848758794528,
      "loss_nats": 1.1652512550354004,
      "step": 4770,
      "unclipped_grad_norm": 0.8750576376914978
    },
    {
      "bits_per_byte": 1.6366992102815003,
      "learning_rate": 0.00013294251947173504,
      "loss_nats": 1.134473443031311,
      "step": 4780,
      "unclipped_grad_norm": 0.8815762996673584
    },
    {
      "bits_per_byte": 1.6734204299457747,
      "learning_rate": 0.00013240071083790996,
      "loss_nats": 1.1599266529083252,
      "step": 4790,
      "unclipped_grad_norm": 0.9372621774673462
    },
    {
      "bits_per_byte": 1.6264474964462192,
      "learning_rate": 0.0001318594592359921,
      "loss_nats": 1.1273674964904785,
      "step": 4800,
      "unclipped_grad_norm": 0.8206531405448914
    },
    {
      "bits_per_byte": 1.6050609378634764,
      "learning_rate": 0.0001313187739144781,
      "loss_nats": 1.1125434637069702,
      "step": 4810,
      "unclipped_grad_norm": 0.867507815361023
    },
    {
      "bits_per_byte": 1.6801829597603513,
      "learning_rate": 0.0001307786641121882,
      "loss_nats": 1.1646140813827515,
      "step": 4820,
      "unclipped_grad_norm": 0.9113548398017883
    },
    {
      "bits_per_byte": 1.6755392562044045,
      "learning_rate": 0.00013023913905810876,
      "loss_nats": 1.1613953113555908,
      "step": 4830,
      "unclipped_grad_norm": 0.9283871650695801
    },
    {
      "bits_per_byte": 1.6755334087942753,
      "learning_rate": 0.00012970020797123432,
      "loss_nats": 1.161391258239746,
      "step": 4840,
      "unclipped_grad_norm": 0.8912690281867981
    },
    {
      "bits_per_byte": 1.6043515094286673,
      "learning_rate": 0.00012916188006041027,
      "loss_nats": 1.1120517253875732,
      "step": 4850,
      "unclipped_grad_norm": 0.8777358531951904
    },
    {
      "bits_per_byte": 1.6436579723007114,
      "learning_rate": 0.00012862416452417516,
      "loss_nats": 1.1392968893051147,
      "step": 4860,
      "unclipped_grad_norm": 0.9008035063743591
    },
    {
      "bits_per_byte": 1.639443709423969,
      "learning_rate": 0.00012808707055060405,
      "loss_nats": 1.1363757848739624,
      "step": 4870,
      "unclipped_grad_norm": 0.8822212219238281
    },
    {
      "bits_per_byte": 1.8005314313372698,
      "learning_rate": 0.000127550607317151,
      "loss_nats": 1.2480332851409912,
      "step": 4880,
      "unclipped_grad_norm": 0.953372061252594
    },
    {
      "bits_per_byte": 1.587463501044543,
      "learning_rate": 0.00012701478399049248,
      "loss_nats": 1.1003458499908447,
      "step": 4890,
      "unclipped_grad_norm": 0.898897111415863
    },
    {
      "bits_per_byte": 1.6710314189426394,
      "learning_rate": 0.0001264796097263708,
      "loss_nats": 1.1582707166671753,
      "step": 4900,
      "unclipped_grad_norm": 0.8573687672615051
    },
    {
      "bits_per_byte": 1.6542582949692914,
      "learning_rate": 0.0001259450936694375,
      "loss_nats": 1.1466444730758667,
      "step": 4910,
      "unclipped_grad_norm": 0.8744381070137024
    },
    {
      "bits_per_byte": 1.5936046575415521,
      "learning_rate": 0.00012541124495309727,
      "loss_nats": 1.104602575302124,
      "step": 4920,
      "unclipped_grad_norm": 0.8256805539131165
    },
    {
      "bits_per_byte": 1.5994107918346805,
      "learning_rate": 0.00012487807269935167,
      "loss_nats": 1.1086270809173584,
      "step": 4930,
      "unclipped_grad_norm": 0.8097874522209167
    },
    {
      "bits_per_byte": 1.6321965324992613,
      "learning_rate": 0.00012434558601864349,
      "loss_nats": 1.131352424621582,
      "step": 4940,
      "unclipped_grad_norm": 0.9138422012329102
    },
    {
      "bits_per_byte": 1.6452641182771193,
      "learning_rate": 0.0001238137940097009,
      "loss_nats": 1.1404101848602295,
      "step": 4950,
      "unclipped_grad_norm": 0.890544593334198
    },
    {
      "bits_per_byte": 1.6569370967391306,
      "learning_rate": 0.0001232827057593821,
      "loss_nats": 1.1485012769699097,
      "step": 4960,
      "unclipped_grad_norm": 0.8706549406051636
    },
    {
      "bits_per_byte": 1.6359316517106997,
      "learning_rate": 0.00012275233034251992,
      "loss_nats": 1.133941411972046,
      "step": 4970,
      "unclipped_grad_norm": 0.9410244226455688
    },
    {
      "bits_per_byte": 1.6744512939550464,
      "learning_rate": 0.0001222226768217669,
      "loss_nats": 1.1606411933898926,
      "step": 4980,
      "unclipped_grad_norm": 0.9115959405899048
    },
    {
      "bits_per_byte": 1.637535905877948,
      "learning_rate": 0.00012169375424744034,
      "loss_nats": 1.1350533962249756,
      "step": 4990,
      "unclipped_grad_norm": 0.8625267744064331
    },
    {
      "bits_per_byte": 1.6562579372508734,
      "learning_rate": 0.00012116557165736772,
      "loss_nats": 1.1480305194854736,
      "step": 5000,
      "unclipped_grad_norm": 0.9143829345703125
    },
    {
      "bits_per_byte": 1.691347557506412,
      "learning_rate": 0.00012063813807673214,
      "loss_nats": 1.1723527908325195,
      "step": 5010,
      "unclipped_grad_norm": 0.8908807635307312
    },
    {
      "bits_per_byte": 1.640470617832271,
      "learning_rate": 0.00012011146251791832,
      "loss_nats": 1.1370875835418701,
      "step": 5020,
      "unclipped_grad_norm": 0.9010583162307739
    },
    {
      "bits_per_byte": 1.5978712031441566,
      "learning_rate": 0.0001195855539803584,
      "loss_nats": 1.1075599193572998,
      "step": 5030,
      "unclipped_grad_norm": 0.8482128381729126
    },
    {
      "bits_per_byte": 1.6379517599277342,
      "learning_rate": 0.00011906042145037817,
      "loss_nats": 1.1353416442871094,
      "step": 5040,
      "unclipped_grad_norm": 0.8523857593536377
    },
    {
      "bits_per_byte": 1.624761722502462,
      "learning_rate": 0.0001185360739010438,
      "loss_nats": 1.1261990070343018,
      "step": 5050,
      "unclipped_grad_norm": 0.8824069499969482
    },
    {
      "bits_per_byte": 1.6994286783051538,
      "learning_rate": 0.00011801252029200808,
      "loss_nats": 1.1779541969299316,
      "step": 5060,
      "unclipped_grad_norm": 0.9452667832374573
    },
    {
      "bits_per_byte": 1.6270504676201443,
      "learning_rate": 0.00011748976956935777,
      "loss_nats": 1.1277854442596436,
      "step": 5070,
      "unclipped_grad_norm": 0.8711076378822327
    },
    {
      "bits_per_byte": 1.7018825267676645,
      "learning_rate": 0.00011696783066546024,
      "loss_nats": 1.1796550750732422,
      "step": 5080,
      "unclipped_grad_norm": 0.9225792288780212
    },
    {
      "bits_per_byte": 1.6371355302667394,
      "learning_rate": 0.00011644671249881147,
      "loss_nats": 1.1347758769989014,
      "step": 5090,
      "unclipped_grad_norm": 0.9880733489990234
    },
    {
      "bits_per_byte": 1.6193408293472662,
      "learning_rate": 0.00011592642397388293,
      "loss_nats": 1.1224415302276611,
      "step": 5100,
      "unclipped_grad_norm": 0.8858473896980286
    },
    {
      "bits_per_byte": 1.648101144085752,
      "learning_rate": 0.00011540697398097015,
      "loss_nats": 1.1423766613006592,
      "step": 5110,
      "unclipped_grad_norm": 0.8950205445289612
    },
    {
      "bits_per_byte": 1.5621068949492831,
      "learning_rate": 0.00011488837139604016,
      "loss_nats": 1.0827699899673462,
      "step": 5120,
      "unclipped_grad_norm": 0.8509450554847717
    },
    {
      "bits_per_byte": 1.6969497203756174,
      "learning_rate": 0.00011437062508058039,
      "loss_nats": 1.1762359142303467,
      "step": 5130,
      "unclipped_grad_norm": 0.9032918810844421
    },
    {
      "bits_per_byte": 1.6594853636769649,
      "learning_rate": 0.00011385374388144673,
      "loss_nats": 1.1502676010131836,
      "step": 5140,
      "unclipped_grad_norm": 0.9204651713371277
    },
    {
      "bits_per_byte": 1.6102843949354817,
      "learning_rate": 0.00011333773663071286,
      "loss_nats": 1.1161640882492065,
      "step": 5150,
      "unclipped_grad_norm": 0.8756265640258789
    },
    {
      "bits_per_byte": 1.6252881613967531,
      "learning_rate": 0.00011282261214551897,
      "loss_nats": 1.1265639066696167,
      "step": 5160,
      "unclipped_grad_norm": 0.9101666212081909
    },
    {
      "bits_per_byte": 1.6855433150224433,
      "learning_rate": 0.00011230837922792112,
      "loss_nats": 1.1683295965194702,
      "step": 5170,
      "unclipped_grad_norm": 0.9160894155502319
    },
    {
      "bits_per_byte": 1.6357713638800955,
      "learning_rate": 0.0001117950466647412,
      "loss_nats": 1.1338303089141846,
      "step": 5180,
      "unclipped_grad_norm": 0.9035648107528687
    },
    {
      "bits_per_byte": 1.5386909410863592,
      "learning_rate": 0.00011128262322741627,
      "loss_nats": 1.0665392875671387,
      "step": 5190,
      "unclipped_grad_norm": 0.8579265475273132
    },
    {
      "bits_per_byte": 1.6228207263048244,
      "learning_rate": 0.00011077111767184913,
      "loss_nats": 1.1248536109924316,
      "step": 5200,
      "unclipped_grad_norm": 0.8737999200820923
    },
    {
      "bits_per_byte": 1.5871657990758994,
      "learning_rate": 0.00011026053873825837,
      "loss_nats": 1.1001394987106323,
      "step": 5210,
      "unclipped_grad_norm": 0.8650596737861633
    },
    {
      "bits_per_byte": 1.6000187505054806,
      "learning_rate": 0.0001097508951510293,
      "loss_nats": 1.1090484857559204,
      "step": 5220,
      "unclipped_grad_norm": 0.857958197593689
    },
    {
      "bits_per_byte": 1.6870952864638293,
      "learning_rate": 0.0001092421956185646,
      "loss_nats": 1.1694053411483765,
      "step": 5230,
      "unclipped_grad_norm": 0.9485795497894287
    },
    {
      "bits_per_byte": 1.65525063486477,
      "learning_rate": 0.00010873444883313581,
      "loss_nats": 1.1473323106765747,
      "step": 5240,
      "unclipped_grad_norm": 0.9584298133850098
    },
    {
      "bits_per_byte": 1.648363245645667,
      "learning_rate": 0.00010822766347073447,
      "loss_nats": 1.1425583362579346,
      "step": 5250,
      "unclipped_grad_norm": 0.9243587851524353
    },
    {
      "bits_per_byte": 1.5949392429122475,
      "learning_rate": 0.00010772184819092424,
      "loss_nats": 1.105527639389038,
      "step": 5260,
      "unclipped_grad_norm": 0.97037672996521
    },
    {
      "bits_per_byte": 1.6406128474845345,
      "learning_rate": 0.0001072170116366926,
      "loss_nats": 1.1371861696243286,
      "step": 5270,
      "unclipped_grad_norm": 0.9107886552810669
    },
    {
      "bits_per_byte": 1.5702697075071845,
      "learning_rate": 0.00010671316243430345,
      "loss_nats": 1.088428020477295,
      "step": 5280,
      "unclipped_grad_norm": 0.9339945316314697
    },
    {
      "bits_per_byte": 1.5822967982473224,
      "learning_rate": 0.00010621030919314944,
      "loss_nats": 1.0967645645141602,
      "step": 5290,
      "unclipped_grad_norm": 0.9108784198760986
    },
    {
      "bits_per_byte": 1.5641275191142703,
      "learning_rate": 0.000105708460505605,
      "loss_nats": 1.0841705799102783,
      "step": 5300,
      "unclipped_grad_norm": 0.9299371838569641
    },
    {
      "bits_per_byte": 1.568704665384333,
      "learning_rate": 0.00010520762494687964,
      "loss_nats": 1.0873432159423828,
      "step": 5310,
      "unclipped_grad_norm": 0.841905951499939
    },
    {
      "bits_per_byte": 1.5839983945949592,
      "learning_rate": 0.00010470781107487111,
      "loss_nats": 1.0979440212249756,
      "step": 5320,
      "unclipped_grad_norm": 0.9190206527709961
    },
    {
      "bits_per_byte": 1.6212247273048175,
      "learning_rate": 0.00010420902743001955,
      "loss_nats": 1.1237473487854004,
      "step": 5330,
      "unclipped_grad_norm": 0.9011974930763245
    },
    {
      "bits_per_byte": 1.6958863516453326,
      "learning_rate": 0.00010371128253516117,
      "loss_nats": 1.1754988431930542,
      "step": 5340,
      "unclipped_grad_norm": 0.9635267853736877
    },
    {
      "bits_per_byte": 1.5503265993131332,
      "learning_rate": 0.000103214584895383,
      "loss_nats": 1.0746045112609863,
      "step": 5350,
      "unclipped_grad_norm": 0.8525294661521912
    },
    {
      "bits_per_byte": 1.6023642498979476,
      "learning_rate": 0.00010271894299787715,
      "loss_nats": 1.110674262046814,
      "step": 5360,
      "unclipped_grad_norm": 0.9040093421936035
    },
    {
      "bits_per_byte": 1.6756675552619482,
      "learning_rate": 0.00010222436531179622,
      "loss_nats": 1.1614842414855957,
      "step": 5370,
      "unclipped_grad_norm": 0.9546249508857727
    },
    {
      "bits_per_byte": 1.5762295942901843,
      "learning_rate": 0.0001017308602881082,
      "loss_nats": 1.0925590991973877,
      "step": 5380,
      "unclipped_grad_norm": 0.9253390431404114
    },
    {
      "bits_per_byte": 1.5803738602280255,
      "learning_rate": 0.00010123843635945242,
      "loss_nats": 1.0954316854476929,
      "step": 5390,
      "unclipped_grad_norm": 0.9185824394226074
    },
    {
      "bits_per_byte": 1.6346944085203927,
      "learning_rate": 0.00010074710193999505,
      "loss_nats": 1.1330838203430176,
      "step": 5400,
      "unclipped_grad_norm": 0.9316674470901489
    },
    {
      "bits_per_byte": 1.5477485793766999,
      "learning_rate": 0.0001002568654252858,
      "loss_nats": 1.0728175640106201,
      "step": 5410,
      "unclipped_grad_norm": 0.9008721709251404
    },
    {
      "bits_per_byte": 1.6909000586488672,
      "learning_rate": 9.976773519211404e-05,
      "loss_nats": 1.1720426082611084,
      "step": 5420,
      "unclipped_grad_norm": 0.9377346038818359
    },
    {
      "bits_per_byte": 1.6161049757712826,
      "learning_rate": 9.92797195983659e-05,
      "loss_nats": 1.1201986074447632,
      "step": 5430,
      "unclipped_grad_norm": 0.9362180233001709
    },
    {
      "bits_per_byte": 1.6092089874196365,
      "learning_rate": 9.879282698288142e-05,
      "loss_nats": 1.1154186725616455,
      "step": 5440,
      "unclipped_grad_norm": 0.8880205154418945
    },
    {
      "bits_per_byte": 1.6243469003485809,
      "learning_rate": 9.830706566531197e-05,
      "loss_nats": 1.1259114742279053,
      "step": 5450,
      "unclipped_grad_norm": 0.9006999731063843
    },
    {
      "bits_per_byte": 1.6430875058477994,
      "learning_rate": 9.782244394597826e-05,
      "loss_nats": 1.1389014720916748,
      "step": 5460,
      "unclipped_grad_norm": 0.9130494594573975
    },
    {
      "bits_per_byte": 1.6231913489174337,
      "learning_rate": 9.733897010572829e-05,
      "loss_nats": 1.1251105070114136,
      "step": 5470,
      "unclipped_grad_norm": 0.9382801651954651
    },
    {
      "bits_per_byte": 1.5843513029945295,
      "learning_rate": 9.685665240579612e-05,
      "loss_nats": 1.0981886386871338,
      "step": 5480,
      "unclipped_grad_norm": 0.9530964493751526
    },
    {
      "bits_per_byte": 1.6238423032509495,
      "learning_rate": 9.637549908766041e-05,
      "loss_nats": 1.1255617141723633,
      "step": 5490,
      "unclipped_grad_norm": 1.0083582401275635
    },
    {
      "bits_per_byte": 1.6558995254064752,
      "learning_rate": 9.589551837290389e-05,
      "loss_nats": 1.1477820873260498,
      "step": 5500,
      "unclipped_grad_norm": 0.9359451532363892
    },
    {
      "bits_per_byte": 1.6193308543535163,
      "learning_rate": 9.541671846307257e-05,
      "loss_nats": 1.1224346160888672,
      "step": 5510,
      "unclipped_grad_norm": 0.9424907565116882
    },
    {
      "bits_per_byte": 1.622570491547819,
      "learning_rate": 9.493910753953597e-05,
      "loss_nats": 1.1246801614761353,
      "step": 5520,
      "unclipped_grad_norm": 0.9104092121124268
    },
    {
      "bits_per_byte": 1.6496316176957802,
      "learning_rate": 9.446269376334689e-05,
      "loss_nats": 1.1434375047683716,
      "step": 5530,
      "unclipped_grad_norm": 0.9198633432388306
    },
    {
      "bits_per_byte": 1.6739231352342467,
      "learning_rate": 9.39874852751024e-05,
      "loss_nats": 1.1602751016616821,
      "step": 5540,
      "unclipped_grad_norm": 1.0088523626327515
    },
    {
      "bits_per_byte": 1.5961751102413477,
      "learning_rate": 9.351349019480433e-05,
      "loss_nats": 1.10638427734375,
      "step": 5550,
      "unclipped_grad_norm": 0.8725464344024658
    },
    {
      "bits_per_byte": 1.5912085952497315,
      "learning_rate": 9.304071662172086e-05,
      "loss_nats": 1.1029417514801025,
      "step": 5560,
      "unclipped_grad_norm": 0.9294795989990234
    },
    {
      "bits_per_byte": 1.6449143055652642,
      "learning_rate": 9.256917263424787e-05,
      "loss_nats": 1.1401677131652832,
      "step": 5570,
      "unclipped_grad_norm": 0.9388754963874817
    },
    {
      "bits_per_byte": 1.5525186901810306,
      "learning_rate": 9.209886628977114e-05,
      "loss_nats": 1.0761239528656006,
      "step": 5580,
      "unclipped_grad_norm": 0.9197003245353699
    },
    {
      "bits_per_byte": 1.6195045568308877,
      "learning_rate": 9.162980562452843e-05,
      "loss_nats": 1.1225550174713135,
      "step": 5590,
      "unclipped_grad_norm": 0.9434337615966797
    },
    {
      "bits_per_byte": 1.5951824263805676,
      "learning_rate": 9.116199865347238e-05,
      "loss_nats": 1.105696201324463,
      "step": 5600,
      "unclipped_grad_norm": 0.9404297471046448
    },
    {
      "bits_per_byte": 1.6108896018838679,
      "learning_rate": 9.069545337013333e-05,
      "loss_nats": 1.1165835857391357,
      "step": 5610,
      "unclipped_grad_norm": 0.9016186594963074
    },
    {
      "bits_per_byte": 1.5417312503883118,
      "learning_rate": 9.0230177746483e-05,
      "loss_nats": 1.0686466693878174,
      "step": 5620,
      "unclipped_grad_norm": 0.9296513795852661
    },
    {
      "bits_per_byte": 1.5687026015925227,
      "learning_rate": 8.976617973279797e-05,
      "loss_nats": 1.0873417854309082,
      "step": 5630,
      "unclipped_grad_norm": 0.9855212569236755
    },
    {
      "bits_per_byte": 1.548401425519375,
      "learning_rate": 8.930346725752419e-05,
      "loss_nats": 1.0732700824737549,
      "step": 5640,
      "unclipped_grad_norm": 0.9301762580871582
    },
    {
      "bits_per_byte": 1.6637563808284919,
      "learning_rate": 8.88420482271411e-05,
      "loss_nats": 1.1532280445098877,
      "step": 5650,
      "unclipped_grad_norm": 1.0132923126220703
    },
    {
      "bits_per_byte": 1.614469592744228,
      "learning_rate": 8.838193052602691e-05,
      "loss_nats": 1.1190650463104248,
      "step": 5660,
      "unclipped_grad_norm": 0.9275335073471069
    },
    {
      "bits_per_byte": 1.6100616774026144,
      "learning_rate": 8.792312201632352e-05,
      "loss_nats": 1.1160097122192383,
      "step": 5670,
      "unclipped_grad_norm": 0.9310563802719116
    },
    {
      "bits_per_byte": 1.6255531866617328,
      "learning_rate": 8.746563053780261e-05,
      "loss_nats": 1.1267476081848145,
      "step": 5680,
      "unclipped_grad_norm": 0.9579228758811951
    },
    {
      "bits_per_byte": 1.6417694308115869,
      "learning_rate": 8.700946390773114e-05,
      "loss_nats": 1.1379878520965576,
      "step": 5690,
      "unclipped_grad_norm": 0.9637964367866516
    },
    {
      "bits_per_byte": 1.6007787418396433,
      "learning_rate": 8.655462992073833e-05,
      "loss_nats": 1.1095752716064453,
      "step": 5700,
      "unclipped_grad_norm": 0.955059826374054
    },
    {
      "bits_per_byte": 1.6229964925740061,
      "learning_rate": 8.610113634868199e-05,
      "loss_nats": 1.1249754428863525,
      "step": 5710,
      "unclipped_grad_norm": 0.9087291955947876
    },
    {
      "bits_per_byte": 1.6524653758340466,
      "learning_rate": 8.564899094051613e-05,
      "loss_nats": 1.1454017162322998,
      "step": 5720,
      "unclipped_grad_norm": 0.9723272919654846
    },
    {
      "bits_per_byte": 1.5500029279642094,
      "learning_rate": 8.519820142215816e-05,
      "loss_nats": 1.0743801593780518,
      "step": 5730,
      "unclipped_grad_norm": 0.9230121374130249
    },
    {
      "bits_per_byte": 1.6318054439511993,
      "learning_rate": 8.474877549635728e-05,
      "loss_nats": 1.1310813426971436,
      "step": 5740,
      "unclipped_grad_norm": 0.9736878275871277
    },
    {
      "bits_per_byte": 1.5815806624891298,
      "learning_rate": 8.430072084256247e-05,
      "loss_nats": 1.0962681770324707,
      "step": 5750,
      "unclipped_grad_norm": 0.9681398868560791
    },
    {
      "bits_per_byte": 1.5890875332166405,
      "learning_rate": 8.385404511679161e-05,
      "loss_nats": 1.1014715433120728,
      "step": 5760,
      "unclipped_grad_norm": 0.9480424523353577
    },
    {
      "bits_per_byte": 1.528980628635994,
      "learning_rate": 8.340875595150034e-05,
      "loss_nats": 1.059808611869812,
      "step": 5770,
      "unclipped_grad_norm": 0.9915046691894531
    },
    {
      "bits_per_byte": 1.610684942529341,
      "learning_rate": 8.296486095545203e-05,
      "loss_nats": 1.1164417266845703,
      "step": 5780,
      "unclipped_grad_norm": 0.9259656667709351
    },
    {
      "bits_per_byte": 1.5796823179889061,
      "learning_rate": 8.252236771358729e-05,
      "loss_nats": 1.0949523448944092,
      "step": 5790,
      "unclipped_grad_norm": 0.9487527012825012
    },
    {
      "bits_per_byte": 1.5821124328455975,
      "learning_rate": 8.208128378689484e-05,
      "loss_nats": 1.0966367721557617,
      "step": 5800,
      "unclipped_grad_norm": 0.9434526562690735
    },
    {
      "bits_per_byte": 1.6104013431380684,
      "learning_rate": 8.16416167122819e-05,
      "loss_nats": 1.116245150566101,
      "step": 5810,
      "unclipped_grad_norm": 0.9532226920127869
    },
    {
      "bits_per_byte": 1.6033849669308184,
      "learning_rate": 8.120337400244578e-05,
      "loss_nats": 1.1113817691802979,
      "step": 5820,
      "unclipped_grad_norm": 0.914564311504364
    },
    {
      "bits_per_byte": 1.5785037208825432,
      "learning_rate": 8.076656314574521e-05,
      "loss_nats": 1.0941354036331177,
      "step": 5830,
      "unclipped_grad_norm": 0.9744407534599304
    },
    {
      "bits_per_byte": 1.5595204478629572,
      "learning_rate": 8.03311916060725e-05,
      "loss_nats": 1.080977201461792,
      "step": 5840,
      "unclipped_grad_norm": 0.9682079553604126
    },
    {
      "bits_per_byte": 1.5459948722858516,
      "learning_rate": 7.989726682272606e-05,
      "loss_nats": 1.0716019868850708,
      "step": 5850,
      "unclipped_grad_norm": 0.908660352230072
    },
    {
      "bits_per_byte": 1.5771918372217617,
      "learning_rate": 7.946479621028318e-05,
      "loss_nats": 1.0932260751724243,
      "step": 5860,
      "unclipped_grad_norm": 0.9774399995803833
    },
    {
      "bits_per_byte": 1.617923176356204,
      "learning_rate": 7.903378715847343e-05,
      "loss_nats": 1.121458888053894,
      "step": 5870,
      "unclipped_grad_norm": 0.9498292803764343
    },
    {
      "bits_per_byte": 1.6179303996275405,
      "learning_rate": 7.860424703205224e-05,
      "loss_nats": 1.1214638948440552,
      "step": 5880,
      "unclipped_grad_norm": 0.929435670375824
    },
    {
      "bits_per_byte": 1.6226986186227117,
      "learning_rate": 7.817618317067534e-05,
      "loss_nats": 1.1247689723968506,
      "step": 5890,
      "unclipped_grad_norm": 0.9540143013000488
    },
    {
      "bits_per_byte": 1.5934973403674138,
      "learning_rate": 7.774960288877295e-05,
      "loss_nats": 1.1045281887054443,
      "step": 5900,
      "unclipped_grad_norm": 1.0085638761520386
    },
    {
      "bits_per_byte": 1.5806963276983932,
      "learning_rate": 7.732451347542522e-05,
      "loss_nats": 1.0956552028656006,
      "step": 5910,
      "unclipped_grad_norm": 1.0136828422546387
    },
    {
      "bits_per_byte": 1.602695832448811,
      "learning_rate": 7.690092219423722e-05,
      "loss_nats": 1.1109040975570679,
      "step": 5920,
      "unclipped_grad_norm": 1.0095367431640625
    },
    {
      "bits_per_byte": 1.6274630539995643,
      "learning_rate": 7.647883628321536e-05,
      "loss_nats": 1.1280714273452759,
      "step": 5930,
      "unclipped_grad_norm": 0.947801411151886
    },
    {
      "bits_per_byte": 1.6238797954688375,
      "learning_rate": 7.605826295464316e-05,
      "loss_nats": 1.1255877017974854,
      "step": 5940,
      "unclipped_grad_norm": 0.9719098806381226
    },
    {
      "bits_per_byte": 1.597647625698035,
      "learning_rate": 7.563920939495851e-05,
      "loss_nats": 1.1074049472808838,
      "step": 5950,
      "unclipped_grad_norm": 0.9620362520217896
    },
    {
      "bits_per_byte": 1.588192019553598,
      "learning_rate": 7.522168276463047e-05,
      "loss_nats": 1.1008508205413818,
      "step": 5960,
      "unclipped_grad_norm": 0.9313361644744873
    },
    {
      "bits_per_byte": 1.619017845928946,
      "learning_rate": 7.480569019803719e-05,
      "loss_nats": 1.1222176551818848,
      "step": 5970,
      "unclipped_grad_norm": 1.0069472789764404
    },
    {
      "bits_per_byte": 1.5640157303912094,
      "learning_rate": 7.439123880334388e-05,
      "loss_nats": 1.0840930938720703,
      "step": 5980,
      "unclipped_grad_norm": 0.9776687026023865
    },
    {
      "bits_per_byte": 1.5433833157324999,
      "learning_rate": 7.39783356623813e-05,
      "loss_nats": 1.0697917938232422,
      "step": 5990,
      "unclipped_grad_norm": 0.9335829615592957
    },
    {
      "bits_per_byte": 1.6083124418606887,
      "learning_rate": 7.356698783052497e-05,
      "loss_nats": 1.1147972345352173,
      "step": 6000,
      "unclipped_grad_norm": 0.9547005295753479
    },
    {
      "bits_per_byte": 1.591078920330981,
      "learning_rate": 7.315720233657431e-05,
      "loss_nats": 1.1028518676757812,
      "step": 6010,
      "unclipped_grad_norm": 0.9792258739471436
    },
    {
      "bits_per_byte": 1.553183919074568,
      "learning_rate": 7.274898618263286e-05,
      "loss_nats": 1.076585054397583,
      "step": 6020,
      "unclipped_grad_norm": 0.9365350604057312
    },
    {
      "bits_per_byte": 1.621642989111716,
      "learning_rate": 7.234234634398831e-05,
      "loss_nats": 1.124037265777588,
      "step": 6030,
      "unclipped_grad_norm": 0.9456943869590759
    },
    {
      "bits_per_byte": 1.55026812521184,
      "learning_rate": 7.193728976899362e-05,
      "loss_nats": 1.074563980102539,
      "step": 6040,
      "unclipped_grad_norm": 0.986164391040802
    },
    {
      "bits_per_byte": 1.573968538379291,
      "learning_rate": 7.153382337894797e-05,
      "loss_nats": 1.0909918546676636,
      "step": 6050,
      "unclipped_grad_norm": 0.9421037435531616
    },
    {
      "bits_per_byte": 1.6258569080231564,
      "learning_rate": 7.113195406797887e-05,
      "loss_nats": 1.1269581317901611,
      "step": 6060,
      "unclipped_grad_norm": 0.9926829934120178
    },
    {
      "bits_per_byte": 1.5943486544891847,
      "learning_rate": 7.073168870292397e-05,
      "loss_nats": 1.1051182746887207,
      "step": 6070,
      "unclipped_grad_norm": 0.9500434994697571
    },
    {
      "bits_per_byte": 1.5776276412590482,
      "learning_rate": 7.033303412321404e-05,
      "loss_nats": 1.093528151512146,
      "step": 6080,
      "unclipped_grad_norm": 0.9243369102478027
    },
    {
      "bits_per_byte": 1.583595095278686,
      "learning_rate": 6.993599714075586e-05,
      "loss_nats": 1.097664475440979,
      "step": 6090,
      "unclipped_grad_norm": 0.9602344036102295
    },
    {
      "bits_per_byte": 1.5816112754009832,
      "learning_rate": 6.954058453981609e-05,
      "loss_nats": 1.0962893962860107,
      "step": 6100,
      "unclipped_grad_norm": 0.9188515543937683
    },
    {
      "bits_per_byte": 1.5481370881849987,
      "learning_rate": 6.914680307690504e-05,
      "loss_nats": 1.0730868577957153,
      "step": 6110,
      "unclipped_grad_norm": 0.9290796518325806
    },
    {
      "bits_per_byte": 1.557837081676312,
      "learning_rate": 6.875465948066139e-05,
      "loss_nats": 1.079810380935669,
      "step": 6120,
      "unclipped_grad_norm": 0.9658544063568115
    },
    {
      "bits_per_byte": 1.5877624068917426,
      "learning_rate": 6.836416045173733e-05,
      "loss_nats": 1.100553035736084,
      "step": 6130,
      "unclipped_grad_norm": 0.9524391889572144
    },
    {
      "bits_per_byte": 1.6144964220377627,
      "learning_rate": 6.797531266268377e-05,
      "loss_nats": 1.1190836429595947,
      "step": 6140,
      "unclipped_grad_norm": 0.9724568724632263
    },
    {
      "bits_per_byte": 1.5513643426284394,
      "learning_rate": 6.758812275783663e-05,
      "loss_nats": 1.0753238201141357,
      "step": 6150,
      "unclipped_grad_norm": 0.9667541980743408
    },
    {
      "bits_per_byte": 1.6439771721007128,
      "learning_rate": 6.720259735320298e-05,
      "loss_nats": 1.139518141746521,
      "step": 6160,
      "unclipped_grad_norm": 1.0203982591629028
    },
    {
      "bits_per_byte": 1.5065515112237209,
      "learning_rate": 6.681874303634842e-05,
      "loss_nats": 1.0442619323730469,
      "step": 6170,
      "unclipped_grad_norm": 0.892124593257904
    },
    {
      "bits_per_byte": 1.5721656442502963,
      "learning_rate": 6.643656636628402e-05,
      "loss_nats": 1.0897421836853027,
      "step": 6180,
      "unclipped_grad_norm": 0.9671148657798767
    },
    {
      "bits_per_byte": 1.5890949284706275,
      "learning_rate": 6.605607387335466e-05,
      "loss_nats": 1.1014766693115234,
      "step": 6190,
      "unclipped_grad_norm": 1.0115299224853516
    },
    {
      "bits_per_byte": 1.5390813417038176,
      "learning_rate": 6.567727205912724e-05,
      "loss_nats": 1.066809892654419,
      "step": 6200,
      "unclipped_grad_norm": 0.9446512460708618
    },
    {
      "bits_per_byte": 1.5691769297436022,
      "learning_rate": 6.530016739627959e-05,
      "loss_nats": 1.0876705646514893,
      "step": 6210,
      "unclipped_grad_norm": 0.9595397710800171
    },
    {
      "bits_per_byte": 1.5550289489530242,
      "learning_rate": 6.492476632848993e-05,
      "loss_nats": 1.0778639316558838,
      "step": 6220,
      "unclipped_grad_norm": 0.927939772605896
    },
    {
      "bits_per_byte": 1.5332487220824564,
      "learning_rate": 6.455107527032678e-05,
      "loss_nats": 1.0627670288085938,
      "step": 6230,
      "unclipped_grad_norm": 0.9231003522872925
    },
    {
      "bits_per_byte": 1.5454189023881122,
      "learning_rate": 6.417910060713926e-05,
      "loss_nats": 1.0712027549743652,
      "step": 6240,
      "unclipped_grad_norm": 0.9556372761726379
    },
    {
      "bits_per_byte": 1.547023500520662,
      "learning_rate": 6.380884869494799e-05,
      "loss_nats": 1.072314977645874,
      "step": 6250,
      "unclipped_grad_norm": 0.9621478915214539
    },
    {
      "bits_per_byte": 1.6040983509665974,
      "learning_rate": 6.34403258603367e-05,
      "loss_nats": 1.1118762493133545,
      "step": 6260,
      "unclipped_grad_norm": 0.9583669304847717
    },
    {
      "bits_per_byte": 1.562768856172454,
      "learning_rate": 6.30735384003437e-05,
      "loss_nats": 1.0832288265228271,
      "step": 6270,
      "unclipped_grad_norm": 0.9386000633239746
    },
    {
      "bits_per_byte": 1.6650833989625493,
      "learning_rate": 6.270849258235484e-05,
      "loss_nats": 1.1541478633880615,
      "step": 6280,
      "unclipped_grad_norm": 0.9854296445846558
    },
    {
      "bits_per_byte": 1.5598107545776136,
      "learning_rate": 6.234519464399581e-05,
      "loss_nats": 1.0811784267425537,
      "step": 6290,
      "unclipped_grad_norm": 0.9466595649719238
    },
    {
      "bits_per_byte": 1.5877661905100615,
      "learning_rate": 6.198365079302614e-05,
      "loss_nats": 1.100555658340454,
      "step": 6300,
      "unclipped_grad_norm": 1.077492117881775
    },
    {
      "bits_per_byte": 1.6323781461785725,
      "learning_rate": 6.16238672072326e-05,
      "loss_nats": 1.1314783096313477,
      "step": 6310,
      "unclipped_grad_norm": 1.0482228994369507
    },
    {
      "bits_per_byte": 1.5889704130314029,
      "learning_rate": 6.126585003432406e-05,
      "loss_nats": 1.1013903617858887,
      "step": 6320,
      "unclipped_grad_norm": 0.9478169083595276
    },
    {
      "bits_per_byte": 1.5225548408518068,
      "learning_rate": 6.090960539182614e-05,
      "loss_nats": 1.0553545951843262,
      "step": 6330,
      "unclipped_grad_norm": 0.9578908085823059
    },
    {
      "bits_per_byte": 1.580892387920377,
      "learning_rate": 6.0555139366976935e-05,
      "loss_nats": 1.0957911014556885,
      "step": 6340,
      "unclipped_grad_norm": 1.0281808376312256
    },
    {
      "bits_per_byte": 1.5773070655978398,
      "learning_rate": 6.020245801662274e-05,
      "loss_nats": 1.0933059453964233,
      "step": 6350,
      "unclipped_grad_norm": 0.9984999299049377
    },
    {
      "bits_per_byte": 1.5368639693862434,
      "learning_rate": 5.985156736711483e-05,
      "loss_nats": 1.0652729272842407,
      "step": 6360,
      "unclipped_grad_norm": 0.9870114922523499
    },
    {
      "bits_per_byte": 1.5938194638724799,
      "learning_rate": 5.950247341420618e-05,
      "loss_nats": 1.104751467704773,
      "step": 6370,
      "unclipped_grad_norm": 0.9943422675132751
    },
    {
      "bits_per_byte": 1.54489022771936,
      "learning_rate": 5.915518212294935e-05,
      "loss_nats": 1.0708363056182861,
      "step": 6380,
      "unclipped_grad_norm": 0.9544562697410583
    },
    {
      "bits_per_byte": 1.5900604390725714,
      "learning_rate": 5.880969942759425e-05,
      "loss_nats": 1.1021459102630615,
      "step": 6390,
      "unclipped_grad_norm": 0.9881879091262817
    },
    {
      "bits_per_byte": 1.5576724942794364,
      "learning_rate": 5.846603123148688e-05,
      "loss_nats": 1.0796962976455688,
      "step": 6400,
      "unclipped_grad_norm": 0.9512554407119751
    },
    {
      "bits_per_byte": 1.5486053969432982,
      "learning_rate": 5.812418340696858e-05,
      "loss_nats": 1.073411464691162,
      "step": 6410,
      "unclipped_grad_norm": 0.9827495217323303
    },
    {
      "bits_per_byte": 1.5454973264769056,
      "learning_rate": 5.778416179527541e-05,
      "loss_nats": 1.0712571144104004,
      "step": 6420,
      "unclipped_grad_norm": 1.0222079753875732
    },
    {
      "bits_per_byte": 1.5609757650545584,
      "learning_rate": 5.7445972206438635e-05,
      "loss_nats": 1.0819859504699707,
      "step": 6430,
      "unclipped_grad_norm": 1.0177998542785645
    },
    {
      "bits_per_byte": 1.5622200595335507,
      "learning_rate": 5.710962041918516e-05,
      "loss_nats": 1.0828484296798706,
      "step": 6440,
      "unclipped_grad_norm": 0.9765349626541138
    },
    {
      "bits_per_byte": 1.5515923916234835,
      "learning_rate": 5.677511218083908e-05,
      "loss_nats": 1.07548189163208,
      "step": 6450,
      "unclipped_grad_norm": 1.028122901916504
    },
    {
      "bits_per_byte": 1.5316869476299713,
      "learning_rate": 5.644245320722316e-05,
      "loss_nats": 1.061684489250183,
      "step": 6460,
      "unclipped_grad_norm": 1.1802783012390137
    },
    {
      "bits_per_byte": 1.566511026672578,
      "learning_rate": 5.61116491825615e-05,
      "loss_nats": 1.0858227014541626,
      "step": 6470,
      "unclipped_grad_norm": 0.9929527044296265
    },
    {
      "bits_per_byte": 1.5861486936786968,
      "learning_rate": 5.5782705759382104e-05,
      "loss_nats": 1.099434494972229,
      "step": 6480,
      "unclipped_grad_norm": 0.996606171131134
    },
    {
      "bits_per_byte": 1.5680002244463989,
      "learning_rate": 5.5455628558420546e-05,
      "loss_nats": 1.0868549346923828,
      "step": 6490,
      "unclipped_grad_norm": 0.9814039468765259
    },
    {
      "bits_per_byte": 1.568809918766661,
      "learning_rate": 5.513042316852368e-05,
      "loss_nats": 1.087416172027588,
      "step": 6500,
      "unclipped_grad_norm": 1.001226782798767
    },
    {
      "bits_per_byte": 1.513912368680649,
      "learning_rate": 5.480709514655443e-05,
      "loss_nats": 1.0493640899658203,
      "step": 6510,
      "unclipped_grad_norm": 0.9533669948577881
    },
    {
      "bits_per_byte": 1.4992986588715345,
      "learning_rate": 5.448565001729653e-05,
      "loss_nats": 1.0392346382141113,
      "step": 6520,
      "unclipped_grad_norm": 1.0103793144226074
    },
    {
      "bits_per_byte": 1.5872271968822576,
      "learning_rate": 5.41660932733604e-05,
      "loss_nats": 1.100182056427002,
      "step": 6530,
      "unclipped_grad_norm": 1.0064735412597656
    },
    {
      "bits_per_byte": 1.551986575859261,
      "learning_rate": 5.384843037508902e-05,
      "loss_nats": 1.0757551193237305,
      "step": 6540,
      "unclipped_grad_norm": 0.9814162254333496
    },
    {
      "bits_per_byte": 1.6102771716641453,
      "learning_rate": 5.353266675046498e-05,
      "loss_nats": 1.1161590814590454,
      "step": 6550,
      "unclipped_grad_norm": 1.0333638191223145
    },
    {
      "bits_per_byte": 1.5466836628025573,
      "learning_rate": 5.321880779501729e-05,
      "loss_nats": 1.0720794200897217,
      "step": 6560,
      "unclipped_grad_norm": 0.9711053967475891
    },
    {
      "bits_per_byte": 1.4930591282982317,
      "learning_rate": 5.290685887172962e-05,
      "loss_nats": 1.034909725189209,
      "step": 6570,
      "unclipped_grad_norm": 0.9939249157905579
    },
    {
      "bits_per_byte": 1.5761067986774682,
      "learning_rate": 5.2596825310948295e-05,
      "loss_nats": 1.0924739837646484,
      "step": 6580,
      "unclipped_grad_norm": 0.9629629850387573
    },
    {
      "bits_per_byte": 1.5551524324963437,
      "learning_rate": 5.228871241029154e-05,
      "loss_nats": 1.0779495239257812,
      "step": 6590,
      "unclipped_grad_norm": 1.007703185081482
    },
    {
      "bits_per_byte": 1.4715650805586997,
      "learning_rate": 5.198252543455865e-05,
      "loss_nats": 1.0200111865997314,
      "step": 6600,
      "unclipped_grad_norm": 0.9724071621894836
    },
    {
      "bits_per_byte": 1.5956089433547074,
      "learning_rate": 5.167826961564027e-05,
      "loss_nats": 1.1059918403625488,
      "step": 6610,
      "unclipped_grad_norm": 0.9970710277557373
    },
    {
      "bits_per_byte": 1.6125643689379698,
      "learning_rate": 5.137595015242884e-05,
      "loss_nats": 1.1177444458007812,
      "step": 6620,
      "unclipped_grad_norm": 1.0349808931350708
    },
    {
      "bits_per_byte": 1.59399093057539,
      "learning_rate": 5.1075572210729925e-05,
      "loss_nats": 1.104870319366455,
      "step": 6630,
      "unclipped_grad_norm": 1.039212703704834
    },
    {
      "bits_per_byte": 1.5557672704731782,
      "learning_rate": 5.07771409231737e-05,
      "loss_nats": 1.0783756971359253,
      "step": 6640,
      "unclipped_grad_norm": 1.023613452911377
    },
    {
      "bits_per_byte": 1.5432283593640725,
      "learning_rate": 5.048066138912756e-05,
      "loss_nats": 1.069684386253357,
      "step": 6650,
      "unclipped_grad_norm": 0.9931045174598694
    },
    {
      "bits_per_byte": 1.5957387902561087,
      "learning_rate": 5.018613867460863e-05,
      "loss_nats": 1.1060818433761597,
      "step": 6660,
      "unclipped_grad_norm": 1.01187002658844
    },
    {
      "bits_per_byte": 1.5766402888604447,
      "learning_rate": 4.989357781219755e-05,
      "loss_nats": 1.092843770980835,
      "step": 6670,
      "unclipped_grad_norm": 1.0118197202682495
    },
    {
      "bits_per_byte": 1.5937730285567469,
      "learning_rate": 4.960298380095222e-05,
      "loss_nats": 1.1047192811965942,
      "step": 6680,
      "unclipped_grad_norm": 1.0237627029418945
    },
    {
      "bits_per_byte": 1.5627511419594151,
      "learning_rate": 4.931436160632256e-05,
      "loss_nats": 1.0832165479660034,
      "step": 6690,
      "unclipped_grad_norm": 1.0074224472045898
    },
    {
      "bits_per_byte": 1.5937723406261435,
      "learning_rate": 4.9027716160065485e-05,
      "loss_nats": 1.104718804359436,
      "step": 6700,
      "unclipped_grad_norm": 1.01242196559906
    },
    {
      "bits_per_byte": 1.5383141270983187,
      "learning_rate": 4.874305236016084e-05,
      "loss_nats": 1.066278100013733,
      "step": 6710,
      "unclipped_grad_norm": 0.9823194146156311
    },
    {
      "bits_per_byte": 1.5376188012408802,
      "learning_rate": 4.846037507072752e-05,
      "loss_nats": 1.065796136856079,
      "step": 6720,
      "unclipped_grad_norm": 1.027299404144287
    },
    {
      "bits_per_byte": 1.5285384612406259,
      "learning_rate": 4.817968912194049e-05,
      "loss_nats": 1.059502124786377,
      "step": 6730,
      "unclipped_grad_norm": 0.9929317235946655
    },
    {
      "bits_per_byte": 1.5454023920536293,
      "learning_rate": 4.790099930994813e-05,
      "loss_nats": 1.0711913108825684,
      "step": 6740,
      "unclipped_grad_norm": 0.9901378154754639
    },
    {
      "bits_per_byte": 1.5351463786520767,
      "learning_rate": 4.762431039679047e-05,
      "loss_nats": 1.064082384109497,
      "step": 6750,
      "unclipped_grad_norm": 1.026999831199646
    },
    {
      "bits_per_byte": 1.501298817101069,
      "learning_rate": 4.734962711031755e-05,
      "loss_nats": 1.040621042251587,
      "step": 6760,
      "unclipped_grad_norm": 0.9864338040351868
    },
    {
      "bits_per_byte": 1.5231093129181885,
      "learning_rate": 4.707695414410892e-05,
      "loss_nats": 1.055738925933838,
      "step": 6770,
      "unclipped_grad_norm": 0.9597692489624023
    },
    {
      "bits_per_byte": 1.5314098835794312,
      "learning_rate": 4.680629615739322e-05,
      "loss_nats": 1.0614924430847168,
      "step": 6780,
      "unclipped_grad_norm": 0.9848454594612122
    },
    {
      "bits_per_byte": 1.6182862317321756,
      "learning_rate": 4.653765777496863e-05,
      "loss_nats": 1.1217105388641357,
      "step": 6790,
      "unclipped_grad_norm": 0.994420051574707
    },
    {
      "bits_per_byte": 1.6079612532876268,
      "learning_rate": 4.6271043587123986e-05,
      "loss_nats": 1.1145538091659546,
      "step": 6800,
      "unclipped_grad_norm": 1.0759928226470947
    },
    {
      "bits_per_byte": 1.5649406530875496,
      "learning_rate": 4.600645814956006e-05,
      "loss_nats": 1.0847342014312744,
      "step": 6810,
      "unclipped_grad_norm": 1.0181862115859985
    },
    {
      "bits_per_byte": 1.5209997737227055,
      "learning_rate": 4.574390598331204e-05,
      "loss_nats": 1.054276704788208,
      "step": 6820,
      "unclipped_grad_norm": 0.9867210984230042
    },
    {
      "bits_per_byte": 1.5408068436399243,
      "learning_rate": 4.548339157467199e-05,
      "loss_nats": 1.068005919456482,
      "step": 6830,
      "unclipped_grad_norm": 1.0285433530807495
    },
    {
      "bits_per_byte": 1.5661266454478995,
      "learning_rate": 4.52249193751124e-05,
      "loss_nats": 1.0855562686920166,
      "step": 6840,
      "unclipped_grad_norm": 0.9897852540016174
    },
    {
      "bits_per_byte": 1.5403562490946636,
      "learning_rate": 4.4968493801209985e-05,
      "loss_nats": 1.0676935911178589,
      "step": 6850,
      "unclipped_grad_norm": 0.9739700555801392
    },
    {
      "bits_per_byte": 1.5589986525002395,
      "learning_rate": 4.471411923457034e-05,
      "loss_nats": 1.080615520477295,
      "step": 6860,
      "unclipped_grad_norm": 1.0206658840179443
    },
    {
      "bits_per_byte": 1.5592929148158658,
      "learning_rate": 4.446180002175291e-05,
      "loss_nats": 1.0808194875717163,
      "step": 6870,
      "unclipped_grad_norm": 1.015761137008667
    },
    {
      "bits_per_byte": 1.5959116328202259,
      "learning_rate": 4.421154047419693e-05,
      "loss_nats": 1.1062016487121582,
      "step": 6880,
      "unclipped_grad_norm": 1.0070849657058716
    },
    {
      "bits_per_byte": 1.4912815156189139,
      "learning_rate": 4.396334486814753e-05,
      "loss_nats": 1.033677577972412,
      "step": 6890,
      "unclipped_grad_norm": 0.9983416199684143
    },
    {
      "bits_per_byte": 1.5029766798428865,
      "learning_rate": 4.371721744458285e-05,
      "loss_nats": 1.0417840480804443,
      "step": 6900,
      "unclipped_grad_norm": 0.9461802840232849
    },
    {
      "bits_per_byte": 1.691495806551456,
      "learning_rate": 4.347316240914145e-05,
      "loss_nats": 1.1724555492401123,
      "step": 6910,
      "unclipped_grad_norm": 1.0603437423706055
    },
    {
      "bits_per_byte": 1.4926446501096524,
      "learning_rate": 4.3231183932050546e-05,
      "loss_nats": 1.0346224308013916,
      "step": 6920,
      "unclipped_grad_norm": 0.9669191241264343
    },
    {
      "bits_per_byte": 1.5121873826924952,
      "learning_rate": 4.299128614805464e-05,
      "loss_nats": 1.048168420791626,
      "step": 6930,
      "unclipped_grad_norm": 1.012385606765747
    },
    {
      "bits_per_byte": 1.5487118542041824,
      "learning_rate": 4.2753473156344965e-05,
      "loss_nats": 1.073485255241394,
      "step": 6940,
      "unclipped_grad_norm": 1.0546537637710571
    },
    {
      "bits_per_byte": 1.5353376233598361,
      "learning_rate": 4.251774902048942e-05,
      "loss_nats": 1.0642149448394775,
      "step": 6950,
      "unclipped_grad_norm": 0.9847853779792786
    },
    {
      "bits_per_byte": 1.5735119244412503,
      "learning_rate": 4.228411776836306e-05,
      "loss_nats": 1.0906753540039062,
      "step": 6960,
      "unclipped_grad_norm": 1.0091943740844727
    },
    {
      "bits_per_byte": 1.5592397721767493,
      "learning_rate": 4.2052583392079427e-05,
      "loss_nats": 1.0807826519012451,
      "step": 6970,
      "unclipped_grad_norm": 1.0198149681091309
    },
    {
      "bits_per_byte": 1.6070454456717824,
      "learning_rate": 4.182314984792214e-05,
      "loss_nats": 1.1139190196990967,
      "step": 6980,
      "unclipped_grad_norm": 1.0826783180236816
    },
    {
      "bits_per_byte": 1.5476336949659235,
      "learning_rate": 4.159582105627748e-05,
      "loss_nats": 1.0727379322052002,
      "step": 6990,
      "unclipped_grad_norm": 1.0041176080703735
    },
    {
      "bits_per_byte": 1.5120640711318265,
      "learning_rate": 4.137060090156724e-05,
      "loss_nats": 1.048082947731018,
      "step": 7000,
      "unclipped_grad_norm": 1.0215020179748535
    },
    {
      "bits_per_byte": 1.5377993830242862,
      "learning_rate": 4.114749323218249e-05,
      "loss_nats": 1.0659213066101074,
      "step": 7010,
      "unclipped_grad_norm": 1.0378373861312866
    },
    {
      "bits_per_byte": 1.5074898485668284,
      "learning_rate": 4.0926501860417706e-05,
      "loss_nats": 1.044912338256836,
      "step": 7020,
      "unclipped_grad_norm": 0.982413113117218
    },
    {
      "bits_per_byte": 1.5236237130269192,
      "learning_rate": 4.070763056240571e-05,
      "loss_nats": 1.0560954809188843,
      "step": 7030,
      "unclipped_grad_norm": 0.9606820344924927
    },
    {
      "bits_per_byte": 1.6050487270952651,
      "learning_rate": 4.049088307805306e-05,
      "loss_nats": 1.112534999847412,
      "step": 7040,
      "unclipped_grad_norm": 1.0613288879394531
    },
    {
      "bits_per_byte": 1.5624771735965908,
      "learning_rate": 4.027626311097629e-05,
      "loss_nats": 1.083026647567749,
      "step": 7050,
      "unclipped_grad_norm": 1.0451524257659912
    },
    {
      "bits_per_byte": 1.5626594752065053,
      "learning_rate": 4.006377432843846e-05,
      "loss_nats": 1.0831530094146729,
      "step": 7060,
      "unclipped_grad_norm": 1.073291540145874
    },
    {
      "bits_per_byte": 1.519674131449855,
      "learning_rate": 3.985342036128656e-05,
      "loss_nats": 1.0533578395843506,
      "step": 7070,
      "unclipped_grad_norm": 1.009161353111267
    },
    {
      "bits_per_byte": 1.4484239549718545,
      "learning_rate": 3.964520480388956e-05,
      "loss_nats": 1.003970980644226,
      "step": 7080,
      "unclipped_grad_norm": 0.9980688691139221
    },
    {
      "bits_per_byte": 1.539162345532374,
      "learning_rate": 3.943913121407678e-05,
      "loss_nats": 1.0668660402297974,
      "step": 7090,
      "unclipped_grad_norm": 1.0500024557113647
    },
    {
      "bits_per_byte": 1.5110017342974469,
      "learning_rate": 3.923520311307735e-05,
      "loss_nats": 1.047346591949463,
      "step": 7100,
      "unclipped_grad_norm": 1.038030743598938
    },
    {
      "bits_per_byte": 1.5757193217650745,
      "learning_rate": 3.9033423985459816e-05,
      "loss_nats": 1.0922054052352905,
      "step": 7110,
      "unclipped_grad_norm": 1.0732139348983765
    },
    {
      "bits_per_byte": 1.5620382738715888,
      "learning_rate": 3.88337972790728e-05,
      "loss_nats": 1.0827224254608154,
      "step": 7120,
      "unclipped_grad_norm": 1.0506101846694946
    },
    {
      "bits_per_byte": 1.4981061311704518,
      "learning_rate": 3.8636326404985845e-05,
      "loss_nats": 1.0384080410003662,
      "step": 7130,
      "unclipped_grad_norm": 1.0116451978683472
    },
    {
      "bits_per_byte": 1.6131864301861407,
      "learning_rate": 3.844101473743143e-05,
      "loss_nats": 1.1181756258010864,
      "step": 7140,
      "unclipped_grad_norm": 1.0758471488952637
    },
    {
      "bits_per_byte": 1.623974385926812,
      "learning_rate": 3.824786561374701e-05,
      "loss_nats": 1.1256532669067383,
      "step": 7150,
      "unclipped_grad_norm": 1.0434077978134155
    },
    {
      "bits_per_byte": 1.5482879169698056,
      "learning_rate": 3.805688233431824e-05,
      "loss_nats": 1.0731914043426514,
      "step": 7160,
      "unclipped_grad_norm": 1.0636146068572998
    },
    {
      "bits_per_byte": 1.5506334163622726,
      "learning_rate": 3.786806816252236e-05,
      "loss_nats": 1.074817180633545,
      "step": 7170,
      "unclipped_grad_norm": 1.0567522048950195
    },
    {
      "bits_per_byte": 1.468200068011918,
      "learning_rate": 3.768142632467269e-05,
      "loss_nats": 1.0176787376403809,
      "step": 7180,
      "unclipped_grad_norm": 0.9437705874443054
    },
    {
      "bits_per_byte": 1.4717380951054677,
      "learning_rate": 3.7496960009963236e-05,
      "loss_nats": 1.0201311111450195,
      "step": 7190,
      "unclipped_grad_norm": 1.010156512260437
    },
    {
      "bits_per_byte": 1.5698447383769025,
      "learning_rate": 3.7314672370414325e-05,
      "loss_nats": 1.088133454322815,
      "step": 7200,
      "unclipped_grad_norm": 1.056505560874939
    },
    {
      "bits_per_byte": 1.56738452655631,
      "learning_rate": 3.713456652081882e-05,
      "loss_nats": 1.086428165435791,
      "step": 7210,
      "unclipped_grad_norm": 1.0318564176559448
    },
    {
      "bits_per_byte": 1.523764394835325,
      "learning_rate": 3.695664553868873e-05,
      "loss_nats": 1.0561929941177368,
      "step": 7220,
      "unclipped_grad_norm": 1.0528209209442139
    },
    {
      "bits_per_byte": 1.5338092135416184,
      "learning_rate": 3.678091246420277e-05,
      "loss_nats": 1.0631555318832397,
      "step": 7230,
      "unclipped_grad_norm": 1.025399923324585
    },
    {
      "bits_per_byte": 1.490086580160719,
      "learning_rate": 3.660737030015427e-05,
      "loss_nats": 1.0328493118286133,
      "step": 7240,
      "unclipped_grad_norm": 0.9961045980453491
    },
    {
      "bits_per_byte": 1.5541721313864256,
      "learning_rate": 3.6436022011900056e-05,
      "loss_nats": 1.0772700309753418,
      "step": 7250,
      "unclipped_grad_norm": 1.0621839761734009
    },
    {
      "bits_per_byte": 1.5116575041451867,
      "learning_rate": 3.626687052730954e-05,
      "loss_nats": 1.04780113697052,
      "step": 7260,
      "unclipped_grad_norm": 1.0816527605056763
    },
    {
      "bits_per_byte": 1.5896555919124404,
      "learning_rate": 3.6099918736714955e-05,
      "loss_nats": 1.101865291595459,
      "step": 7270,
      "unclipped_grad_norm": 1.059293270111084
    },
    {
      "bits_per_byte": 1.5429449319554507,
      "learning_rate": 3.5935169492861716e-05,
      "loss_nats": 1.0694879293441772,
      "step": 7280,
      "unclipped_grad_norm": 1.0573333501815796
    },
    {
      "bits_per_byte": 1.5487371356538593,
      "learning_rate": 3.5772625610859886e-05,
      "loss_nats": 1.073502779006958,
      "step": 7290,
      "unclipped_grad_norm": 1.0477967262268066
    },
    {
      "bits_per_byte": 1.5296157605656304,
      "learning_rate": 3.5612289868135887e-05,
      "loss_nats": 1.060248851776123,
      "step": 7300,
      "unclipped_grad_norm": 1.0505754947662354
    },
    {
      "bits_per_byte": 1.57645196785775,
      "learning_rate": 3.545416500438522e-05,
      "loss_nats": 1.0927132368087769,
      "step": 7310,
      "unclipped_grad_norm": 1.0852391719818115
    },
    {
      "bits_per_byte": 1.5598148821612343,
      "learning_rate": 3.52982537215255e-05,
      "loss_nats": 1.081181287765503,
      "step": 7320,
      "unclipped_grad_norm": 1.079657793045044
    },
    {
      "bits_per_byte": 1.5641130725715977,
      "learning_rate": 3.5144558683650396e-05,
      "loss_nats": 1.084160566329956,
      "step": 7330,
      "unclipped_grad_norm": 1.0734436511993408
    },
    {
      "bits_per_byte": 1.5372211773520854,
      "learning_rate": 3.499308251698404e-05,
      "loss_nats": 1.0655205249786377,
      "step": 7340,
      "unclipped_grad_norm": 1.0373533964157104
    },
    {
      "bits_per_byte": 1.585016531888067,
      "learning_rate": 3.484382780983616e-05,
      "loss_nats": 1.0986497402191162,
      "step": 7350,
      "unclipped_grad_norm": 1.034577488899231
    },
    {
      "bits_per_byte": 1.5482229075277794,
      "learning_rate": 3.469679711255795e-05,
      "loss_nats": 1.0731463432312012,
      "step": 7360,
      "unclipped_grad_norm": 0.9948747754096985
    },
    {
      "bits_per_byte": 1.5566985565276004,
      "learning_rate": 3.4551992937498284e-05,
      "loss_nats": 1.0790212154388428,
      "step": 7370,
      "unclipped_grad_norm": 1.0546287298202515
    },
    {
      "bits_per_byte": 1.614560055618582,
      "learning_rate": 3.440941775896104e-05,
      "loss_nats": 1.1191277503967285,
      "step": 7380,
      "unclipped_grad_norm": 1.1286121606826782
    },
    {
      "bits_per_byte": 1.5888789182611438,
      "learning_rate": 3.4269074013162596e-05,
      "loss_nats": 1.1013269424438477,
      "step": 7390,
      "unclipped_grad_norm": 1.063069224357605
    },
    {
      "bits_per_byte": 1.5174531474966126,
      "learning_rate": 3.41309640981904e-05,
      "loss_nats": 1.0518183708190918,
      "step": 7400,
      "unclipped_grad_norm": 1.038301944732666
    },
    {
      "bits_per_byte": 1.5128250943618944,
      "learning_rate": 3.3995090373961795e-05,
      "loss_nats": 1.0486104488372803,
      "step": 7410,
      "unclipped_grad_norm": 0.9619468450546265
    },
    {
      "bits_per_byte": 1.6064108296900985,
      "learning_rate": 3.3861455162183876e-05,
      "loss_nats": 1.1134791374206543,
      "step": 7420,
      "unclipped_grad_norm": 1.0911381244659424
    },
    {
      "bits_per_byte": 1.5182607780250643,
      "learning_rate": 3.3730060746313664e-05,
      "loss_nats": 1.0523781776428223,
      "step": 7430,
      "unclipped_grad_norm": 1.0977153778076172
    },
    {
      "bits_per_byte": 1.527175670715189,
      "learning_rate": 3.3600909371519225e-05,
      "loss_nats": 1.0585575103759766,
      "step": 7440,
      "unclipped_grad_norm": 1.035995602607727
    },
    {
      "bits_per_byte": 1.4906346888690187,
      "learning_rate": 3.3474003244641194e-05,
      "loss_nats": 1.0332292318344116,
      "step": 7450,
      "unclipped_grad_norm": 1.1210148334503174
    },
    {
      "bits_per_byte": 1.5053576076614312,
      "learning_rate": 3.334934453415514e-05,
      "loss_nats": 1.0434343814849854,
      "step": 7460,
      "unclipped_grad_norm": 1.0129456520080566
    },
    {
      "bits_per_byte": 1.555529418467035,
      "learning_rate": 3.3226935370134486e-05,
      "loss_nats": 1.0782108306884766,
      "step": 7470,
      "unclipped_grad_norm": 1.0514966249465942
    },
    {
      "bits_per_byte": 1.4886658314819414,
      "learning_rate": 3.31067778442141e-05,
      "loss_nats": 1.0318645238876343,
      "step": 7480,
      "unclipped_grad_norm": 1.032109260559082
    },
    {
      "bits_per_byte": 1.51988842183283,
      "learning_rate": 3.298887400955457e-05,
      "loss_nats": 1.0535063743591309,
      "step": 7490,
      "unclipped_grad_norm": 1.0880528688430786
    },
    {
      "bits_per_byte": 1.5815053340880518,
      "learning_rate": 3.287322588080714e-05,
      "loss_nats": 1.0962159633636475,
      "step": 7500,
      "unclipped_grad_norm": 1.0377991199493408
    },
    {
      "bits_per_byte": 1.5370581377990675,
      "learning_rate": 3.275983543407924e-05,
      "loss_nats": 1.0654075145721436,
      "step": 7510,
      "unclipped_grad_norm": 1.0226150751113892
    },
    {
      "bits_per_byte": 1.5380518535557528,
      "learning_rate": 3.2648704606900735e-05,
      "loss_nats": 1.066096305847168,
      "step": 7520,
      "unclipped_grad_norm": 1.0527992248535156
    },
    {
      "bits_per_byte": 1.562738587225902,
      "learning_rate": 3.253983529819086e-05,
      "loss_nats": 1.0832078456878662,
      "step": 7530,
      "unclipped_grad_norm": 1.072679042816162
    },
    {
      "bits_per_byte": 1.6011416252329638,
      "learning_rate": 3.243322936822571e-05,
      "loss_nats": 1.1098268032073975,
      "step": 7540,
      "unclipped_grad_norm": 1.109021544456482
    },
    {
      "bits_per_byte": 1.5189222233002826,
      "learning_rate": 3.232888863860648e-05,
      "loss_nats": 1.0528366565704346,
      "step": 7550,
      "unclipped_grad_norm": 1.0227179527282715
    },
    {
      "bits_per_byte": 1.5563348132210255,
      "learning_rate": 3.222681489222838e-05,
      "loss_nats": 1.0787690877914429,
      "step": 7560,
      "unclipped_grad_norm": 1.0419527292251587
    },
    {
      "bits_per_byte": 1.5726612982500827,
      "learning_rate": 3.212700987325009e-05,
      "loss_nats": 1.090085744857788,
      "step": 7570,
      "unclipped_grad_norm": 1.03652822971344
    },
    {
      "bits_per_byte": 1.5146027070412125,
      "learning_rate": 3.202947528706403e-05,
      "loss_nats": 1.0498425960540771,
      "step": 7580,
      "unclipped_grad_norm": 1.0102300643920898
    },
    {
      "bits_per_byte": 1.5228090312097822,
      "learning_rate": 3.1934212800267116e-05,
      "loss_nats": 1.0555307865142822,
      "step": 7590,
      "unclipped_grad_norm": 1.049957513809204
    },
    {
      "bits_per_byte": 1.5153031923781766,
      "learning_rate": 3.1841224040632484e-05,
      "loss_nats": 1.0503281354904175,
      "step": 7600,
      "unclipped_grad_norm": 1.020882248878479
    },
    {
      "bits_per_byte": 1.4615728885435706,
      "learning_rate": 3.175051059708143e-05,
      "loss_nats": 1.013085126876831,
      "step": 7610,
      "unclipped_grad_norm": 1.0289204120635986
    },
    {
      "bits_per_byte": 1.5704728190178536,
      "learning_rate": 3.1662074019656414e-05,
      "loss_nats": 1.0885688066482544,
      "step": 7620,
      "unclipped_grad_norm": 1.062748670578003
    },
    {
      "bits_per_byte": 1.5187743182205407,
      "learning_rate": 3.157591581949455e-05,
      "loss_nats": 1.052734136581421,
      "step": 7630,
      "unclipped_grad_norm": 1.036866307258606
    },
    {
      "bits_per_byte": 1.5251964943690597,
      "learning_rate": 3.149203746880175e-05,
      "loss_nats": 1.0571856498718262,
      "step": 7640,
      "unclipped_grad_norm": 1.0375516414642334
    },
    {
      "bits_per_byte": 1.625032595177571,
      "learning_rate": 3.141044040082763e-05,
      "loss_nats": 1.1263867616653442,
      "step": 7650,
      "unclipped_grad_norm": 1.15651273727417
    },
    {
      "bits_per_byte": 1.5325501285546517,
      "learning_rate": 3.133112600984091e-05,
      "loss_nats": 1.0622828006744385,
      "step": 7660,
      "unclipped_grad_norm": 1.0661544799804688
    },
    {
      "bits_per_byte": 1.5264065643005305,
      "learning_rate": 3.125409565110571e-05,
      "loss_nats": 1.0580244064331055,
      "step": 7670,
      "unclipped_grad_norm": 1.0767730474472046
    },
    {
      "bits_per_byte": 1.490845023651024,
      "learning_rate": 3.117935064085835e-05,
      "loss_nats": 1.0333750247955322,
      "step": 7680,
      "unclipped_grad_norm": 1.040376901626587
    },
    {
      "bits_per_byte": 1.53468460523451,
      "learning_rate": 3.110689225628479e-05,
      "loss_nats": 1.0637623071670532,
      "step": 7690,
      "unclipped_grad_norm": 1.0646589994430542
    },
    {
      "bits_per_byte": 1.5464913861988927,
      "learning_rate": 3.1036721735498926e-05,
      "loss_nats": 1.071946144104004,
      "step": 7700,
      "unclipped_grad_norm": 1.1067028045654297
    },
    {
      "bits_per_byte": 1.4665762078224713,
      "learning_rate": 3.096884027752132e-05,
      "loss_nats": 1.0165531635284424,
      "step": 7710,
      "unclipped_grad_norm": 1.0252745151519775
    },
    {
      "bits_per_byte": 1.4804371495687594,
      "learning_rate": 3.090324904225885e-05,
      "loss_nats": 1.0261608362197876,
      "step": 7720,
      "unclipped_grad_norm": 1.0019656419754028
    },
    {
      "bits_per_byte": 1.5129640563437916,
      "learning_rate": 3.083994915048472e-05,
      "loss_nats": 1.0487067699432373,
      "step": 7730,
      "unclipped_grad_norm": 1.0048695802688599
    },
    {
      "bits_per_byte": 1.5117082390271912,
      "learning_rate": 3.077894168381942e-05,
      "loss_nats": 1.0478363037109375,
      "step": 7740,
      "unclipped_grad_norm": 1.034838080406189
    },
    {
      "bits_per_byte": 1.5481713127325205,
      "learning_rate": 3.0720227684712254e-05,
      "loss_nats": 1.073110580444336,
      "step": 7750,
      "unclipped_grad_norm": 1.047455072402954
    },
    {
      "bits_per_byte": 1.5013908278192805,
      "learning_rate": 3.0663808156423456e-05,
      "loss_nats": 1.0406848192214966,
      "step": 7760,
      "unclipped_grad_norm": 1.096320629119873
    },
    {
      "bits_per_byte": 1.5005262710333933,
      "learning_rate": 3.060968406300711e-05,
      "loss_nats": 1.0400855541229248,
      "step": 7770,
      "unclipped_grad_norm": 1.0564236640930176
    },
    {
      "bits_per_byte": 1.6389760885962732,
      "learning_rate": 3.055785632929463e-05,
      "loss_nats": 1.1360516548156738,
      "step": 7780,
      "unclipped_grad_norm": 1.134232997894287
    },
    {
      "bits_per_byte": 1.5295576304296388,
      "learning_rate": 3.0508325840878984e-05,
      "loss_nats": 1.0602085590362549,
      "step": 7790,
      "unclipped_grad_norm": 1.0709397792816162
    },
    {
      "bits_per_byte": 1.5624484524938966,
      "learning_rate": 3.046109344409957e-05,
      "loss_nats": 1.083006739616394,
      "step": 7800,
      "unclipped_grad_norm": 1.099833607673645
    },
    {
      "bits_per_byte": 1.476959144420361,
      "learning_rate": 3.0416159946027727e-05,
      "loss_nats": 1.0237500667572021,
      "step": 7810,
      "unclipped_grad_norm": 1.057004690170288
    },
    {
      "bits_per_byte": 1.5143643390871164,
      "learning_rate": 3.0373526114452946e-05,
      "loss_nats": 1.0496773719787598,
      "step": 7820,
      "unclipped_grad_norm": 1.0959510803222656
    },
    {
      "bits_per_byte": 1.5593489811600472,
      "learning_rate": 3.0333192677869803e-05,
      "loss_nats": 1.0808583498001099,
      "step": 7830,
      "unclipped_grad_norm": 1.063362956047058
    },
    {
      "bits_per_byte": 1.4978546925348903,
      "learning_rate": 3.029516032546544e-05,
      "loss_nats": 1.038233757019043,
      "step": 7840,
      "unclipped_grad_norm": 1.0525480508804321
    },
    {
      "bits_per_byte": 1.4918236049344333,
      "learning_rate": 3.02594297071078e-05,
      "loss_nats": 1.0340533256530762,
      "step": 7850,
      "unclipped_grad_norm": 1.030691146850586
    },
    {
      "bits_per_byte": 1.5064691315339576,
      "learning_rate": 3.0226001433334603e-05,
      "loss_nats": 1.044204831123352,
      "step": 7860,
      "unclipped_grad_norm": 1.0765224695205688
    },
    {
      "bits_per_byte": 1.5562261201856802,
      "learning_rate": 3.019487607534278e-05,
      "loss_nats": 1.0786937475204468,
      "step": 7870,
      "unclipped_grad_norm": 1.0955042839050293
    },
    {
      "bits_per_byte": 1.5252102529811287,
      "learning_rate": 3.016605416497886e-05,
      "loss_nats": 1.0571951866149902,
      "step": 7880,
      "unclipped_grad_norm": 1.043196201324463
    },
    {
      "bits_per_byte": 1.5900611270031748,
      "learning_rate": 3.0139536194729734e-05,
      "loss_nats": 1.1021463871002197,
      "step": 7890,
      "unclipped_grad_norm": 1.104681372642517
    },
    {
      "bits_per_byte": 1.5606699799013244,
      "learning_rate": 3.0115322617714362e-05,
      "loss_nats": 1.0817739963531494,
      "step": 7900,
      "unclipped_grad_norm": 1.0641751289367676
    },
    {
      "bits_per_byte": 1.5589052659208211,
      "learning_rate": 3.0093413847675965e-05,
      "loss_nats": 1.0805507898330688,
      "step": 7910,
      "unclipped_grad_norm": 1.1519651412963867
    },
    {
      "bits_per_byte": 1.481516856650876,
      "learning_rate": 3.0073810258974985e-05,
      "loss_nats": 1.0269092321395874,
      "step": 7920,
      "unclipped_grad_norm": 1.0578134059906006
    },
    {
      "bits_per_byte": 1.4791508913229565,
      "learning_rate": 3.0056512186582658e-05,
      "loss_nats": 1.0252692699432373,
      "step": 7930,
      "unclipped_grad_norm": 1.0473592281341553
    },
    {
      "bits_per_byte": 1.5474438261193708,
      "learning_rate": 3.0041519926075316e-05,
      "loss_nats": 1.0726063251495361,
      "step": 7940,
      "unclipped_grad_norm": 1.1337641477584839
    },
    {
      "bits_per_byte": 1.4870933941051028,
      "learning_rate": 3.0028833733629313e-05,
      "loss_nats": 1.0307745933532715,
      "step": 7950,
      "unclipped_grad_norm": 1.0619428157806396
    },
    {
      "bits_per_byte": 1.5210572159280935,
      "learning_rate": 3.0018453826016686e-05,
      "loss_nats": 1.054316520690918,
      "step": 7960,
      "unclipped_grad_norm": 1.0414892435073853
    },
    {
      "bits_per_byte": 1.5042326691421377,
      "learning_rate": 3.001038038060143e-05,
      "loss_nats": 1.0426546335220337,
      "step": 7970,
      "unclipped_grad_norm": 1.0393376350402832
    },
    {
      "bits_per_byte": 1.5355154534208282,
      "learning_rate": 3.0004613535336433e-05,
      "loss_nats": 1.064338207244873,
      "step": 7980,
      "unclipped_grad_norm": 1.0560662746429443
    },
    {
      "bits_per_byte": 1.5624764856659872,
      "learning_rate": 3.000115338876117e-05,
      "loss_nats": 1.0830261707305908,
      "step": 7990,
      "unclipped_grad_norm": 1.1011888980865479
    },
    {
      "bits_per_byte": 1.4795096471326563,
      "learning_rate": 3e-05,
      "loss_nats": 1.0255179405212402,
      "step": 8000,
      "unclipped_grad_norm": 1.0237547159194946
    }
  ]
}