{
  "architecture": "block",
  "batch_size": 32,
  "canonical_sha256_without_self": "a4c4ddb026a74a7ed70ffc58516bae9bdba5550dc232a1cb003063f2c7f9ca1d",
  "depth": 16,
  "diagnostics": [
    {
      "activation_grad_rms_by_block": [
        0.0003044046170543879,
        0.0002627764770295471,
        0.0002327302354387939,
        0.0002064981235889718,
        0.0001888095575850457,
        0.00017376731557305902,
        0.00016044666699599475,
        0.0001515327749075368,
        0.000144502759212628,
        0.00014044572890270501,
        0.00013478762411978096,
        0.00013080857752356678,
        0.00012769536988344043,
        0.00012482800229918212,
        0.00012209055421408266,
        0.0001202532512252219
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.0002516023632779252,
        "first_to_last_ratio": 2.0336961080101905,
        "imbalance_abs_log_ratio": 0.7098548803728145,
        "last_quartile_mean": 0.00012371679440548178,
        "mean": 0.00017039860222212155,
        "normalized": [
          1.7864267258341946,
          1.5421281254819637,
          1.3657989702017657,
          1.2118533890306977,
          1.1080463989893807,
          1.0197696067162934,
          0.941596145177564,
          0.8892841427772256,
          0.8480278437041564,
          0.8242187850791655,
          0.7910136724248463,
          0.7676622684560079,
          0.7493921206993488,
          0.7325647081098923,
          0.7164997401500549,
          0.7057173571674424
        ],
        "population_cv": 0.31342321744292045
      },
      "activation_output_rms_by_block": [
        0.005092422943562269,
        0.007266214583069086,
        0.00560376700013876,
        0.007757550571113825,
        0.005751485005021095,
        0.008497623726725578,
        0.006417509168386459,
        0.009013078175485134,
        0.006685016211122274,
        0.010579346679151058,
        0.007517023477703333,
        0.010370824486017227,
        0.008143824525177479,
        0.011447666212916374,
        0.009210918098688126,
        0.013363312929868698
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.006429988774470985,
        "first_to_last_ratio": 0.6099730781372781,
        "imbalance_abs_log_ratio": 0.4943404569899061,
        "last_quartile_mean": 0.01054143044166267,
        "mean": 0.008294848987134174,
        "normalized": [
          0.6139259378273111,
          0.8759911837261217,
          0.675571913223562,
          0.93522505149235,
          0.6933803151741528,
          1.0244458627162376,
          0.7736740208701105,
          1.0865873736176486,
          0.8059237993954019,
          1.275411607319227,
          0.9062278878569946,
          1.250272850307826,
          0.9817929823447126,
          1.3800933845417096,
          1.1104383109294493,
          1.6110375186571844
        ],
        "population_cv": 0.2672227042610828
      },
      "bits_per_byte": 8.019773211317682,
      "branch_output_rms_by_sublayer": [
        0.0034366261679679155,
        0.003824356012046337,
        0.0035537455696612597,
        0.003800130682066083,
        0.004073208663612604,
        0.0038295022677630186,
        0.004064237233251333,
        0.0037886372301727533,
        0.004280691966414452,
        0.0038349912501871586,
        0.0050888159312307835,
        0.0038198749534785748,
        0.005177160259336233,
        0.0037978272885084152,
        0.005646859761327505,
        0.003781275823712349,
        0.0056319693103432655,
        0.003643249161541462,
        0.006857539992779493,
        0.0037886397913098335,
        0.006555364932864904,
        0.003700864966958761,
        0.0065165311098098755,
        0.0036778782960027456,
        0.0071639628149569035,
        0.003695491934195161,
        0.007599730510264635,
        0.0037103460635989904,
        0.00840203370898962,
        0.0037034295964986086,
        0.008103356696665287,
        0.0036014309152960777
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.006980641236819168,
        0.006790815718474037,
        0.006855853249106197,
        0.00601204909014858,
        0.005899153963056259,
        0.006300783956032697,
        0.005704077288608638,
        0.005731973180528048,
        0.005353243010845193,
        0.00629832195028187,
        0.005348066742481173,
        0.005021425359765903,
        0.004977505888369926,
        0.004996266682376707,
        0.0052721592596541,
        0.004713786845169383
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.006659839823636995,
        "first_to_last_ratio": 1.3346560503958143,
        "imbalance_abs_log_ratio": 0.2886736185043388,
        "last_quartile_mean": 0.004989929668892529,
        "mean": 0.005766007713857367,
        "normalized": [
          1.2106541619850226,
          1.1777326801269727,
          1.189012153526888,
          1.0426710106023453,
          1.0230915836062624,
          1.092746362598529,
          0.9892593925776594,
          0.9940973833164453,
          0.9284141257702132,
          1.0923193763936871,
          0.9275164043967956,
          0.8708669167573224,
          0.8632499530667561,
          0.8665036417431853,
          0.914351752770568,
          0.8175131007613471
        ],
        "population_cv": 0.12132893681311732
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6931471824645996,
          "mean_weights": [
            0.5,
            0.5
          ],
          "sources": 2
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0986123085021973,
          "mean_weights": [
            0.3333333432674408,
            0.3333333432674408,
            0.3333333432674408
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862943649291992,
          "mean_weights": [
            0.25,
            0.25,
            0.25,
            0.25
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6094379425048828,
          "mean_weights": [
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343,
            0.20000001788139343
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791759729385376,
          "mean_weights": [
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592,
            0.1666666567325592
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9459102153778076,
          "mean_weights": [
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548,
            0.1428571492433548
          ],
          "sources": 7
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079441547393799,
          "mean_weights": [
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125,
            0.125
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1972246170043945,
          "mean_weights": [
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519,
            0.11111113429069519
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.028142472729086876,
        0.014130397699773312,
        0.014246499165892601,
        0.01435180939733982,
        0.01446386519819498,
        0.009747256524860859,
        0.009819178842008114,
        0.00989843625575304,
        0.009975779801607132,
        0.007541276980191469,
        0.007608718704432249,
        0.007703919429332018,
        0.0077462210319936275,
        0.00631322804838419,
        0.006355294957756996,
        0.006443085614591837,
        0.006486334837973118,
        0.005488897208124399,
        0.00551867438480258,
        0.005632160697132349,
        0.0056700571440160275,
        0.004926848225295544,
        0.004952999297529459,
        0.005005215294659138,
        0.005028663668781519,
        0.004523467272520065,
        0.004554289858788252,
        0.004648423288017511,
        0.004657648038119078,
        0.004240449517965317,
        0.0042638350278139114,
        0.004358324687927961
      ],
      "loss_nats": 5.558883190155029,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 2.1972246170043945,
        "mean_weights": [
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519,
          0.11111113429069519
        ],
        "sources": 9
      },
      "step": 0,
      "stream_state_rms_by_sublayer": [
        0.0034366261679679155,
        0.005092422943562269,
        0.006235890556126833,
        0.007266214583069086,
        0.004073208663612604,
        0.00560376700013876,
        0.006768420804291964,
        0.007757550571113825,
        0.004280691966414452,
        0.005751485005021095,
        0.00762521754950285,
        0.008497623726725578,
        0.005177160259336233,
        0.006417509168386459,
        0.008189847692847252,
        0.009013078175485134,
        0.0056319693103432655,
        0.006685016211122274,
        0.009837556630373001,
        0.010579346679151058,
        0.006555364932864904,
        0.007517023477703333,
        0.009742370806634426,
        0.010370824486017227,
        0.0071639628149569035,
        0.008143824525177479,
        0.01089458167552948,
        0.011447666212916374,
        0.00840203370898962,
        0.009210918098688126,
        0.012791652232408524,
        0.013363312929868698
      ]
    },
    {
      "activation_grad_rms_by_block": [
        7.495960016967729e-05,
        6.749085150659084e-05,
        5.70677402720321e-05,
        5.0705322792055085e-05,
        4.370361421024427e-05,
        3.821287100436166e-05,
        3.268615546403453e-05,
        2.8849746740888804e-05,
        2.5781664589885622e-05,
        2.37614458455937e-05,
        2.279552791151218e-05,
        2.1788309823023155e-05,
        2.0940320609952323e-05,
        2.0449986550374888e-05,
        1.9821660316665657e-05,
        1.9541532310540788e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 6.255587868508883e-05,
        "first_to_last_ratio": 3.098608919721194,
        "imbalance_abs_log_ratio": 1.1309532752053548,
        "last_quartile_mean": 2.0188374946883414e-05,
        "mean": 3.5534771882339555e-05,
        "normalized": [
          2.1094718271409953,
          1.8992904113768394,
          1.605968949540217,
          1.4269212972563121,
          1.229883066506037,
          1.0753655920710468,
          0.9198358039911673,
          0.811873700397296,
          0.7255334204832441,
          0.6686815360535046,
          0.6414992049756577,
          0.6131546276747519,
          0.5892909817822544,
          0.575492270446749,
          0.5578102592665534,
          0.549927051037374
        ],
        "population_cv": 0.4936925451995582
      },
      "activation_output_rms_by_block": [
        0.009570118971168995,
        0.01747945137321949,
        0.011283415369689465,
        0.020727796480059624,
        0.012305145151913166,
        0.023130038753151894,
        0.015205477364361286,
        0.028237100690603256,
        0.017268717288970947,
        0.03324708342552185,
        0.021605173125863075,
        0.04313390702009201,
        0.02657386288046837,
        0.05039682611823082,
        0.03066091425716877,
        0.0630461722612381
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.014765195548534393,
        "first_to_last_ratio": 0.34603674681838265,
        "imbalance_abs_log_ratio": 1.0612103048863237,
        "last_quartile_mean": 0.042669443879276514,
        "mean": 0.02649195003323257,
        "normalized": [
          0.3612463015808142,
          0.6598023683153774,
          0.42591864153205383,
          0.7824186763925585,
          0.46448619812724606,
          0.8730968737347248,
          0.5739659536307038,
          1.0658747527147492,
          0.6518477223197376,
          1.2549881523940432,
          0.8155372895827101,
          1.628189203361138,
          1.0030919901001265,
          1.9023449030747457,
          1.1573672084805566,
          2.3798237646587146
        ],
        "population_cv": 0.544699846184418
      },
      "bits_per_byte": 5.090461856197109,
      "branch_output_rms_by_sublayer": [
        0.004518123809248209,
        0.008520865812897682,
        0.004510615952312946,
        0.009082728065550327,
        0.005324283614754677,
        0.0099419467151165,
        0.004656728357076645,
        0.010662632063031197,
        0.00614966033026576,
        0.010214399546384811,
        0.0060766469687223434,
        0.011413711123168468,
        0.0087440712377429,
        0.011133860796689987,
        0.009014265611767769,
        0.011885596439242363,
        0.010757530108094215,
        0.011522865854203701,
        0.011306028813123703,
        0.012577465735375881,
        0.014357471838593483,
        0.01213979721069336,
        0.01672077551484108,
        0.013535024598240852,
        0.01905498467385769,
        0.012188552878797054,
        0.018409499898552895,
        0.013168000616133213,
        0.021850280463695526,
        0.012649565935134888,
        0.02609730139374733,
        0.013150178827345371
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00025280350140969286,
        0.00022135664689195577,
        0.00022214031151724322,
        0.0001810293778015987,
        0.00018338803861517842,
        0.00016050318916432303,
        0.0001626860438404638,
        0.00015310192441761306,
        0.00014075054270255738,
        0.00013076915091262195,
        0.00013639133469956174,
        0.0001377615870880476,
        0.00013681686670328103,
        0.0001370125878547912,
        0.0001387461931789955,
        0.0001464231652858264
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00021933245940512265,
        "first_to_last_ratio": 1.5694663694832551,
        "imbalance_abs_log_ratio": 0.45073566953494,
        "last_quartile_mean": 0.00013974970325572353,
        "mean": 0.0001651050288802345,
        "normalized": [
          1.531167785283354,
          1.3407020270262369,
          1.3454484882976008,
          1.0964498115494445,
          1.1107356320939579,
          0.9721278040582907,
          0.9853488106560013,
          0.9273001885888748,
          0.8524909486836794,
          0.7920361469273026,
          0.8260883125401264,
          0.8343875896595401,
          0.8286656538034743,
          0.8298510880257848,
          0.8403511032946221,
          0.8868486095117084
        ],
        "population_cv": 0.21861370307121916
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.6929093599319458,
          "mean_weights": [
            0.4913179874420166,
            0.5086820125579834
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6919252872467041,
          "mean_weights": [
            0.5235024690628052,
            0.4764975309371948
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6930201053619385,
          "mean_weights": [
            0.4991857707500458,
            0.5008142590522766
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6926943063735962,
          "mean_weights": [
            0.5129889249801636,
            0.48701101541519165
          ],
          "sources": 2
        },
        {
          "entropy_mean": 1.097741961479187,
          "mean_weights": [
            0.3480756878852844,
            0.33060726523399353,
            0.32131701707839966
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0982271432876587,
          "mean_weights": [
            0.3435906767845154,
            0.327891081571579,
            0.328518271446228
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.097235918045044,
          "mean_weights": [
            0.35132721066474915,
            0.33742234110832214,
            0.3112504482269287
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.098421573638916,
          "mean_weights": [
            0.340750515460968,
            0.3319299519062042,
            0.32731953263282776
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.3854012489318848,
          "mean_weights": [
            0.26307183504104614,
            0.2547699213027954,
            0.2451326996088028,
            0.23702551424503326
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3862006664276123,
          "mean_weights": [
            0.25451934337615967,
            0.25035756826400757,
            0.24778643250465393,
            0.24733665585517883
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3854570388793945,
          "mean_weights": [
            0.26253488659858704,
            0.2546469569206238,
            0.2452046275138855,
            0.2376135289669037
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3861933946609497,
          "mean_weights": [
            0.2505362629890442,
            0.254558801651001,
            0.2493266463279724,
            0.24557828903198242
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.6085797548294067,
          "mean_weights": [
            0.2100008726119995,
            0.20838744938373566,
            0.19692982733249664,
            0.19323000311851501,
            0.19145184755325317
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6093010902404785,
          "mean_weights": [
            0.19938474893569946,
            0.20374906063079834,
            0.20229801535606384,
            0.19861868023872375,
            0.1959494650363922
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6085379123687744,
          "mean_weights": [
            0.20589761435985565,
            0.21123698353767395,
            0.20005744695663452,
            0.1941205859184265,
            0.18868738412857056
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6093740463256836,
          "mean_weights": [
            0.19956088066101074,
            0.2027459442615509,
            0.20118315517902374,
            0.1998024433851242,
            0.1967075765132904
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.789860725402832,
          "mean_weights": [
            0.17310243844985962,
            0.1810513734817505,
            0.17217648029327393,
            0.1649792194366455,
            0.1559385061264038,
            0.15275199711322784
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791734218597412,
          "mean_weights": [
            0.16535353660583496,
            0.16847901046276093,
            0.1662793755531311,
            0.16693073511123657,
            0.16613470017910004,
            0.1668226420879364
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7890782356262207,
          "mean_weights": [
            0.17188119888305664,
            0.18350960314273834,
            0.17409591376781464,
            0.16590778529644012,
            0.15644267201423645,
            0.148162841796875
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.791684627532959,
          "mean_weights": [
            0.16539111733436584,
            0.16517111659049988,
            0.16458021104335785,
            0.16693061590194702,
            0.16811248660087585,
            0.16981446743011475
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.9417986869812012,
          "mean_weights": [
            0.14831508696079254,
            0.16164575517177582,
            0.15476199984550476,
            0.14618372917175293,
            0.13434207439422607,
            0.1278611123561859,
            0.12689024209976196
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9456918239593506,
          "mean_weights": [
            0.14120136201381683,
            0.1386352777481079,
            0.14049293100833893,
            0.1421712040901184,
            0.14506074786186218,
            0.1458473801612854,
            0.14659109711647034
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9406665563583374,
          "mean_weights": [
            0.14782890677452087,
            0.1637636125087738,
            0.15569329261779785,
            0.14784488081932068,
            0.13445764780044556,
            0.12887068092823029,
            0.12154098600149155
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9455351829528809,
          "mean_weights": [
            0.14214086532592773,
            0.1382882297039032,
            0.1401882916688919,
            0.1396511197090149,
            0.14402973651885986,
            0.14592906832695007,
            0.14977267384529114
          ],
          "sources": 7
        },
        {
          "entropy_mean": 2.0713937282562256,
          "mean_weights": [
            0.1321096122264862,
            0.1488187313079834,
            0.13977529108524323,
            0.13292929530143738,
            0.12093928456306458,
            0.11470195651054382,
            0.10486428439617157,
            0.10586154460906982
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0790562629699707,
          "mean_weights": [
            0.12411445379257202,
            0.1212267130613327,
            0.12224400043487549,
            0.12172792106866837,
            0.12399555742740631,
            0.12670958042144775,
            0.1301731914281845,
            0.12980858981609344
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.068070888519287,
          "mean_weights": [
            0.13267414271831512,
            0.15216735005378723,
            0.1437777280807495,
            0.13539820909500122,
            0.1199566125869751,
            0.11211173236370087,
            0.10534092038869858,
            0.09857328981161118
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.079178810119629,
          "mean_weights": [
            0.12415283918380737,
            0.12161671370267868,
            0.12301617860794067,
            0.12307952344417572,
            0.12411122024059296,
            0.12585631012916565,
            0.12885767221450806,
            0.1293095499277115
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.1886847019195557,
          "mean_weights": [
            0.11892519891262054,
            0.13333868980407715,
            0.12725916504859924,
            0.12129361927509308,
            0.10894724726676941,
            0.10413059592247009,
            0.09661625325679779,
            0.09465864300727844,
            0.09483058750629425
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1969594955444336,
          "mean_weights": [
            0.1101110577583313,
            0.10734203457832336,
            0.10833269357681274,
            0.10946638882160187,
            0.11091698706150055,
            0.11324633657932281,
            0.11330728977918625,
            0.11495929211378098,
            0.11231791973114014
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.186551570892334,
          "mean_weights": [
            0.11992302536964417,
            0.13721907138824463,
            0.12899263203144073,
            0.12160105258226395,
            0.10788539052009583,
            0.10353681445121765,
            0.09616796672344208,
            0.09295656532049179,
            0.09171747416257858
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.02856776863336563,
        0.014094225130975246,
        0.01573058031499386,
        0.015321774408221245,
        0.017058780416846275,
        0.011773223988711834,
        0.013085381127893925,
        0.013459180481731892,
        0.015175317414104939,
        0.011872655712068081,
        0.013221408240497112,
        0.013565259985625744,
        0.015276176854968071,
        0.012822598218917847,
        0.014198821038007736,
        0.014604844152927399,
        0.0162027720361948,
        0.014113145880401134,
        0.015384308062493801,
        0.015927067026495934,
        0.017611440271139145,
        0.01576142944395542,
        0.017285119742155075,
        0.01772329770028591,
        0.019923174753785133,
        0.017852960154414177,
        0.019824320450425148,
        0.01966804265975952,
        0.022481828927993774,
        0.020090021193027496,
        0.02255004271864891,
        0.022322168573737144
      ],
      "loss_nats": 3.5284392833709717,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 2.193685531616211,
        "mean_weights": [
          0.10035791993141174,
          0.09579961001873016,
          0.10334210842847824,
          0.108812615275383,
          0.11718772351741791,
          0.11711248755455017,
          0.12047697603702545,
          0.11920724809169769,
          0.11770330369472504
        ],
        "sources": 9
      },
      "step": 100,
      "stream_state_rms_by_sublayer": [
        0.004518123809248209,
        0.009570118971168995,
        0.010795549489557743,
        0.01747945137321949,
        0.005324283614754677,
        0.011283415369689465,
        0.012478979304432869,
        0.020727796480059624,
        0.00614966033026576,
        0.012305145151913166,
        0.014650273136794567,
        0.023130038753151894,
        0.0087440712377429,
        0.015205477364361286,
        0.019528985023498535,
        0.028237100690603256,
        0.010757530108094215,
        0.017268717288970947,
        0.02412688545882702,
        0.03324708342552185,
        0.014357471838593483,
        0.021605173125863075,
        0.03339937701821327,
        0.04313390702009201,
        0.01905498467385769,
        0.02657386288046837,
        0.04070562124252319,
        0.05039682611823082,
        0.021850280463695526,
        0.03066091425716877,
        0.052838243544101715,
        0.0630461722612381
      ]
    },
    {
      "activation_grad_rms_by_block": [
        0.00015504666953347623,
        0.00015570323739666492,
        0.00013642148405779153,
        0.00012308824807405472,
        0.00010802906763274223,
        9.803942521102726e-05,
        8.508376777172089e-05,
        7.592589827254415e-05,
        7.549385918537155e-05,
        6.5492422436364e-05,
        6.254414620343596e-05,
        5.1708771934499964e-05,
        4.589578747982159e-05,
        4.085309410584159e-05,
        3.8330261304508895e-05,
        3.7662506656488404e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 0.00014256490976549685,
        "first_to_last_ratio": 3.504079260905398,
        "imbalance_abs_log_ratio": 1.2539277929397175,
        "last_quartile_mean": 4.068541238666512e-05,
        "mean": 8.470741545352212e-05,
        "normalized": [
          1.8303789426623265,
          1.8381299507608906,
          1.610502260367562,
          1.4530988510869118,
          1.2753200774023898,
          1.1573889332607517,
          1.0044429677872213,
          0.8963311873705286,
          0.8912308182368528,
          0.7731604380291695,
          0.7383550291149319,
          0.6104397313700582,
          0.5418154624845569,
          0.4822847431610892,
          0.45250183941145294,
          0.44461876749330576
        ],
        "population_cv": 0.46442228143345987
      },
      "activation_output_rms_by_block": [
        0.02604864165186882,
        0.04358876869082451,
        0.022780675441026688,
        0.03670342266559601,
        0.027025330811738968,
        0.05019363388419151,
        0.040244098752737045,
        0.04879867285490036,
        0.029798155650496483,
        0.04518449306488037,
        0.02838377095758915,
        0.051337067037820816,
        0.03289651870727539,
        0.0653979703783989,
        0.04346369579434395,
        0.08895192295312881
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.032280377112329006,
        "first_to_last_ratio": 0.5596699237065876,
        "imbalance_abs_log_ratio": 0.5804080909828722,
        "last_quartile_mean": 0.05767752695828676,
        "mean": 0.04254980245605111,
        "normalized": [
          0.6121918351741814,
          1.0244176511946566,
          0.5353885124274412,
          0.8625991320055194,
          0.6351458585419503,
          1.1796443458470947,
          0.9458116472880084,
          1.1468601506506073,
          0.7003124322674456,
          1.0619201607704427,
          0.6670717446198772,
          1.2065171651700595,
          0.7731297634402319,
          1.5369747120670472,
          1.0214782040230805,
          2.0905366845123567
        ],
        "population_cv": 0.38343105399742256
      },
      "bits_per_byte": 3.1146838872489564,
      "branch_output_rms_by_sublayer": [
        0.01352347619831562,
        0.020455393940210342,
        0.016984812915325165,
        0.0142062958329916,
        0.01867547444999218,
        0.011138904839754105,
        0.019149528816342354,
        0.0099713783711195,
        0.023481043055653572,
        0.011000796221196651,
        0.02560650371015072,
        0.010696547105908394,
        0.0353584885597229,
        0.012510747648775578,
        0.023770872503519058,
        0.012255984358489513,
        0.023732667788863182,
        0.014518127776682377,
        0.021041536703705788,
        0.015058746561408043,
        0.021396929398179054,
        0.018767673522233963,
        0.022012433037161827,
        0.023688357323408127,
        0.020202528685331345,
        0.02677428349852562,
        0.015811076387763023,
        0.03653915971517563,
        0.014067054726183414,
        0.0391332171857357,
        0.010850721038877964,
        0.043844256550073624
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0006567095640112879,
        0.00045868421249221203,
        0.00039439577454117915,
        0.0003702348543320087,
        0.0003691077163774564,
        0.00041151853265381964,
        0.0004306774133910072,
        0.00035317417015185023,
        0.00038561746855580054,
        0.0003833308661148803,
        0.00032325470855348767,
        0.0002967607909467712,
        0.00021968458301266372,
        0.00017220390788738176,
        0.00016255203232502606,
        0.0001597780110258732
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00047000610134417196,
        "first_to_last_ratio": 2.6322817390176136,
        "imbalance_abs_log_ratio": 0.967851051434004,
        "last_quartile_mean": 0.0001785546335627362,
        "mean": 0.00034673028789829414,
        "normalized": [
          1.8940069181493584,
          1.3228847565424027,
          1.137471366957324,
          1.0677891930819992,
          1.0645384301867686,
          1.1868548754371575,
          1.242110736854166,
          1.0185847111673332,
          1.1121539767789572,
          1.105559218487778,
          0.9322944081778846,
          0.8558836689623711,
          0.6335892498583898,
          0.49665089522809175,
          0.46881405518489694,
          0.46081353894511995
        ],
        "population_cv": 0.3565995806724142
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.6422993540763855,
          "mean_weights": [
            0.370745986700058,
            0.6292540431022644
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6814000606536865,
          "mean_weights": [
            0.4577118158340454,
            0.5422881841659546
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6871446371078491,
          "mean_weights": [
            0.49052417278289795,
            0.5094758868217468
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6793373227119446,
          "mean_weights": [
            0.4676857590675354,
            0.5323142409324646
          ],
          "sources": 2
        },
        {
          "entropy_mean": 1.0915337800979614,
          "mean_weights": [
            0.29751554131507874,
            0.351987361907959,
            0.3504970669746399
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0892891883850098,
          "mean_weights": [
            0.3054355978965759,
            0.34222692251205444,
            0.35233747959136963
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.093106746673584,
          "mean_weights": [
            0.30644628405570984,
            0.34674984216690063,
            0.3468038737773895
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0885257720947266,
          "mean_weights": [
            0.3052448034286499,
            0.3325393795967102,
            0.3622158169746399
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.381313443183899,
          "mean_weights": [
            0.2415536344051361,
            0.24482353031635284,
            0.25347256660461426,
            0.2601502537727356
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3802430629730225,
          "mean_weights": [
            0.24572397768497467,
            0.2491092085838318,
            0.25083041191101074,
            0.254336416721344
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3813848495483398,
          "mean_weights": [
            0.2464662790298462,
            0.23372246325016022,
            0.2595708668231964,
            0.260240375995636
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3772881031036377,
          "mean_weights": [
            0.24203142523765564,
            0.2443695217370987,
            0.2359469085931778,
            0.27765214443206787
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.6055738925933838,
          "mean_weights": [
            0.20364060997962952,
            0.18423235416412354,
            0.18416231870651245,
            0.20781230926513672,
            0.22015240788459778
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.602186679840088,
          "mean_weights": [
            0.19954648613929749,
            0.1888481080532074,
            0.18637022376060486,
            0.21049518883228302,
            0.21474000811576843
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6062731742858887,
          "mean_weights": [
            0.19444647431373596,
            0.1968603879213333,
            0.18490161001682281,
            0.20200669765472412,
            0.2217848002910614
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.6028468608856201,
          "mean_weights": [
            0.20196445286273956,
            0.19006873667240143,
            0.1860634982585907,
            0.1993759572505951,
            0.22252735495567322
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.7881723642349243,
          "mean_weights": [
            0.15337401628494263,
            0.17425060272216797,
            0.1601826399564743,
            0.1535373479127884,
            0.17563557624816895,
            0.18301980197429657
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7856398820877075,
          "mean_weights": [
            0.1621149629354477,
            0.15582670271396637,
            0.15492847561836243,
            0.1618887484073639,
            0.1784553974866867,
            0.1867857277393341
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7870616912841797,
          "mean_weights": [
            0.1564463973045349,
            0.18363508582115173,
            0.15811879932880402,
            0.15924012660980225,
            0.15980733931064606,
            0.18275225162506104
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7877882719039917,
          "mean_weights": [
            0.17034202814102173,
            0.15881556272506714,
            0.1534808874130249,
            0.16709929704666138,
            0.16602207720279694,
            0.1842401623725891
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.9344044923782349,
          "mean_weights": [
            0.12371677905321121,
            0.1564711332321167,
            0.13837161660194397,
            0.1230623871088028,
            0.12980599701404572,
            0.152274027466774,
            0.17629806697368622
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9417431354522705,
          "mean_weights": [
            0.14328667521476746,
            0.1450044810771942,
            0.14678549766540527,
            0.13414078950881958,
            0.1328907310962677,
            0.1347782164812088,
            0.16311360895633698
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.9288355112075806,
          "mean_weights": [
            0.12207633256912231,
            0.16755375266075134,
            0.13929706811904907,
            0.12109649926424026,
            0.12036697566509247,
            0.15116539597511292,
            0.1784439980983734
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.938415288925171,
          "mean_weights": [
            0.1377446949481964,
            0.1613207459449768,
            0.156096950173378,
            0.13170479238033295,
            0.12767548859119415,
            0.13129547238349915,
            0.15416184067726135
          ],
          "sources": 7
        },
        {
          "entropy_mean": 2.0616660118103027,
          "mean_weights": [
            0.10379506647586823,
            0.1403447985649109,
            0.12122530490159988,
            0.10852085053920746,
            0.10464349389076233,
            0.1186104342341423,
            0.13291975855827332,
            0.1699402928352356
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.0691847801208496,
          "mean_weights": [
            0.11521603912115097,
            0.14423911273479462,
            0.13052678108215332,
            0.12005230039358139,
            0.10937757790088654,
            0.12122473865747452,
            0.12272199243307114,
            0.1366414725780487
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.054232597351074,
          "mean_weights": [
            0.10057584941387177,
            0.16718831658363342,
            0.1352493166923523,
            0.11277885735034943,
            0.09778623282909393,
            0.11311028897762299,
            0.13487182557582855,
            0.1384393274784088
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.060153007507324,
          "mean_weights": [
            0.11611463129520416,
            0.15179042518138885,
            0.14188171923160553,
            0.1159345731139183,
            0.10906632244586945,
            0.12251802533864975,
            0.12206383049488068,
            0.12063045799732208
          ],
          "sources": 8
        },
        {
          "entropy_mean": 2.1772594451904297,
          "mean_weights": [
            0.09167024493217468,
            0.14425000548362732,
            0.1316833794116974,
            0.09630900621414185,
            0.09528810530900955,
            0.1022176742553711,
            0.12638677656650543,
            0.10320886224508286,
            0.10898594558238983
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.191457986831665,
          "mean_weights": [
            0.10587693750858307,
            0.12391996383666992,
            0.11278913170099258,
            0.11095453053712845,
            0.10569566488265991,
            0.11266011744737625,
            0.11442379653453827,
            0.11124123632907867,
            0.10243862122297287
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.1550254821777344,
          "mean_weights": [
            0.08980270475149155,
            0.17290523648262024,
            0.13377593457698822,
            0.10608047246932983,
            0.09299036115407944,
            0.1071130782365799,
            0.12428411841392517,
            0.09728670865297318,
            0.07576138526201248
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.04350041598081589,
        0.017946822568774223,
        0.025110680609941483,
        0.028106264770030975,
        0.031104078516364098,
        0.02212802693247795,
        0.02379882149398327,
        0.02537565305829048,
        0.026546044275164604,
        0.02055767923593521,
        0.022140897810459137,
        0.02340756356716156,
        0.02546003647148609,
        0.020307816565036774,
        0.02165406011044979,
        0.021843569353222847,
        0.023044398054480553,
        0.018649185076355934,
        0.019831404089927673,
        0.019260063767433167,
        0.02048473060131073,
        0.01540753897279501,
        0.01695212349295616,
        0.01553493645042181,
        0.017758244648575783,
        0.013987960293889046,
        0.016796667128801346,
        0.01665738970041275,
        0.01972099393606186,
        0.01747863180935383,
        0.020752927288413048,
        0.019990377128124237
      ],
      "loss_nats": 2.1589343547821045,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 2.1796395778656006,
        "mean_weights": [
          0.12411575019359589,
          0.09159804880619049,
          0.10466736555099487,
          0.10315616428852081,
          0.09333926439285278,
          0.09817550331354141,
          0.11351706087589264,
          0.1361408531665802,
          0.1352899968624115
        ],
        "sources": 9
      },
      "step": 500,
      "stream_state_rms_by_sublayer": [
        0.01352347619831562,
        0.02604864165186882,
        0.03525395691394806,
        0.04358876869082451,
        0.01867547444999218,
        0.022780675441026688,
        0.03279132395982742,
        0.03670342266559601,
        0.023481043055653572,
        0.027025330811738968,
        0.04530210420489311,
        0.05019363388419151,
        0.0353584885597229,
        0.040244098752737045,
        0.04290904477238655,
        0.04879867285490036,
        0.023732667788863182,
        0.029798155650496483,
        0.03790274262428284,
        0.04518449306488037,
        0.021396929398179054,
        0.02838377095758915,
        0.03953101858496666,
        0.051337067037820816,
        0.020202528685331345,
        0.03289651870727539,
        0.039050742983818054,
        0.0653979703783989,
        0.014067054726183414,
        0.04346369579434395,
        0.048713527619838715,
        0.08895192295312881
      ]
    },
    {
      "activation_grad_rms_by_block": [
        7.91841812315397e-05,
        8.04664014140144e-05,
        6.887180643389001e-05,
        6.264931289479136e-05,
        6.203571683727205e-05,
        6.256609776755795e-05,
        7.726731564616784e-05,
        7.684857700951397e-05,
        0.0001297412090934813,
        0.00010362039029132575,
        0.0001698000414762646,
        0.00012187293032184243,
        0.00013834609126206487,
        8.384700049646199e-05,
        4.917434125673026e-05,
        4.311763041187078e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 7.279292549355887e-05,
        "first_to_last_ratio": 0.9258681439467011,
        "imbalance_abs_log_ratio": 0.07702344762093526,
        "last_quartile_mean": 7.862126585678197e-05,
        "mean": 8.808806524029933e-05,
        "normalized": [
          0.8989206541831708,
          0.91347677116652,
          0.7818517326497246,
          0.7112122706281212,
          0.7042465590320558,
          0.7102675895637067,
          0.8771598676323168,
          0.8724062311946045,
          1.4728579716168646,
          1.1763272358026606,
          1.9276168799151117,
          1.3835351019389504,
          1.570542966117651,
          0.9518542638861694,
          0.5582406779236824,
          0.4894832267486894
        ],
        "population_cv": 0.3870844002085187
      },
      "activation_output_rms_by_block": [
        0.10205306112766266,
        0.1637924462556839,
        0.1347782015800476,
        0.18643726408481598,
        0.08823998272418976,
        0.14544272422790527,
        0.051831819117069244,
        0.08038940280675888,
        0.045988935977220535,
        0.08743251860141754,
        0.060310039669275284,
        0.09679152816534042,
        0.05415637046098709,
        0.0906868502497673,
        0.09274918586015701,
        0.19966518878936768
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.14676524326205254,
        "first_to_last_ratio": 1.3425975426784762,
        "imbalance_abs_log_ratio": 0.2946062022500781,
        "last_quartile_mean": 0.10931439884006977,
        "mean": 0.10504659498110414,
        "normalized": [
          0.9715027997434856,
          1.5592361302634037,
          1.2830325590685887,
          1.7748054005782146,
          0.840008024439654,
          1.3845543899263715,
          0.49341741278137374,
          0.7652737608603057,
          0.4377955895238019,
          0.8323213010107083,
          0.5741265547933648,
          0.9214151889716308,
          0.5155461771105363,
          0.8633011880687815,
          0.8829337674090323,
          1.9007297554507465
        ],
        "population_cv": 0.43941183630186603
      },
      "bits_per_byte": 2.0898630043632984,
      "branch_output_rms_by_sublayer": [
        0.04098131135106087,
        0.09075035154819489,
        0.05343177169561386,
        0.04652838781476021,
        0.13196392357349396,
        0.018610643222928047,
        0.06760989129543304,
        0.01781350187957287,
        0.0839577317237854,
        0.0164200272411108,
        0.07349737733602524,
        0.016965076327323914,
        0.04464862495660782,
        0.016964683309197426,
        0.0346895232796669,
        0.021388454362750053,
        0.03376077860593796,
        0.026654686778783798,
        0.04212235286831856,
        0.042033009231090546,
        0.0419253334403038,
        0.040410157293081284,
        0.028147248551249504,
        0.05647630989551544,
        0.025301706045866013,
        0.04489820450544357,
        0.015376480296254158,
        0.058749374002218246,
        0.025178087875247,
        0.0875716358423233,
        0.01239082682877779,
        0.1223064437508583
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00040335839120789034,
        0.00023897821552144955,
        0.00022638555923562337,
        0.00017534070905568056,
        0.00017358160175308344,
        0.00015895432307312814,
        0.00019058997613466408,
        0.00021161798453125184,
        0.00037807554491806804,
        0.0004740713531015684,
        0.0009894587672613622,
        0.0007545475239678793,
        0.0006624146206615605,
        0.0004575991184432586,
        0.0002840295434450018,
        0.00025831777406547436
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00026101571875516093,
        "first_to_last_ratio": 0.6280602344874717,
        "imbalance_abs_log_ratio": 0.46511920232449994,
        "last_quartile_mean": 0.00041559026415382384,
        "mean": 0.00037733256289855906,
        "normalized": [
          1.0689731840512475,
          0.633335786568984,
          0.5999629544203535,
          0.464684806709403,
          0.460022852042453,
          0.4212579000658921,
          0.5050981412009801,
          0.5608261924326488,
          1.001969037640965,
          1.2563754091613246,
          2.6222459033501577,
          1.9996883337384523,
          1.7555193635372572,
          1.2127209878949095,
          0.7527300089426935,
          0.6845891382422771
        ],
        "population_cv": 0.6186471183066214
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.4914427399635315,
          "mean_weights": [
            0.7962706685066223,
            0.2037293016910553
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6456852555274963,
          "mean_weights": [
            0.6051734089851379,
            0.39482662081718445
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.5276303291320801,
          "mean_weights": [
            0.7698293924331665,
            0.2301705777645111
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.667390763759613,
          "mean_weights": [
            0.5093090534210205,
            0.4906909465789795
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.9880077838897705,
          "mean_weights": [
            0.5180364847183228,
            0.2465059459209442,
            0.23545756936073303
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.08414626121521,
          "mean_weights": [
            0.2851155400276184,
            0.3203214704990387,
            0.3945629596710205
          ],
          "sources": 3
        },
        {
          "entropy_mean": 0.995106041431427,
          "mean_weights": [
            0.5032885074615479,
            0.23357318341732025,
            0.2631382942199707
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0889081954956055,
          "mean_weights": [
            0.2929039001464844,
            0.3457888960838318,
            0.36130720376968384
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.3091163635253906,
          "mean_weights": [
            0.3480933904647827,
            0.15371239185333252,
            0.20259280502796173,
            0.29560136795043945
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3786191940307617,
          "mean_weights": [
            0.22527548670768738,
            0.26118987798690796,
            0.22951865196228027,
            0.284015953540802
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.308091163635254,
          "mean_weights": [
            0.28280675411224365,
            0.12650004029273987,
            0.24999231100082397,
            0.3407008647918701
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3780848979949951,
          "mean_weights": [
            0.23603014647960663,
            0.2631712555885315,
            0.21569286286830902,
            0.28510570526123047
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.5637545585632324,
          "mean_weights": [
            0.21651385724544525,
            0.1117275133728981,
            0.18637539446353912,
            0.22181779146194458,
            0.2635654807090759
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.585724115371704,
          "mean_weights": [
            0.18968582153320312,
            0.1962759643793106,
            0.1558789312839508,
            0.1798120141029358,
            0.2783472537994385
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5276048183441162,
          "mean_weights": [
            0.2346682846546173,
            0.15222865343093872,
            0.12689977884292603,
            0.14891111850738525,
            0.3372921645641327
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.59938383102417,
          "mean_weights": [
            0.20912373065948486,
            0.21754522621631622,
            0.17016828060150146,
            0.17554700374603271,
            0.22761575877666473
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.762988805770874,
          "mean_weights": [
            0.15954776108264923,
            0.12860070168972015,
            0.13993129134178162,
            0.14435188472270966,
            0.1894117146730423,
            0.23815666139125824
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.730720043182373,
          "mean_weights": [
            0.17182672023773193,
            0.16310763359069824,
            0.12390270084142685,
            0.11688461899757385,
            0.1326894462108612,
            0.2915888726711273
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.4424301385879517,
          "mean_weights": [
            0.1671508103609085,
            0.1365666687488556,
            0.058271970599889755,
            0.05150005966424942,
            0.10222888737916946,
            0.48428159952163696
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7727878093719482,
          "mean_weights": [
            0.1808200478553772,
            0.17958280444145203,
            0.1469326913356781,
            0.13797307014465332,
            0.14092227816581726,
            0.2137690931558609
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.5771737098693848,
          "mean_weights": [
            0.09523316472768784,
            0.08390435576438904,
            0.04673727601766586,
            0.04574791342020035,
            0.07810044288635254,
            0.19550111889839172,
            0.45477572083473206
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.866133689880371,
          "mean_weights": [
            0.13646650314331055,
            0.12685352563858032,
            0.11049237847328186,
            0.10180146992206573,
            0.1026906818151474,
            0.1334708034992218,
            0.28822463750839233
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.5755802392959595,
          "mean_weights": [
            0.09840903431177139,
            0.08739377558231354,
            0.04908968508243561,
            0.048939045518636703,
            0.07940414547920227,
            0.16869762539863586,
            0.46806666254997253
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.8938008546829224,
          "mean_weights": [
            0.14332836866378784,
            0.14485859870910645,
            0.1094474121928215,
            0.10118386149406433,
            0.10821370780467987,
            0.14364898204803467,
            0.24931909143924713
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.7736961841583252,
          "mean_weights": [
            0.08116129040718079,
            0.07111318409442902,
            0.04280973970890045,
            0.041376810520887375,
            0.06107718497514725,
            0.1160595640540123,
            0.31332647800445557,
            0.2730756998062134
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.9035927057266235,
          "mean_weights": [
            0.10087604820728302,
            0.09542917460203171,
            0.06220542639493942,
            0.057604290544986725,
            0.0788167417049408,
            0.11659102141857147,
            0.18421337008476257,
            0.3042638897895813
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.7131891250610352,
          "mean_weights": [
            0.0740746557712555,
            0.07079474627971649,
            0.04342404007911682,
            0.04123558849096298,
            0.055328819900751114,
            0.09829520434141159,
            0.21916092932224274,
            0.39768603444099426
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.9703106880187988,
          "mean_weights": [
            0.11310406029224396,
            0.11765976250171661,
            0.07340675592422485,
            0.0660770833492279,
            0.08407266438007355,
            0.12691649794578552,
            0.20392963290214539,
            0.2148335576057434
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.9610494375228882,
          "mean_weights": [
            0.07792562991380692,
            0.0730937123298645,
            0.040804408490657806,
            0.03727761656045914,
            0.05315954610705376,
            0.0937754213809967,
            0.19250980019569397,
            0.26475197076797485,
            0.16670189797878265
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.9782097339630127,
          "mean_weights": [
            0.08397573232650757,
            0.08872128278017044,
            0.0407399907708168,
            0.036753468215465546,
            0.058452628552913666,
            0.11618129909038544,
            0.22768989205360413,
            0.21488600969314575,
            0.13259968161582947
          ],
          "sources": 9
        },
        {
          "entropy_mean": 2.0315680503845215,
          "mean_weights": [
            0.08375388383865356,
            0.10381080955266953,
            0.0493716336786747,
            0.043391548097133636,
            0.06576120853424072,
            0.10571172833442688,
            0.18509769439697266,
            0.2181759774684906,
            0.14492549002170563
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.05570797994732857,
        0.04578034579753876,
        0.058472853153944016,
        0.058589257299900055,
        0.09159071743488312,
        0.0729803666472435,
        0.0950615182518959,
        0.08442825824022293,
        0.11022263020277023,
        0.07856440544128418,
        0.0938955619931221,
        0.10307082533836365,
        0.10586344450712204,
        0.08232943713665009,
        0.08149705082178116,
        0.07412237673997879,
        0.0869104191660881,
        0.0654435008764267,
        0.0601484477519989,
        0.04590349644422531,
        0.06966738402843475,
        0.028740333393216133,
        0.04708963632583618,
        0.03821079805493355,
        0.0513334795832634,
        0.034749191254377365,
        0.034582361578941345,
        0.034875962883234024,
        0.041408468037843704,
        0.03493491932749748,
        0.0412786528468132,
        0.04175883159041405
      ],
      "loss_nats": 1.448582649230957,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 1.893564224243164,
        "mean_weights": [
          0.09584413468837738,
          0.060246460139751434,
          0.05342919006943703,
          0.050279051065444946,
          0.05668472498655319,
          0.06780324131250381,
          0.0837026908993721,
          0.2589869201183319,
          0.2730236053466797
        ],
        "sources": 9
      },
      "step": 2000,
      "stream_state_rms_by_sublayer": [
        0.04098131135106087,
        0.10205306112766266,
        0.13058459758758545,
        0.1637924462556839,
        0.13196392357349396,
        0.1347782015800476,
        0.18347561359405518,
        0.18643726408481598,
        0.0839577317237854,
        0.08823998272418976,
        0.14121752977371216,
        0.14544272422790527,
        0.04464862495660782,
        0.051831819117069244,
        0.07186394184827805,
        0.08038940280675888,
        0.03376077860593796,
        0.045988935977220535,
        0.06799308955669403,
        0.08743251860141754,
        0.0419253334403038,
        0.060310039669275284,
        0.07043248414993286,
        0.09679152816534042,
        0.025301706045866013,
        0.05415637046098709,
        0.05762659013271332,
        0.0906868502497673,
        0.025178087875247,
        0.09274918586015701,
        0.0948956087231636,
        0.19966518878936768
      ]
    },
    {
      "activation_grad_rms_by_block": [
        5.6704131566220894e-05,
        5.7463053963147104e-05,
        5.5973152484511957e-05,
        4.9604332161834463e-05,
        4.9622754886513576e-05,
        5.175438491278328e-05,
        6.153700815048069e-05,
        5.993711238261312e-05,
        0.00010211551852989942,
        8.014418563107029e-05,
        0.00014815112808719277,
        0.00010670819756342098,
        0.00014386174734681845,
        8.734197035664693e-05,
        5.547845285036601e-05,
        4.4348533265292645e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 5.4936167543928605e-05,
        "first_to_last_ratio": 0.6638196023526066,
        "imbalance_abs_log_ratio": 0.4097448496221764,
        "last_quartile_mean": 8.275767595478101e-05,
        "mean": 7.567160400867579e-05,
        "normalized": [
          0.7493449135784109,
          0.7593740705768433,
          0.7396850273993744,
          0.6555210876215509,
          0.6557645438680578,
          0.6839340277080636,
          0.8132113618660104,
          0.792068744515331,
          1.3494562440911366,
          1.0591051515424692,
          1.9578166741411636,
          1.4101484825296797,
          1.9011325216566657,
          1.1542238531990563,
          0.7331475733487212,
          0.5860657223574653
        ],
        "population_cv": 0.42524285954419916
      },
      "activation_output_rms_by_block": [
        0.16565655171871185,
        0.24343863129615784,
        0.21182885766029358,
        0.28418517112731934,
        0.12311498820781708,
        0.18273214995861053,
        0.06521740555763245,
        0.10707039386034012,
        0.06429450213909149,
        0.1264011114835739,
        0.0799982100725174,
        0.12911832332611084,
        0.06644809991121292,
        0.1084718257188797,
        0.10412788391113281,
        0.23786598443984985
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.22627730295062065,
        "first_to_last_ratio": 1.750986764798193,
        "imbalance_abs_log_ratio": 0.5601794946219173,
        "last_quartile_mean": 0.12922844849526882,
        "mean": 0.14374813064932823,
        "normalized": [
          1.152408389385104,
          1.6935081534383452,
          1.4736112163924233,
          1.976966029705009,
          0.8564632294812553,
          1.2711967044940802,
          0.4536921994257406,
          0.7448472086328344,
          0.4472719182410596,
          0.8793235147657527,
          0.5565165245012613,
          0.8982261038308273,
          0.4622536627854468,
          0.7545964266032466,
          0.7243773080093228,
          1.654741410308291
        ],
        "population_cv": 0.4691623243633533
      },
      "bits_per_byte": 1.9440733112268958,
      "branch_output_rms_by_sublayer": [
        0.06033274158835411,
        0.15077131986618042,
        0.0633837878704071,
        0.06536085903644562,
        0.20718541741371155,
        0.025066526606678963,
        0.08910981565713882,
        0.0229792483150959,
        0.11816111207008362,
        0.01873566210269928,
        0.06798963993787766,
        0.025694282725453377,
        0.05406087636947632,
        0.025319905951619148,
        0.05011621490120888,
        0.03549748659133911,
        0.04759044572710991,
        0.038602542132139206,
        0.05848398804664612,
        0.0661846399307251,
        0.052182719111442566,
        0.057186078280210495,
        0.03716399520635605,
        0.08089981228113174,
        0.03087303601205349,
        0.055707309395074844,
        0.02278325706720352,
        0.07142244279384613,
        0.0293489471077919,
        0.09790552407503128,
        0.01774938404560089,
        0.15747468173503876
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.00039487297360913916,
        0.00021664788561845028,
        0.0001999236891396226,
        0.000155700993556381,
        0.00014909408964546676,
        0.0001300025003667294,
        0.0001780845905877945,
        0.00021758503566474802,
        0.0003627569164977402,
        0.0004938025952783663,
        0.0010348495424230683,
        0.0008427402552292494,
        0.0008322795274123101,
        0.0005665088497243569,
        0.00036612876328907985,
        0.0003194282437589591
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.00024178638548089825,
        "first_to_last_ratio": 0.464004453994026,
        "imbalance_abs_log_ratio": 0.7678611276768971,
        "last_quartile_mean": 0.0005210863460461766,
        "mean": 0.00040377540323759136,
        "normalized": [
          0.9779520259107666,
          0.5365554312652605,
          0.49513587884892185,
          0.3856128736617538,
          0.36925005448570164,
          0.3219673593892314,
          0.44104863535485134,
          0.5388763998997622,
          0.8984126164918598,
          1.2229635369537375,
          2.5629335866556917,
          2.087151046031797,
          2.061243752687373,
          1.4030296179061932,
          0.9067634143965939,
          0.7911037700605048
        ],
        "population_cv": 0.6734220009206653
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.2750788629055023,
          "mean_weights": [
            0.9180313944816589,
            0.08196862041950226
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.5785284042358398,
          "mean_weights": [
            0.7040213346481323,
            0.2959786057472229
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.4888499975204468,
          "mean_weights": [
            0.7967724800109863,
            0.20322750508785248
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.651995062828064,
          "mean_weights": [
            0.5629274845123291,
            0.4370725154876709
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.9925327301025391,
          "mean_weights": [
            0.4965394139289856,
            0.22532734274864197,
            0.27813321352005005
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0594394207000732,
          "mean_weights": [
            0.2700359523296356,
            0.27400293946266174,
            0.455961138010025
          ],
          "sources": 3
        },
        {
          "entropy_mean": 0.9841134548187256,
          "mean_weights": [
            0.49647465348243713,
            0.21238623559474945,
            0.2911390960216522
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0744149684906006,
          "mean_weights": [
            0.26788532733917236,
            0.3177185654640198,
            0.41439610719680786
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.307446002960205,
          "mean_weights": [
            0.33086591958999634,
            0.13986191153526306,
            0.23523268103599548,
            0.2940394878387451
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.376603603363037,
          "mean_weights": [
            0.21549805998802185,
            0.245200514793396,
            0.24268285930156708,
            0.29661858081817627
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.270529866218567,
          "mean_weights": [
            0.2375539094209671,
            0.10673755407333374,
            0.29245367646217346,
            0.3632548451423645
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3762142658233643,
          "mean_weights": [
            0.22679615020751953,
            0.24722415208816528,
            0.2286011129617691,
            0.2973785698413849
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.5536365509033203,
          "mean_weights": [
            0.20543108880519867,
            0.10043682903051376,
            0.22657056152820587,
            0.26188796758651733,
            0.20567354559898376
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5861575603485107,
          "mean_weights": [
            0.1991366446018219,
            0.19079142808914185,
            0.16120794415473938,
            0.1751076579093933,
            0.27375632524490356
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.4829623699188232,
          "mean_weights": [
            0.2499978393316269,
            0.13764306902885437,
            0.10969669371843338,
            0.13450437784194946,
            0.3681580424308777
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5994092226028442,
          "mean_weights": [
            0.2161637395620346,
            0.20309701561927795,
            0.17752361297607422,
            0.17640036344528198,
            0.22681528329849243
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.757448434829712,
          "mean_weights": [
            0.1632193624973297,
            0.11841119825839996,
            0.1406504064798355,
            0.1471654176712036,
            0.19204986095428467,
            0.23850372433662415
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.719214677810669,
          "mean_weights": [
            0.17631781101226807,
            0.15250492095947266,
            0.11967972666025162,
            0.11400137841701508,
            0.13607722520828247,
            0.3014189600944519
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.3921294212341309,
          "mean_weights": [
            0.18369951844215393,
            0.1281489133834839,
            0.04485193267464638,
            0.042511094361543655,
            0.10842494666576385,
            0.4923636019229889
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.769790768623352,
          "mean_weights": [
            0.1860106885433197,
            0.17010365426540375,
            0.1509144902229309,
            0.1396978795528412,
            0.13775232434272766,
            0.215520977973938
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.5072013139724731,
          "mean_weights": [
            0.09493429213762283,
            0.07214362919330597,
            0.03675283491611481,
            0.03880048543214798,
            0.07758605480194092,
            0.19188526272773743,
            0.48789745569229126
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.8300648927688599,
          "mean_weights": [
            0.13595148921012878,
            0.11550325155258179,
            0.1085444986820221,
            0.09914489090442657,
            0.09707978367805481,
            0.12102536857128143,
            0.3227507174015045
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.5086650848388672,
          "mean_weights": [
            0.10266698896884918,
            0.07744058966636658,
            0.03928206115961075,
            0.0407833606004715,
            0.07880876958370209,
            0.16221223771572113,
            0.4988059997558594
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.8771061897277832,
          "mean_weights": [
            0.1470566689968109,
            0.13075211644172668,
            0.10456536710262299,
            0.09934695065021515,
            0.1088101863861084,
            0.1398048996925354,
            0.26966381072998047
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.6780993938446045,
          "mean_weights": [
            0.08037982881069183,
            0.05826176330447197,
            0.03183364123106003,
            0.03276165574789047,
            0.053941305726766586,
            0.10113498568534851,
            0.30644336342811584,
            0.33524346351623535
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.8294007778167725,
          "mean_weights": [
            0.09819990396499634,
            0.08131108433008194,
            0.05535554885864258,
            0.052511729300022125,
            0.07309524714946747,
            0.1031290739774704,
            0.1768367737531662,
            0.35956063866615295
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.620237112045288,
          "mean_weights": [
            0.07192602753639221,
            0.0598294772207737,
            0.03551005944609642,
            0.03596649691462517,
            0.052555397152900696,
            0.08522239327430725,
            0.2067522257566452,
            0.45223796367645264
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.922456979751587,
          "mean_weights": [
            0.11432191729545593,
            0.10134892165660858,
            0.06334333121776581,
            0.05966266989707947,
            0.07981826364994049,
            0.11505833268165588,
            0.20316767692565918,
            0.26327887177467346
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.8568212985992432,
          "mean_weights": [
            0.07325166463851929,
            0.05971681326627731,
            0.03186998516321182,
            0.03226577863097191,
            0.04611574858427048,
            0.07693300396203995,
            0.17066776752471924,
            0.3028550446033478,
            0.20632418990135193
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.9784047603607178,
          "mean_weights": [
            0.08589711785316467,
            0.07969395816326141,
            0.03702075034379959,
            0.037476278841495514,
            0.0731067955493927,
            0.12099474668502808,
            0.22441011667251587,
            0.20659996569156647,
            0.1348002701997757
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.9687256813049316,
          "mean_weights": [
            0.07957327365875244,
            0.0893080085515976,
            0.03895219415426254,
            0.03689703717827797,
            0.0598301999270916,
            0.09126800298690796,
            0.16929352283477783,
            0.23949456214904785,
            0.1953832060098648
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.05895887315273285,
        0.05461842566728592,
        0.07232892513275146,
        0.07006148993968964,
        0.12035266309976578,
        0.10343272238969803,
        0.14036840200424194,
        0.12118023633956909,
        0.16676878929138184,
        0.11889537423849106,
        0.13824602961540222,
        0.15437477827072144,
        0.1504182070493698,
        0.12633125483989716,
        0.11159109324216843,
        0.09292281419038773,
        0.11971895396709442,
        0.08979442715644836,
        0.08137664943933487,
        0.06381704658269882,
        0.09776479750871658,
        0.03986503183841705,
        0.06484787911176682,
        0.05485111102461815,
        0.07206991314888,
        0.046776797622442245,
        0.04656966030597687,
        0.04702001437544823,
        0.055061157792806625,
        0.04538688063621521,
        0.05328623950481415,
        0.053778134286403656
      ],
      "loss_nats": 1.3475289344787598,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 1.7764334678649902,
        "mean_weights": [
          0.08629432320594788,
          0.05034743621945381,
          0.046479545533657074,
          0.04277106374502182,
          0.04524604231119156,
          0.05579439178109169,
          0.07300648093223572,
          0.2586928606033325,
          0.3413679003715515
        ],
        "sources": 9
      },
      "step": 4000,
      "stream_state_rms_by_sublayer": [
        0.06033274158835411,
        0.16565655171871185,
        0.19572967290878296,
        0.24343863129615784,
        0.20718541741371155,
        0.21182885766029358,
        0.280988872051239,
        0.28418517112731934,
        0.11816111207008362,
        0.12311498820781708,
        0.17678114771842957,
        0.18273214995861053,
        0.05406087636947632,
        0.06521740555763245,
        0.0933222770690918,
        0.10707039386034012,
        0.04759044572710991,
        0.06429450213909149,
        0.09436589479446411,
        0.1264011114835739,
        0.052182719111442566,
        0.0799982100725174,
        0.0912039652466774,
        0.12911832332611084,
        0.03087303601205349,
        0.06644809991121292,
        0.07178688794374466,
        0.1084718257188797,
        0.0293489471077919,
        0.10412788391113281,
        0.10697942227125168,
        0.23786598443984985
      ]
    },
    {
      "activation_grad_rms_by_block": [
        7.570489106001332e-05,
        7.69118414609693e-05,
        7.822692714398727e-05,
        6.853767990833148e-05,
        6.88084473949857e-05,
        7.10485182935372e-05,
        8.91982635948807e-05,
        8.665756467962638e-05,
        0.000171030915225856,
        0.00012665522808674723,
        0.00023480529489461333,
        0.00016227969899773598,
        0.00023323859204538167,
        0.000136202055728063,
        9.528713417239487e-05,
        6.303648115135729e-05
      ],
      "activation_grad_statistics": {
        "first_quartile_mean": 7.484533489332534e-05,
        "first_to_last_ratio": 0.5672633797074002,
        "imbalance_abs_log_ratio": 0.5669315686327029,
        "last_quartile_mean": 0.0001319410657742992,
        "mean": 0.00011485184586490504,
        "normalized": [
          0.6591525847051819,
          0.6696613439842941,
          0.6811116230208613,
          0.5967486146365397,
          0.5991061517280438,
          0.6186101560536372,
          0.7766376145125305,
          0.7545160813658816,
          1.4891438090340474,
          1.1027705051926284,
          2.0444189914962623,
          1.4129481139434028,
          2.0307779147035174,
          1.1858934847966764,
          0.8296526142424977,
          0.5488503965839976
        ],
        "population_cv": 0.4839100882418841
      },
      "activation_output_rms_by_block": [
        0.16869333386421204,
        0.24958865344524384,
        0.23942448198795319,
        0.31786683201789856,
        0.1331564038991928,
        0.19950589537620544,
        0.06165311112999916,
        0.10097606480121613,
        0.05752932280302048,
        0.11994150280952454,
        0.07362622767686844,
        0.12106145173311234,
        0.06137929856777191,
        0.09901073575019836,
        0.0852273628115654,
        0.19495683908462524
      ],
      "activation_output_statistics": {
        "first_quartile_mean": 0.2438933253288269,
        "first_to_last_ratio": 2.2143221757549307,
        "imbalance_abs_log_ratio": 0.7949463412042855,
        "last_quartile_mean": 0.11014355905354023,
        "mean": 0.142724844859913,
        "normalized": [
          1.1819479224502754,
          1.7487400577679353,
          1.6775249062134392,
          2.2271303383085868,
          0.9329588273848762,
          1.3978357837559683,
          0.43197182095740094,
          0.7074876480007805,
          0.4030785450107621,
          0.8403687734062649,
          0.5158613256797294,
          0.8482156827841455,
          0.430053356358641,
          0.6937175923882012,
          0.5971445468742151,
          1.3659628726587785
        ],
        "population_cv": 0.529368044298578
      },
      "bits_per_byte": 1.883020158101202,
      "branch_output_rms_by_sublayer": [
        0.06029115617275238,
        0.1536618322134018,
        0.06518179178237915,
        0.06745198369026184,
        0.23389145731925964,
        0.026561493054032326,
        0.09204838424921036,
        0.022710271179676056,
        0.12803514301776886,
        0.017566412687301636,
        0.07296492159366608,
        0.024595333263278008,
        0.05156426504254341,
        0.023977436125278473,
        0.0450289212167263,
        0.03369171544909477,
        0.04221063107252121,
        0.034665971994400024,
        0.05162743479013443,
        0.07154636830091476,
        0.04606551304459572,
        0.05490076169371605,
        0.03287900984287262,
        0.08022980391979218,
        0.027681373059749603,
        0.05210135132074356,
        0.019983114674687386,
        0.06804364919662476,
        0.02240605279803276,
        0.08063697069883347,
        0.016830744221806526,
        0.13339926302433014
      ],
      "capture": {
        "all_gradients_finite": true,
        "all_gradients_present": true,
        "count": 16,
        "dtypes": [
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32",
          "torch.float32"
        ],
        "position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
        "shape": [
          16,
          256,
          192
        ],
        "storage_unique": true
      },
      "core_parameter_grad_rms_by_block": [
        0.0005343193187437567,
        0.0003255019836390766,
        0.00031277964721318437,
        0.00022314651478922115,
        0.0002396348782003974,
        0.00021519463019459167,
        0.00025456643186586784,
        0.0003008996254324453,
        0.00059437697364117,
        0.0008147654749400623,
        0.001560733102975404,
        0.001320101058102649,
        0.0015032932063515777,
        0.0009169853654825589,
        0.0006174439085696804,
        0.00045539974927406244
      ],
      "core_parameter_grad_statistics": {
        "first_quartile_mean": 0.0003489368660963097,
        "first_to_last_ratio": 0.3995701760811125,
        "imbalance_abs_log_ratio": 0.9173658694246768,
        "last_quartile_mean": 0.0008732805574194699,
        "mean": 0.0006368213668384816,
        "normalized": [
          0.8390411292202721,
          0.5111354621391565,
          0.4911575890834005,
          0.3504067646113047,
          0.3762984263390404,
          0.33791992762889184,
          0.39974543117117817,
          0.4725024019314401,
          0.9333496088423855,
          1.2794254674352237,
          2.450817739869046,
          2.072953463631928,
          2.3606199236290095,
          1.4399412664731395,
          0.9695715953046595,
          0.7151138026899253
        ],
        "population_cv": 0.7016607196207993
      },
      "depth_weights": [
        {
          "entropy_mean": 0.0,
          "mean_weights": [
            1.0
          ],
          "sources": 1
        },
        {
          "entropy_mean": 0.2716493308544159,
          "mean_weights": [
            0.9192699193954468,
            0.0807301327586174
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.5622239112854004,
          "mean_weights": [
            0.72159743309021,
            0.27840256690979004
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.48637402057647705,
          "mean_weights": [
            0.7995253801345825,
            0.20047461986541748
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.6475558280944824,
          "mean_weights": [
            0.5858595371246338,
            0.41414040327072144
          ],
          "sources": 2
        },
        {
          "entropy_mean": 0.9933564066886902,
          "mean_weights": [
            0.4998217821121216,
            0.2229493409395218,
            0.2772288918495178
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.0489425659179688,
          "mean_weights": [
            0.2563193440437317,
            0.269240140914917,
            0.4744405150413513
          ],
          "sources": 3
        },
        {
          "entropy_mean": 0.9815441966056824,
          "mean_weights": [
            0.5070900917053223,
            0.19802016019821167,
            0.29488980770111084
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.071545958518982,
          "mean_weights": [
            0.2598530650138855,
            0.3191677927970886,
            0.4209791421890259
          ],
          "sources": 3
        },
        {
          "entropy_mean": 1.3049530982971191,
          "mean_weights": [
            0.3487145006656647,
            0.13604819774627686,
            0.23624856770038605,
            0.27898871898651123
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3723504543304443,
          "mean_weights": [
            0.2135777771472931,
            0.23673927783966064,
            0.23828449845314026,
            0.311398446559906
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.265662670135498,
          "mean_weights": [
            0.24546673893928528,
            0.10275381803512573,
            0.2848597466945648,
            0.36691969633102417
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.3751296997070312,
          "mean_weights": [
            0.23200613260269165,
            0.24026888608932495,
            0.22661332786083221,
            0.30111163854599
          ],
          "sources": 4
        },
        {
          "entropy_mean": 1.5548532009124756,
          "mean_weights": [
            0.2040066421031952,
            0.09975787997245789,
            0.22992464900016785,
            0.259371280670166,
            0.20693954825401306
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5716381072998047,
          "mean_weights": [
            0.1984836459159851,
            0.17718908190727234,
            0.15249167382717133,
            0.17045220732688904,
            0.301383376121521
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.4117064476013184,
          "mean_weights": [
            0.25685301423072815,
            0.12795110046863556,
            0.08630626648664474,
            0.10788753628730774,
            0.4210020899772644
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.5975096225738525,
          "mean_weights": [
            0.2208021879196167,
            0.19316574931144714,
            0.17618897557258606,
            0.17661380767822266,
            0.23322927951812744
          ],
          "sources": 5
        },
        {
          "entropy_mean": 1.7283259630203247,
          "mean_weights": [
            0.1650083363056183,
            0.10661210119724274,
            0.12329171597957611,
            0.13269363343715668,
            0.19600069522857666,
            0.27639350295066833
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7028545141220093,
          "mean_weights": [
            0.17663681507110596,
            0.13835734128952026,
            0.11904168128967285,
            0.11434897780418396,
            0.12915222346782684,
            0.3224629759788513
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.27894127368927,
          "mean_weights": [
            0.17268770933151245,
            0.10967432707548141,
            0.03322022780776024,
            0.031795307993888855,
            0.09717383980751038,
            0.555448591709137
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.7646939754486084,
          "mean_weights": [
            0.1884503960609436,
            0.1533900499343872,
            0.15274858474731445,
            0.14330559968948364,
            0.1338358074426651,
            0.2282695174217224
          ],
          "sources": 6
        },
        {
          "entropy_mean": 1.4344873428344727,
          "mean_weights": [
            0.09210026264190674,
            0.06219030171632767,
            0.029036182910203934,
            0.0315997451543808,
            0.07091090828180313,
            0.1949847936630249,
            0.5191777944564819
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.802895426750183,
          "mean_weights": [
            0.13523127138614655,
            0.10325458645820618,
            0.10260490328073502,
            0.09636874496936798,
            0.09404134750366211,
            0.12264730036258698,
            0.3458518385887146
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.4361302852630615,
          "mean_weights": [
            0.10137400776147842,
            0.06668317317962646,
            0.03099493309855461,
            0.033589236438274384,
            0.07286111265420914,
            0.16290563344955444,
            0.5315918922424316
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.8628063201904297,
          "mean_weights": [
            0.14754432439804077,
            0.11970964819192886,
            0.10172373056411743,
            0.09891469776630402,
            0.10695713013410568,
            0.14022418856620789,
            0.28492626547813416
          ],
          "sources": 7
        },
        {
          "entropy_mean": 1.6038438081741333,
          "mean_weights": [
            0.07592754065990448,
            0.048247914761304855,
            0.024802275002002716,
            0.026826059445738792,
            0.049207836389541626,
            0.09689367562532425,
            0.29891306161880493,
            0.3791816532611847
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.7748796939849854,
          "mean_weights": [
            0.09652430564165115,
            0.07308399677276611,
            0.04972345381975174,
            0.050231896340847015,
            0.0702696442604065,
            0.09669448435306549,
            0.16264159977436066,
            0.40083062648773193
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.526896357536316,
          "mean_weights": [
            0.06852421164512634,
            0.05069870874285698,
            0.02793668583035469,
            0.029434790834784508,
            0.046449482440948486,
            0.08027796447277069,
            0.19853916764259338,
            0.49813902378082275
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.9032639265060425,
          "mean_weights": [
            0.11537295579910278,
            0.09104456007480621,
            0.05989263206720352,
            0.057612620294094086,
            0.07744243741035461,
            0.11209231615066528,
            0.20047283172607422,
            0.2860696315765381
          ],
          "sources": 8
        },
        {
          "entropy_mean": 1.7657151222229004,
          "mean_weights": [
            0.06711487472057343,
            0.04881071671843529,
            0.02479729615151882,
            0.025201791897416115,
            0.040198031812906265,
            0.06934815645217896,
            0.15364184975624084,
            0.3216337561607361,
            0.24925348162651062
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.9778118133544922,
          "mean_weights": [
            0.08567827194929123,
            0.08225203305482864,
            0.030744101852178574,
            0.03173041343688965,
            0.07107861340045929,
            0.128017395734787,
            0.2095082402229309,
            0.20655101537704468,
            0.15443992614746094
          ],
          "sources": 9
        },
        {
          "entropy_mean": 1.88002610206604,
          "mean_weights": [
            0.07342863082885742,
            0.07307932525873184,
            0.030178282409906387,
            0.029849819839000702,
            0.050495631992816925,
            0.08296258747577667,
            0.14574018120765686,
            0.2550249397754669,
            0.25924059748649597
          ],
          "sources": 9
        }
      ],
      "layer_input_rms_by_sublayer": [
        0.061662707477808,
        0.0571652427315712,
        0.07283993810415268,
        0.07206818461418152,
        0.1187756210565567,
        0.10985641181468964,
        0.15539979934692383,
        0.12964090704917908,
        0.18306857347488403,
        0.129257932305336,
        0.14960098266601562,
        0.16845102608203888,
        0.16322582960128784,
        0.13869591057300568,
        0.11515536159276962,
        0.08842873573303223,
        0.1271047443151474,
        0.08747127652168274,
        0.08377104997634888,
        0.05908897519111633,
        0.10309968143701553,
        0.03653678670525551,
        0.06588983535766602,
        0.052161477506160736,
        0.07435949891805649,
        0.04259056970477104,
        0.04427788779139519,
        0.043995074927806854,
        0.053472407162189484,
        0.04156629368662834,
        0.04980214312672615,
        0.0496271476149559
      ],
      "loss_nats": 1.3052101135253906,
      "loss_scale": 1.0,
      "output_weights": {
        "entropy_mean": 1.609398603439331,
        "mean_weights": [
          0.0809592753648758,
          0.04013277217745781,
          0.03442784771323204,
          0.03263507038354874,
          0.03443540260195732,
          0.047240644693374634,
          0.05792279541492462,
          0.2393622249364853,
          0.43288397789001465
        ],
        "sources": 9
      },
      "step": 8000,
      "stream_state_rms_by_sublayer": [
        0.06029115617275238,
        0.16869333386421204,
        0.20056062936782837,
        0.24958865344524384,
        0.23389145731925964,
        0.23942448198795319,
        0.31467917561531067,
        0.31786683201789856,
        0.12803514301776886,
        0.1331564038991928,
        0.19432871043682098,
        0.19950589537620544,
        0.05156426504254341,
        0.06165311112999916,
        0.08753261715173721,
        0.10097606480121613,
        0.04221063107252121,
        0.05752932280302048,
        0.08379971981048584,
        0.11994150280952454,
        0.04606551304459572,
        0.07362622767686844,
        0.08261217921972275,
        0.12106145173311234,
        0.027681373059749603,
        0.06137929856777191,
        0.06537999957799911,
        0.09901073575019836,
        0.02240605279803276,
        0.0852273628115654,
        0.08763451874256134,
        0.19495683908462524
      ]
    }
  ],
  "environment": {
    "autocast": "cuda-bfloat16-forward-fp32-cross-entropy",
    "compile": false,
    "compute_capability": [
      12,
      0
    ],
    "cublas_workspace_config": ":4096:8",
    "cuda": "12.8",
    "deterministic_algorithms": true,
    "gpu": "NVIDIA GeForce RTX 5090",
    "python": "3.10.14",
    "torch": "2.11.0+cu128"
  },
  "evaluations": [
    {
      "bits_per_byte": 8.015965601418904,
      "cross_entropy_nats": 5.55624395608902,
      "step": 0
    },
    {
      "bits_per_byte": 5.1077262486126465,
      "cross_entropy_nats": 3.540406048297882,
      "step": 100
    },
    {
      "bits_per_byte": 3.1449978072640112,
      "cross_entropy_nats": 2.1799463629722595,
      "step": 500
    },
    {
      "bits_per_byte": 1.978178932745523,
      "cross_entropy_nats": 1.3711691498756409,
      "step": 2000
    },
    {
      "bits_per_byte": 1.8301036761704788,
      "cross_entropy_nats": 1.2685312032699585,
      "step": 4000
    },
    {
      "bits_per_byte": 1.72661337398776,
      "cross_entropy_nats": 1.1967971920967102,
      "step": 8000
    }
  ],
  "gradient_gate": null,
  "hashes": {
    "final_mixer_parameters": "ad1807a65c19f598a4bde3ce03f75bb64bba14b79befbf1740b5aaddab905279",
    "final_model_state": "6df5cb12d848445089dff93c7b22def426897ab70173e1fcca78c3db41e914dc",
    "final_optimizer_state": "a8e9a566e68720356f0b3f878d687f8c4cac616f59f729e02056853409f8d145",
    "final_public_parameters": "16b3a7903fb4172275d1673bd0f390d3b3c03a9d0d151ac66d732eb978cff06f",
    "initial_mixer_parameters": "64fe0b90f5ea84ed88b6d93838558443c700c2b89ade2455ed526bd8db5d506f",
    "initial_public_parameter_elements": 9541824,
    "initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe",
    "initial_public_parameter_tensors": 115,
    "initial_public_parameters": "7a565cd353efdb3b95e9b8b1844581082c029991ea18d438b4b18566970c8c61"
  },
  "manifest": {
    "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716",
    "file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371",
    "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
    "input_gate_tensor_hashes": {
      "0": "3eff5c07fa0cfbd5ceed5922cbd09525ad514f801cc0a440c03fd2fa15df4c84",
      "1": "e83e4001c5811b19c69a54c7ea7a802af996a34aa57072cde43a76bc8369e2d0",
      "7999": "d04c4ca879f0209183a48ac54a0c4ffe3a0f62a78888479d4641f6c304426442"
    },
    "path": "experiments/k3/attnres_gradient/manifest.json",
    "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338"
  },
  "model": {
    "attnres_aggregation_groups": 8,
    "context": 256,
    "d_ff": 768,
    "d_head": 32,
    "d_model": 192,
    "heads": 6,
    "layers": 16,
    "parameters": {
      "core": 9541824,
      "embedding": 98304,
      "mixer": 12672,
      "total": 9554496
    },
    "sublayers": 32,
    "sublayers_per_attnres_group": 4,
    "transformer_blocks_per_attnres_group": 2,
    "vocabulary": 256
  },
  "optimizer": {
    "betas": [
      0.9,
      0.95
    ],
    "epsilon": 1e-08,
    "grad_clip": 1.0,
    "min_lr": 3e-05,
    "name": "AdamW",
    "peak_lr": 0.0003,
    "warmup_steps": 400,
    "weight_decay_ndim_ge_2": 0.1
  },
  "protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
  "run_kind": "formal",
  "schema_version": 1,
  "seed": 2026073003,
  "steps": 8000,
  "target_bytes_seen": 65536000,
  "timing": {
    "mean_ms": 54.54310071476275,
    "measured_steps": 7980,
    "median_ms": 54.49852900346741,
    "p95_ms": 54.8562506082817,
    "peak_allocated_bytes": 6536921088,
    "peak_reserved_bytes": 6889144320,
    "warmup_steps_excluded": 20
  },
  "training_history": [
    {
      "bits_per_byte": 8.030921126746609,
      "learning_rate": 7.499999999999999e-07,
      "loss_nats": 5.566610336303711,
      "step": 1,
      "unclipped_grad_norm": 17.857467651367188
    },
    {
      "bits_per_byte": 7.460191884344154,
      "learning_rate": 7.499999999999999e-06,
      "loss_nats": 5.171010971069336,
      "step": 10,
      "unclipped_grad_norm": 12.879206657409668
    },
    {
      "bits_per_byte": 6.7765918040416455,
      "learning_rate": 1.4999999999999999e-05,
      "loss_nats": 4.6971755027771,
      "step": 20,
      "unclipped_grad_norm": 4.5564117431640625
    },
    {
      "bits_per_byte": 6.553465640395858,
      "learning_rate": 2.2499999999999998e-05,
      "loss_nats": 4.542516231536865,
      "step": 30,
      "unclipped_grad_norm": 3.0218148231506348
    },
    {
      "bits_per_byte": 6.42131485940348,
      "learning_rate": 2.9999999999999997e-05,
      "loss_nats": 4.450916290283203,
      "step": 40,
      "unclipped_grad_norm": 2.865441083908081
    },
    {
      "bits_per_byte": 6.256338093806218,
      "learning_rate": 3.75e-05,
      "loss_nats": 4.3365631103515625,
      "step": 50,
      "unclipped_grad_norm": 2.8344333171844482
    },
    {
      "bits_per_byte": 5.946641367160921,
      "learning_rate": 4.4999999999999996e-05,
      "loss_nats": 4.1218976974487305,
      "step": 60,
      "unclipped_grad_norm": 2.6930737495422363
    },
    {
      "bits_per_byte": 5.708490111205155,
      "learning_rate": 5.2499999999999995e-05,
      "loss_nats": 3.9568238258361816,
      "step": 70,
      "unclipped_grad_norm": 2.6865241527557373
    },
    {
      "bits_per_byte": 5.509431199721109,
      "learning_rate": 5.9999999999999995e-05,
      "loss_nats": 3.8188467025756836,
      "step": 80,
      "unclipped_grad_norm": 2.448140859603882
    },
    {
      "bits_per_byte": 5.311447867735559,
      "learning_rate": 6.749999999999999e-05,
      "loss_nats": 3.681615114212036,
      "step": 90,
      "unclipped_grad_norm": 2.2759759426116943
    },
    {
      "bits_per_byte": 5.0864742664542035,
      "learning_rate": 7.5e-05,
      "loss_nats": 3.5256752967834473,
      "step": 100,
      "unclipped_grad_norm": 2.1487066745758057
    },
    {
      "bits_per_byte": 4.9338616775927715,
      "learning_rate": 8.249999999999999e-05,
      "loss_nats": 3.4198923110961914,
      "step": 110,
      "unclipped_grad_norm": 1.9959912300109863
    },
    {
      "bits_per_byte": 4.731707078393195,
      "learning_rate": 8.999999999999999e-05,
      "loss_nats": 3.2797694206237793,
      "step": 120,
      "unclipped_grad_norm": 1.7048262357711792
    },
    {
      "bits_per_byte": 4.582112785054405,
      "learning_rate": 9.75e-05,
      "loss_nats": 3.1760785579681396,
      "step": 130,
      "unclipped_grad_norm": 1.5641754865646362
    },
    {
      "bits_per_byte": 4.436114304979855,
      "learning_rate": 0.00010499999999999999,
      "loss_nats": 3.0748801231384277,
      "step": 140,
      "unclipped_grad_norm": 1.380750060081482
    },
    {
      "bits_per_byte": 4.229324085408923,
      "learning_rate": 0.0001125,
      "loss_nats": 2.931544065475464,
      "step": 150,
      "unclipped_grad_norm": 1.1272099018096924
    },
    {
      "bits_per_byte": 4.108468093029308,
      "learning_rate": 0.00011999999999999999,
      "loss_nats": 2.8477730751037598,
      "step": 160,
      "unclipped_grad_norm": 0.8926234245300293
    },
    {
      "bits_per_byte": 4.025982461953091,
      "learning_rate": 0.00012749999999999998,
      "loss_nats": 2.7905983924865723,
      "step": 170,
      "unclipped_grad_norm": 0.915916919708252
    },
    {
      "bits_per_byte": 3.9299425341970746,
      "learning_rate": 0.00013499999999999997,
      "loss_nats": 2.7240285873413086,
      "step": 180,
      "unclipped_grad_norm": 0.7168256640434265
    },
    {
      "bits_per_byte": 3.8518424607178594,
      "learning_rate": 0.0001425,
      "loss_nats": 2.669893741607666,
      "step": 190,
      "unclipped_grad_norm": 0.8937639594078064
    },
    {
      "bits_per_byte": 3.7890044716415114,
      "learning_rate": 0.00015,
      "loss_nats": 2.626337766647339,
      "step": 200,
      "unclipped_grad_norm": 0.7755346894264221
    },
    {
      "bits_per_byte": 3.7186869570791403,
      "learning_rate": 0.0001575,
      "loss_nats": 2.5775973796844482,
      "step": 210,
      "unclipped_grad_norm": 0.731957197189331
    },
    {
      "bits_per_byte": 3.706538446587493,
      "learning_rate": 0.00016499999999999997,
      "loss_nats": 2.56917667388916,
      "step": 220,
      "unclipped_grad_norm": 0.8748186230659485
    },
    {
      "bits_per_byte": 3.655573795761412,
      "learning_rate": 0.0001725,
      "loss_nats": 2.53385066986084,
      "step": 230,
      "unclipped_grad_norm": 0.8331225514411926
    },
    {
      "bits_per_byte": 3.652975825837478,
      "learning_rate": 0.00017999999999999998,
      "loss_nats": 2.5320498943328857,
      "step": 240,
      "unclipped_grad_norm": 1.103366494178772
    },
    {
      "bits_per_byte": 3.571490789823981,
      "learning_rate": 0.0001875,
      "loss_nats": 2.4755687713623047,
      "step": 250,
      "unclipped_grad_norm": 0.6647337675094604
    },
    {
      "bits_per_byte": 3.5855342051628347,
      "learning_rate": 0.000195,
      "loss_nats": 2.4853029251098633,
      "step": 260,
      "unclipped_grad_norm": 0.8029841780662537
    },
    {
      "bits_per_byte": 3.576924065730039,
      "learning_rate": 0.00020249999999999996,
      "loss_nats": 2.479334831237793,
      "step": 270,
      "unclipped_grad_norm": 0.7433680891990662
    },
    {
      "bits_per_byte": 3.5153632198189984,
      "learning_rate": 0.00020999999999999998,
      "loss_nats": 2.43666410446167,
      "step": 280,
      "unclipped_grad_norm": 0.824142336845398
    },
    {
      "bits_per_byte": 3.511655617831698,
      "learning_rate": 0.00021749999999999997,
      "loss_nats": 2.434094190597534,
      "step": 290,
      "unclipped_grad_norm": 0.8560227751731873
    },
    {
      "bits_per_byte": 3.4998641433232422,
      "learning_rate": 0.000225,
      "loss_nats": 2.4259209632873535,
      "step": 300,
      "unclipped_grad_norm": 0.8836220502853394
    },
    {
      "bits_per_byte": 3.4156380478202935,
      "learning_rate": 0.00023249999999999996,
      "loss_nats": 2.367539882659912,
      "step": 310,
      "unclipped_grad_norm": 0.9033657908439636
    },
    {
      "bits_per_byte": 3.481727196928552,
      "learning_rate": 0.00023999999999999998,
      "loss_nats": 2.4133493900299072,
      "step": 320,
      "unclipped_grad_norm": 1.119826316833496
    },
    {
      "bits_per_byte": 3.4809216301919106,
      "learning_rate": 0.0002475,
      "loss_nats": 2.4127910137176514,
      "step": 330,
      "unclipped_grad_norm": 0.9842678308486938
    },
    {
      "bits_per_byte": 3.422977579428515,
      "learning_rate": 0.00025499999999999996,
      "loss_nats": 2.3726272583007812,
      "step": 340,
      "unclipped_grad_norm": 1.075486421585083
    },
    {
      "bits_per_byte": 3.385370477129646,
      "learning_rate": 0.0002625,
      "loss_nats": 2.346560001373291,
      "step": 350,
      "unclipped_grad_norm": 0.7245537042617798
    },
    {
      "bits_per_byte": 3.385430671057448,
      "learning_rate": 0.00026999999999999995,
      "loss_nats": 2.346601724624634,
      "step": 360,
      "unclipped_grad_norm": 1.202561616897583
    },
    {
      "bits_per_byte": 3.3953991294667585,
      "learning_rate": 0.00027749999999999997,
      "loss_nats": 2.353511333465576,
      "step": 370,
      "unclipped_grad_norm": 0.7641725540161133
    },
    {
      "bits_per_byte": 3.295281837024086,
      "learning_rate": 0.000285,
      "loss_nats": 2.2841153144836426,
      "step": 380,
      "unclipped_grad_norm": 1.061362862586975
    },
    {
      "bits_per_byte": 3.380484450018634,
      "learning_rate": 0.0002925,
      "loss_nats": 2.3431732654571533,
      "step": 390,
      "unclipped_grad_norm": 1.0704491138458252
    },
    {
      "bits_per_byte": 3.354825670405808,
      "learning_rate": 0.0003,
      "loss_nats": 2.325387954711914,
      "step": 400,
      "unclipped_grad_norm": 1.066178560256958
    },
    {
      "bits_per_byte": 3.38879568360423,
      "learning_rate": 0.0002999988466112388,
      "loss_nats": 2.3489341735839844,
      "step": 410,
      "unclipped_grad_norm": 0.9359472990036011
    },
    {
      "bits_per_byte": 3.3128674070401103,
      "learning_rate": 0.00029999538646466355,
      "loss_nats": 2.296304702758789,
      "step": 420,
      "unclipped_grad_norm": 0.9575046896934509
    },
    {
      "bits_per_byte": 3.3331142365955855,
      "learning_rate": 0.0002999896196193985,
      "loss_nats": 2.3103387355804443,
      "step": 430,
      "unclipped_grad_norm": 0.9350728988647461
    },
    {
      "bits_per_byte": 3.2769842588334903,
      "learning_rate": 0.00029998154617398326,
      "loss_nats": 2.271432399749756,
      "step": 440,
      "unclipped_grad_norm": 1.028943419456482
    },
    {
      "bits_per_byte": 3.3396626480098375,
      "learning_rate": 0.0002999711662663707,
      "loss_nats": 2.31487774848938,
      "step": 450,
      "unclipped_grad_norm": 1.1271060705184937
    },
    {
      "bits_per_byte": 3.2591895579140173,
      "learning_rate": 0.0002999584800739247,
      "loss_nats": 2.2590980529785156,
      "step": 460,
      "unclipped_grad_norm": 1.1319539546966553
    },
    {
      "bits_per_byte": 3.2327510089621794,
      "learning_rate": 0.0002999434878134173,
      "loss_nats": 2.240772247314453,
      "step": 470,
      "unclipped_grad_norm": 1.1909105777740479
    },
    {
      "bits_per_byte": 3.2186394884935843,
      "learning_rate": 0.000299926189741025,
      "loss_nats": 2.2309908866882324,
      "step": 480,
      "unclipped_grad_norm": 1.3586266040802002
    },
    {
      "bits_per_byte": 3.169725902761693,
      "learning_rate": 0.000299906586152324,
      "loss_nats": 2.1970865726470947,
      "step": 490,
      "unclipped_grad_norm": 1.5543664693832397
    },
    {
      "bits_per_byte": 3.101864644418944,
      "learning_rate": 0.0002998846773822856,
      "loss_nats": 2.1500487327575684,
      "step": 500,
      "unclipped_grad_norm": 1.9330527782440186
    },
    {
      "bits_per_byte": 3.1523112835006266,
      "learning_rate": 0.0002998604638052702,
      "loss_nats": 2.1850156784057617,
      "step": 510,
      "unclipped_grad_norm": 1.367343544960022
    },
    {
      "bits_per_byte": 3.137257298105303,
      "learning_rate": 0.0002998339458350211,
      "loss_nats": 2.1745810508728027,
      "step": 520,
      "unclipped_grad_norm": 1.4051828384399414
    },
    {
      "bits_per_byte": 2.96818390576449,
      "learning_rate": 0.0002998051239246572,
      "loss_nats": 2.0573883056640625,
      "step": 530,
      "unclipped_grad_norm": 0.88100665807724
    },
    {
      "bits_per_byte": 2.9700385666713944,
      "learning_rate": 0.00029977399856666537,
      "loss_nats": 2.058673858642578,
      "step": 540,
      "unclipped_grad_norm": 1.1694613695144653
    },
    {
      "bits_per_byte": 2.981698990399893,
      "learning_rate": 0.0002997405702928922,
      "loss_nats": 2.066756248474121,
      "step": 550,
      "unclipped_grad_norm": 2.2637624740600586
    },
    {
      "bits_per_byte": 2.9860140351100406,
      "learning_rate": 0.0002997048396745345,
      "loss_nats": 2.06974720954895,
      "step": 560,
      "unclipped_grad_norm": 2.0154106616973877
    },
    {
      "bits_per_byte": 2.908624249978889,
      "learning_rate": 0.0002996668073221302,
      "loss_nats": 2.0161046981811523,
      "step": 570,
      "unclipped_grad_norm": 1.2818057537078857
    },
    {
      "bits_per_byte": 2.9258029090429716,
      "learning_rate": 0.000299626473885547,
      "loss_nats": 2.0280120372772217,
      "step": 580,
      "unclipped_grad_norm": 1.5401909351348877
    },
    {
      "bits_per_byte": 2.865440266260492,
      "learning_rate": 0.00029958384005397226,
      "loss_nats": 1.986171841621399,
      "step": 590,
      "unclipped_grad_norm": 1.2538588047027588
    },
    {
      "bits_per_byte": 2.8203701488100825,
      "learning_rate": 0.0002995389065559004,
      "loss_nats": 1.954931616783142,
      "step": 600,
      "unclipped_grad_norm": 1.5663292407989502
    },
    {
      "bits_per_byte": 2.8417604910111445,
      "learning_rate": 0.000299491674159121,
      "loss_nats": 1.9697582721710205,
      "step": 610,
      "unclipped_grad_norm": 1.3150807619094849
    },
    {
      "bits_per_byte": 2.806068931477586,
      "learning_rate": 0.00029944214367070535,
      "loss_nats": 1.9450187683105469,
      "step": 620,
      "unclipped_grad_norm": 1.0626381635665894
    },
    {
      "bits_per_byte": 2.766462358879786,
      "learning_rate": 0.00029939031593699284,
      "loss_nats": 1.9175655841827393,
      "step": 630,
      "unclipped_grad_norm": 2.55611515045166
    },
    {
      "bits_per_byte": 2.7411267346679313,
      "learning_rate": 0.00029933619184357654,
      "loss_nats": 1.900004267692566,
      "step": 640,
      "unclipped_grad_norm": 0.9799621105194092
    },
    {
      "bits_per_byte": 2.7289859633955733,
      "learning_rate": 0.00029927977231528774,
      "loss_nats": 1.8915889263153076,
      "step": 650,
      "unclipped_grad_norm": 1.4919346570968628
    },
    {
      "bits_per_byte": 2.7446909031244124,
      "learning_rate": 0.00029922105831618057,
      "loss_nats": 1.9024747610092163,
      "step": 660,
      "unclipped_grad_norm": 1.184576153755188
    },
    {
      "bits_per_byte": 2.706914225931793,
      "learning_rate": 0.0002991600508495153,
      "loss_nats": 1.876289963722229,
      "step": 670,
      "unclipped_grad_norm": 1.2165472507476807
    },
    {
      "bits_per_byte": 2.7108362902847194,
      "learning_rate": 0.00029909675095774114,
      "loss_nats": 1.8790085315704346,
      "step": 680,
      "unclipped_grad_norm": 2.3380022048950195
    },
    {
      "bits_per_byte": 2.6699751042488753,
      "learning_rate": 0.0002990311597224787,
      "loss_nats": 1.850685715675354,
      "step": 690,
      "unclipped_grad_norm": 1.790702223777771
    },
    {
      "bits_per_byte": 2.6284332108809165,
      "learning_rate": 0.00029896327826450103,
      "loss_nats": 1.8218910694122314,
      "step": 700,
      "unclipped_grad_norm": 1.6820380687713623
    },
    {
      "bits_per_byte": 2.6224609413470548,
      "learning_rate": 0.0002988931077437152,
      "loss_nats": 1.817751407623291,
      "step": 710,
      "unclipped_grad_norm": 1.3093626499176025
    },
    {
      "bits_per_byte": 2.6378994799497075,
      "learning_rate": 0.00029882064935914164,
      "loss_nats": 1.8284525871276855,
      "step": 720,
      "unclipped_grad_norm": 1.689491868019104
    },
    {
      "bits_per_byte": 2.604935393276182,
      "learning_rate": 0.0002987459043488943,
      "loss_nats": 1.8056036233901978,
      "step": 730,
      "unclipped_grad_norm": 1.4546315670013428
    },
    {
      "bits_per_byte": 2.542083817570416,
      "learning_rate": 0.0002986688739901591,
      "loss_nats": 1.762038230895996,
      "step": 740,
      "unclipped_grad_norm": 1.6848912239074707
    },
    {
      "bits_per_byte": 2.5572903515770546,
      "learning_rate": 0.0002985895595991723,
      "loss_nats": 1.7725785970687866,
      "step": 750,
      "unclipped_grad_norm": 1.477650761604309
    },
    {
      "bits_per_byte": 2.448532478772384,
      "learning_rate": 0.00029850796253119824,
      "loss_nats": 1.6971933841705322,
      "step": 760,
      "unclipped_grad_norm": 1.4033355712890625
    },
    {
      "bits_per_byte": 2.5017822630804796,
      "learning_rate": 0.00029842408418050544,
      "loss_nats": 1.7341033220291138,
      "step": 770,
      "unclipped_grad_norm": 1.3305320739746094
    },
    {
      "bits_per_byte": 2.4567315796696176,
      "learning_rate": 0.0002983379259803436,
      "loss_nats": 1.7028765678405762,
      "step": 780,
      "unclipped_grad_norm": 1.7414066791534424
    },
    {
      "bits_per_byte": 2.463544844366199,
      "learning_rate": 0.0002982494894029186,
      "loss_nats": 1.70759916305542,
      "step": 790,
      "unclipped_grad_norm": 1.3770233392715454
    },
    {
      "bits_per_byte": 2.390281954925151,
      "learning_rate": 0.0002981587759593675,
      "loss_nats": 1.6568171977996826,
      "step": 800,
      "unclipped_grad_norm": 1.4013909101486206
    },
    {
      "bits_per_byte": 2.4192710065892844,
      "learning_rate": 0.00029806578719973283,
      "loss_nats": 1.6769108772277832,
      "step": 810,
      "unclipped_grad_norm": 1.360757827758789
    },
    {
      "bits_per_byte": 2.3909581907083437,
      "learning_rate": 0.00029797052471293596,
      "loss_nats": 1.6572859287261963,
      "step": 820,
      "unclipped_grad_norm": 1.4178366661071777
    },
    {
      "bits_per_byte": 2.4097108349931227,
      "learning_rate": 0.0002978729901267499,
      "loss_nats": 1.6702842712402344,
      "step": 830,
      "unclipped_grad_norm": 1.1894793510437012
    },
    {
      "bits_per_byte": 2.395687713607071,
      "learning_rate": 0.0002977731851077716,
      "loss_nats": 1.6605641841888428,
      "step": 840,
      "unclipped_grad_norm": 1.7937202453613281
    },
    {
      "bits_per_byte": 2.326945216161301,
      "learning_rate": 0.0002976711113613935,
      "loss_nats": 1.6129155158996582,
      "step": 850,
      "unclipped_grad_norm": 1.4669246673583984
    },
    {
      "bits_per_byte": 2.3940583499727963,
      "learning_rate": 0.0002975667706317743,
      "loss_nats": 1.6594347953796387,
      "step": 860,
      "unclipped_grad_norm": 1.3730710744857788
    },
    {
      "bits_per_byte": 2.3742401011832714,
      "learning_rate": 0.00029746016470180915,
      "loss_nats": 1.645697832107544,
      "step": 870,
      "unclipped_grad_norm": 1.3468742370605469
    },
    {
      "bits_per_byte": 2.3406023584663167,
      "learning_rate": 0.00029735129539309926,
      "loss_nats": 1.6223819255828857,
      "step": 880,
      "unclipped_grad_norm": 1.3746341466903687
    },
    {
      "bits_per_byte": 2.2996515694693733,
      "learning_rate": 0.0002972401645659207,
      "loss_nats": 1.5939970016479492,
      "step": 890,
      "unclipped_grad_norm": 1.5287024974822998
    },
    {
      "bits_per_byte": 2.338895602639154,
      "learning_rate": 0.00029712677411919283,
      "loss_nats": 1.6211988925933838,
      "step": 900,
      "unclipped_grad_norm": 1.2028048038482666
    },
    {
      "bits_per_byte": 2.322601450348459,
      "learning_rate": 0.0002970111259904454,
      "loss_nats": 1.6099046468734741,
      "step": 910,
      "unclipped_grad_norm": 1.5355594158172607
    },
    {
      "bits_per_byte": 2.307329906899794,
      "learning_rate": 0.0002968932221557859,
      "loss_nats": 1.5993192195892334,
      "step": 920,
      "unclipped_grad_norm": 1.5230286121368408
    },
    {
      "bits_per_byte": 2.2540851100885737,
      "learning_rate": 0.0002967730646298655,
      "loss_nats": 1.5624127388000488,
      "step": 930,
      "unclipped_grad_norm": 1.3632712364196777
    },
    {
      "bits_per_byte": 2.306159220995371,
      "learning_rate": 0.0002966506554658449,
      "loss_nats": 1.5985077619552612,
      "step": 940,
      "unclipped_grad_norm": 1.4536393880844116
    },
    {
      "bits_per_byte": 2.330239371855927,
      "learning_rate": 0.00029652599675535876,
      "loss_nats": 1.6151988506317139,
      "step": 950,
      "unclipped_grad_norm": 1.4820353984832764
    },
    {
      "bits_per_byte": 2.256347369878023,
      "learning_rate": 0.0002963990906284808,
      "loss_nats": 1.5639808177947998,
      "step": 960,
      "unclipped_grad_norm": 1.5091183185577393
    },
    {
      "bits_per_byte": 2.2135099312011146,
      "learning_rate": 0.0002962699392536863,
      "loss_nats": 1.5342881679534912,
      "step": 970,
      "unclipped_grad_norm": 1.3972843885421753
    },
    {
      "bits_per_byte": 2.3381242604500345,
      "learning_rate": 0.0002961385448378161,
      "loss_nats": 1.6206642389297485,
      "step": 980,
      "unclipped_grad_norm": 1.3591992855072021
    },
    {
      "bits_per_byte": 2.202303885636196,
      "learning_rate": 0.00029600490962603817,
      "loss_nats": 1.5265207290649414,
      "step": 990,
      "unclipped_grad_norm": 1.2794140577316284
    },
    {
      "bits_per_byte": 2.194666824041982,
      "learning_rate": 0.00029586903590180956,
      "loss_nats": 1.5212271213531494,
      "step": 1000,
      "unclipped_grad_norm": 1.2117714881896973
    },
    {
      "bits_per_byte": 2.205168084703665,
      "learning_rate": 0.00029573092598683735,
      "loss_nats": 1.5285060405731201,
      "step": 1010,
      "unclipped_grad_norm": 1.4173126220703125
    },
    {
      "bits_per_byte": 2.1627664500640433,
      "learning_rate": 0.00029559058224103895,
      "loss_nats": 1.4991154670715332,
      "step": 1020,
      "unclipped_grad_norm": 1.5233873128890991
    },
    {
      "bits_per_byte": 2.171931921476475,
      "learning_rate": 0.0002954480070625017,
      "loss_nats": 1.505468487739563,
      "step": 1030,
      "unclipped_grad_norm": 1.5871013402938843
    },
    {
      "bits_per_byte": 2.2533218510840447,
      "learning_rate": 0.00029530320288744205,
      "loss_nats": 1.5618836879730225,
      "step": 1040,
      "unclipped_grad_norm": 1.408625602722168
    },
    {
      "bits_per_byte": 2.203781560572409,
      "learning_rate": 0.0002951561721901638,
      "loss_nats": 1.5275449752807617,
      "step": 1050,
      "unclipped_grad_norm": 1.9470032453536987
    },
    {
      "bits_per_byte": 2.2074819392883733,
      "learning_rate": 0.00029500691748301594,
      "loss_nats": 1.5301098823547363,
      "step": 1060,
      "unclipped_grad_norm": 1.358320713043213
    },
    {
      "bits_per_byte": 2.2549842353872847,
      "learning_rate": 0.00029485544131634963,
      "loss_nats": 1.5630359649658203,
      "step": 1070,
      "unclipped_grad_norm": 1.3524690866470337
    },
    {
      "bits_per_byte": 2.232926944466177,
      "learning_rate": 0.0002947017462784745,
      "loss_nats": 1.547747015953064,
      "step": 1080,
      "unclipped_grad_norm": 1.3045321702957153
    },
    {
      "bits_per_byte": 2.1562297334700493,
      "learning_rate": 0.00029454583499561476,
      "loss_nats": 1.494584560394287,
      "step": 1090,
      "unclipped_grad_norm": 1.3637882471084595
    },
    {
      "bits_per_byte": 2.1766043461351154,
      "learning_rate": 0.00029438771013186413,
      "loss_nats": 1.5087071657180786,
      "step": 1100,
      "unclipped_grad_norm": 1.3327077627182007
    },
    {
      "bits_per_byte": 2.113153238908442,
      "learning_rate": 0.0002942273743891401,
      "loss_nats": 1.464726209640503,
      "step": 1110,
      "unclipped_grad_norm": 1.3556562662124634
    },
    {
      "bits_per_byte": 2.158234707213808,
      "learning_rate": 0.00029406483050713824,
      "loss_nats": 1.4959743022918701,
      "step": 1120,
      "unclipped_grad_norm": 1.5276546478271484
    },
    {
      "bits_per_byte": 2.1778080527085044,
      "learning_rate": 0.00029390008126328504,
      "loss_nats": 1.5095415115356445,
      "step": 1130,
      "unclipped_grad_norm": 1.3574931621551514
    },
    {
      "bits_per_byte": 2.120840347454057,
      "learning_rate": 0.00029373312947269045,
      "loss_nats": 1.4700545072555542,
      "step": 1140,
      "unclipped_grad_norm": 1.49063241481781
    },
    {
      "bits_per_byte": 2.172341240185528,
      "learning_rate": 0.00029356397798809995,
      "loss_nats": 1.5057522058486938,
      "step": 1150,
      "unclipped_grad_norm": 1.270938515663147
    },
    {
      "bits_per_byte": 2.081790654679751,
      "learning_rate": 0.0002933926296998457,
      "loss_nats": 1.442987322807312,
      "step": 1160,
      "unclipped_grad_norm": 1.387803077697754
    },
    {
      "bits_per_byte": 2.1352082940550883,
      "learning_rate": 0.0002932190875357972,
      "loss_nats": 1.4800136089324951,
      "step": 1170,
      "unclipped_grad_norm": 1.2486525774002075
    },
    {
      "bits_per_byte": 2.1410305947173978,
      "learning_rate": 0.00029304335446131126,
      "loss_nats": 1.4840493202209473,
      "step": 1180,
      "unclipped_grad_norm": 1.3077996969223022
    },
    {
      "bits_per_byte": 2.1241448221077346,
      "learning_rate": 0.00029286543347918116,
      "loss_nats": 1.472344994544983,
      "step": 1190,
      "unclipped_grad_norm": 1.846550464630127
    },
    {
      "bits_per_byte": 2.128028362346868,
      "learning_rate": 0.0002926853276295856,
      "loss_nats": 1.475036859512329,
      "step": 1200,
      "unclipped_grad_norm": 1.1341090202331543
    },
    {
      "bits_per_byte": 2.1484036629425374,
      "learning_rate": 0.0002925030399900368,
      "loss_nats": 1.4891599416732788,
      "step": 1210,
      "unclipped_grad_norm": 1.2377034425735474
    },
    {
      "bits_per_byte": 2.096246140450072,
      "learning_rate": 0.00029231857367532726,
      "loss_nats": 1.4530071020126343,
      "step": 1220,
      "unclipped_grad_norm": 1.182817816734314
    },
    {
      "bits_per_byte": 2.11190705262029,
      "learning_rate": 0.0002921319318374776,
      "loss_nats": 1.463862419128418,
      "step": 1230,
      "unclipped_grad_norm": 1.3176664113998413
    },
    {
      "bits_per_byte": 2.0884407078406633,
      "learning_rate": 0.00029194311766568174,
      "loss_nats": 1.447596788406372,
      "step": 1240,
      "unclipped_grad_norm": 1.3174140453338623
    },
    {
      "bits_per_byte": 2.01052448381522,
      "learning_rate": 0.000291752134386253,
      "loss_nats": 1.3935893774032593,
      "step": 1250,
      "unclipped_grad_norm": 1.3046211004257202
    },
    {
      "bits_per_byte": 2.0807407005962335,
      "learning_rate": 0.00029155898526256856,
      "loss_nats": 1.4422595500946045,
      "step": 1260,
      "unclipped_grad_norm": 1.3089001178741455
    },
    {
      "bits_per_byte": 2.0962698740558907,
      "learning_rate": 0.0002913636735950141,
      "loss_nats": 1.4530235528945923,
      "step": 1270,
      "unclipped_grad_norm": 1.318946361541748
    },
    {
      "bits_per_byte": 2.1189218809836823,
      "learning_rate": 0.0002911662027209272,
      "loss_nats": 1.4687247276306152,
      "step": 1280,
      "unclipped_grad_norm": 1.1660429239273071
    },
    {
      "bits_per_byte": 2.0943455601753866,
      "learning_rate": 0.00029096657601454013,
      "loss_nats": 1.4516897201538086,
      "step": 1290,
      "unclipped_grad_norm": 1.5281683206558228
    },
    {
      "bits_per_byte": 2.0472271293532014,
      "learning_rate": 0.00029076479688692266,
      "loss_nats": 1.419029712677002,
      "step": 1300,
      "unclipped_grad_norm": 1.416712999343872
    },
    {
      "bits_per_byte": 2.0644360573638356,
      "learning_rate": 0.00029056086878592323,
      "loss_nats": 1.4309580326080322,
      "step": 1310,
      "unclipped_grad_norm": 1.2950114011764526
    },
    {
      "bits_per_byte": 2.044286569988749,
      "learning_rate": 0.00029035479519611045,
      "loss_nats": 1.4169914722442627,
      "step": 1320,
      "unclipped_grad_norm": 1.1484317779541016
    },
    {
      "bits_per_byte": 2.0443660259734475,
      "learning_rate": 0.0002901465796387134,
      "loss_nats": 1.4170465469360352,
      "step": 1330,
      "unclipped_grad_norm": 1.206559181213379
    },
    {
      "bits_per_byte": 2.040463051694767,
      "learning_rate": 0.00028993622567156154,
      "loss_nats": 1.4143412113189697,
      "step": 1340,
      "unclipped_grad_norm": 1.1644346714019775
    },
    {
      "bits_per_byte": 2.073596541279392,
      "learning_rate": 0.0002897237368890237,
      "loss_nats": 1.437307596206665,
      "step": 1350,
      "unclipped_grad_norm": 1.295727014541626
    },
    {
      "bits_per_byte": 2.028423750186418,
      "learning_rate": 0.0002895091169219469,
      "loss_nats": 1.4059962034225464,
      "step": 1360,
      "unclipped_grad_norm": 1.2224998474121094
    },
    {
      "bits_per_byte": 2.049744783379182,
      "learning_rate": 0.00028929236943759425,
      "loss_nats": 1.4207748174667358,
      "step": 1370,
      "unclipped_grad_norm": 1.2044527530670166
    },
    {
      "bits_per_byte": 2.01407248590252,
      "learning_rate": 0.00028907349813958225,
      "loss_nats": 1.396048665046692,
      "step": 1380,
      "unclipped_grad_norm": 1.241443157196045
    },
    {
      "bits_per_byte": 2.0615071928196422,
      "learning_rate": 0.0002888525067678175,
      "loss_nats": 1.4289278984069824,
      "step": 1390,
      "unclipped_grad_norm": 1.1473987102508545
    },
    {
      "bits_per_byte": 2.1662895146669676,
      "learning_rate": 0.00028862939909843273,
      "loss_nats": 1.501557469367981,
      "step": 1400,
      "unclipped_grad_norm": 1.1963618993759155
    },
    {
      "bits_per_byte": 2.064517577140345,
      "learning_rate": 0.0002884041789437225,
      "loss_nats": 1.4310145378112793,
      "step": 1410,
      "unclipped_grad_norm": 1.5827157497406006
    },
    {
      "bits_per_byte": 2.0563607839752236,
      "learning_rate": 0.0002881768501520778,
      "loss_nats": 1.4253606796264648,
      "step": 1420,
      "unclipped_grad_norm": 1.2223409414291382
    },
    {
      "bits_per_byte": 1.996439276692207,
      "learning_rate": 0.0002879474166079205,
      "loss_nats": 1.3838262557983398,
      "step": 1430,
      "unclipped_grad_norm": 1.1598049402236938
    },
    {
      "bits_per_byte": 2.0215600945731342,
      "learning_rate": 0.00028771588223163694,
      "loss_nats": 1.4012386798858643,
      "step": 1440,
      "unclipped_grad_norm": 1.1466177701950073
    },
    {
      "bits_per_byte": 2.0019069491284376,
      "learning_rate": 0.00028748225097951054,
      "loss_nats": 1.3876161575317383,
      "step": 1450,
      "unclipped_grad_norm": 1.1313462257385254
    },
    {
      "bits_per_byte": 2.1132414660083345,
      "learning_rate": 0.000287246526843655,
      "loss_nats": 1.4647873640060425,
      "step": 1460,
      "unclipped_grad_norm": 1.1495121717453003
    },
    {
      "bits_per_byte": 2.102186421210873,
      "learning_rate": 0.00028700871385194533,
      "loss_nats": 1.4571245908737183,
      "step": 1470,
      "unclipped_grad_norm": 1.337440013885498
    },
    {
      "bits_per_byte": 2.0400563127254765,
      "learning_rate": 0.00028676881606794945,
      "loss_nats": 1.4140592813491821,
      "step": 1480,
      "unclipped_grad_norm": 1.06326162815094
    },
    {
      "bits_per_byte": 2.057447370363375,
      "learning_rate": 0.00028652683759085854,
      "loss_nats": 1.4261138439178467,
      "step": 1490,
      "unclipped_grad_norm": 1.1691441535949707
    },
    {
      "bits_per_byte": 1.9426222936015662,
      "learning_rate": 0.0002862827825554171,
      "loss_nats": 1.3465231657028198,
      "step": 1500,
      "unclipped_grad_norm": 1.0966376066207886
    },
    {
      "bits_per_byte": 1.953619208263036,
      "learning_rate": 0.00028603665513185245,
      "loss_nats": 1.3541456460952759,
      "step": 1510,
      "unclipped_grad_norm": 1.2253862619400024
    },
    {
      "bits_per_byte": 1.9856435817322469,
      "learning_rate": 0.000285788459525803,
      "loss_nats": 1.3763432502746582,
      "step": 1520,
      "unclipped_grad_norm": 1.1588174104690552
    },
    {
      "bits_per_byte": 2.043245215037775,
      "learning_rate": 0.00028553819997824707,
      "loss_nats": 1.4162696599960327,
      "step": 1530,
      "unclipped_grad_norm": 1.1396243572235107
    },
    {
      "bits_per_byte": 2.030725050037613,
      "learning_rate": 0.00028528588076542964,
      "loss_nats": 1.4075913429260254,
      "step": 1540,
      "unclipped_grad_norm": 1.1333942413330078
    },
    {
      "bits_per_byte": 2.081739919797746,
      "learning_rate": 0.00028503150619878996,
      "loss_nats": 1.4429521560668945,
      "step": 1550,
      "unclipped_grad_norm": 1.1568241119384766
    },
    {
      "bits_per_byte": 1.9638389333252566,
      "learning_rate": 0.0002847750806248876,
      "loss_nats": 1.361229419708252,
      "step": 1560,
      "unclipped_grad_norm": 1.0923500061035156
    },
    {
      "bits_per_byte": 1.9637097743544585,
      "learning_rate": 0.000284516608425328,
      "loss_nats": 1.3611398935317993,
      "step": 1570,
      "unclipped_grad_norm": 1.181723952293396
    },
    {
      "bits_per_byte": 1.984943956308537,
      "learning_rate": 0.000284256094016688,
      "loss_nats": 1.3758583068847656,
      "step": 1580,
      "unclipped_grad_norm": 1.15809965133667
    },
    {
      "bits_per_byte": 1.9967120411764756,
      "learning_rate": 0.0002839935418504399,
      "loss_nats": 1.3840153217315674,
      "step": 1590,
      "unclipped_grad_norm": 1.2961064577102661
    },
    {
      "bits_per_byte": 1.9713934432470563,
      "learning_rate": 0.000283728956412876,
      "loss_nats": 1.3664658069610596,
      "step": 1600,
      "unclipped_grad_norm": 1.1767901182174683
    },
    {
      "bits_per_byte": 1.9459826626167747,
      "learning_rate": 0.00028346234222503136,
      "loss_nats": 1.3488523960113525,
      "step": 1610,
      "unclipped_grad_norm": 1.1538722515106201
    },
    {
      "bits_per_byte": 1.9305324291938633,
      "learning_rate": 0.00028319370384260675,
      "loss_nats": 1.3381431102752686,
      "step": 1620,
      "unclipped_grad_norm": 1.0682140588760376
    },
    {
      "bits_per_byte": 1.9835519287324535,
      "learning_rate": 0.00028292304585589105,
      "loss_nats": 1.3748934268951416,
      "step": 1630,
      "unclipped_grad_norm": 1.1328599452972412
    },
    {
      "bits_per_byte": 1.952673819631243,
      "learning_rate": 0.0002826503728896824,
      "loss_nats": 1.3534903526306152,
      "step": 1640,
      "unclipped_grad_norm": 1.1611915826797485
    },
    {
      "bits_per_byte": 1.9528083100642177,
      "learning_rate": 0.0002823756896032095,
      "loss_nats": 1.353583574295044,
      "step": 1650,
      "unclipped_grad_norm": 1.2886978387832642
    },
    {
      "bits_per_byte": 1.9660846827802232,
      "learning_rate": 0.0002820990006900518,
      "loss_nats": 1.362786054611206,
      "step": 1660,
      "unclipped_grad_norm": 1.1841517686843872
    },
    {
      "bits_per_byte": 1.9119278619714772,
      "learning_rate": 0.00028182031087805947,
      "loss_nats": 1.3252474069595337,
      "step": 1670,
      "unclipped_grad_norm": 1.1872814893722534
    },
    {
      "bits_per_byte": 1.93973676868539,
      "learning_rate": 0.0002815396249292725,
      "loss_nats": 1.3445230722427368,
      "step": 1680,
      "unclipped_grad_norm": 1.1895164251327515
    },
    {
      "bits_per_byte": 1.9710771671521197,
      "learning_rate": 0.0002812569476398391,
      "loss_nats": 1.3662465810775757,
      "step": 1690,
      "unclipped_grad_norm": 1.0209845304489136
    },
    {
      "bits_per_byte": 1.9189868898761413,
      "learning_rate": 0.00028097228383993446,
      "loss_nats": 1.3301403522491455,
      "step": 1700,
      "unclipped_grad_norm": 1.2754039764404297
    },
    {
      "bits_per_byte": 1.9587719804655366,
      "learning_rate": 0.0002806856383936774,
      "loss_nats": 1.3577172756195068,
      "step": 1710,
      "unclipped_grad_norm": 0.9888415336608887
    },
    {
      "bits_per_byte": 1.8892168649944399,
      "learning_rate": 0.00028039701619904774,
      "loss_nats": 1.3095053434371948,
      "step": 1720,
      "unclipped_grad_norm": 1.1411211490631104
    },
    {
      "bits_per_byte": 1.970308060737461,
      "learning_rate": 0.0002801064221878024,
      "loss_nats": 1.3657134771347046,
      "step": 1730,
      "unclipped_grad_norm": 1.1278765201568604
    },
    {
      "bits_per_byte": 1.9812284431192968,
      "learning_rate": 0.00027981386132539134,
      "loss_nats": 1.3732829093933105,
      "step": 1740,
      "unclipped_grad_norm": 1.0859284400939941
    },
    {
      "bits_per_byte": 1.9480999410315467,
      "learning_rate": 0.0002795193386108724,
      "loss_nats": 1.3503199815750122,
      "step": 1750,
      "unclipped_grad_norm": 1.063671588897705
    },
    {
      "bits_per_byte": 1.9687923776354073,
      "learning_rate": 0.00027922285907682625,
      "loss_nats": 1.3646628856658936,
      "step": 1760,
      "unclipped_grad_norm": 1.02903151512146
    },
    {
      "bits_per_byte": 1.9746323925107556,
      "learning_rate": 0.00027892442778927005,
      "loss_nats": 1.3687108755111694,
      "step": 1770,
      "unclipped_grad_norm": 1.1287299394607544
    },
    {
      "bits_per_byte": 2.0187137817013547,
      "learning_rate": 0.00027862404984757116,
      "loss_nats": 1.3992657661437988,
      "step": 1780,
      "unclipped_grad_norm": 1.1014562845230103
    },
    {
      "bits_per_byte": 1.8992608237874788,
      "learning_rate": 0.0002783217303843597,
      "loss_nats": 1.31646728515625,
      "step": 1790,
      "unclipped_grad_norm": 1.2110038995742798
    },
    {
      "bits_per_byte": 2.0498698147663594,
      "learning_rate": 0.00027801747456544134,
      "loss_nats": 1.4208614826202393,
      "step": 1800,
      "unclipped_grad_norm": 1.5255318880081177
    },
    {
      "bits_per_byte": 1.8942502812372417,
      "learning_rate": 0.00027771128758970843,
      "loss_nats": 1.3129942417144775,
      "step": 1810,
      "unclipped_grad_norm": 1.0947346687316895
    },
    {
      "bits_per_byte": 1.9627021280030534,
      "learning_rate": 0.00027740317468905165,
      "loss_nats": 1.3604414463043213,
      "step": 1820,
      "unclipped_grad_norm": 1.1853052377700806
    },
    {
      "bits_per_byte": 1.9780426364947141,
      "learning_rate": 0.0002770931411282704,
      "loss_nats": 1.3710746765136719,
      "step": 1830,
      "unclipped_grad_norm": 1.0857312679290771
    },
    {
      "bits_per_byte": 1.975353859731125,
      "learning_rate": 0.0002767811922049827,
      "loss_nats": 1.369210958480835,
      "step": 1840,
      "unclipped_grad_norm": 1.1014374494552612
    },
    {
      "bits_per_byte": 1.8969483450639775,
      "learning_rate": 0.00027646733324953503,
      "loss_nats": 1.3148643970489502,
      "step": 1850,
      "unclipped_grad_norm": 1.0865846872329712
    },
    {
      "bits_per_byte": 1.943441103002324,
      "learning_rate": 0.00027615156962491097,
      "loss_nats": 1.347090721130371,
      "step": 1860,
      "unclipped_grad_norm": 0.9979931116104126
    },
    {
      "bits_per_byte": 2.018713093770751,
      "learning_rate": 0.0002758339067266396,
      "loss_nats": 1.3992652893066406,
      "step": 1870,
      "unclipped_grad_norm": 1.1247007846832275
    },
    {
      "bits_per_byte": 1.895716605318498,
      "learning_rate": 0.0002755143499827035,
      "loss_nats": 1.3140106201171875,
      "step": 1880,
      "unclipped_grad_norm": 1.2656805515289307
    },
    {
      "bits_per_byte": 1.939341724536358,
      "learning_rate": 0.0002751929048534456,
      "loss_nats": 1.3442492485046387,
      "step": 1890,
      "unclipped_grad_norm": 1.1298930644989014
    },
    {
      "bits_per_byte": 1.9376002282137212,
      "learning_rate": 0.0002748695768314763,
      "loss_nats": 1.3430421352386475,
      "step": 1900,
      "unclipped_grad_norm": 1.1912769079208374
    },
    {
      "bits_per_byte": 1.9616757355427041,
      "learning_rate": 0.00027454437144157947,
      "loss_nats": 1.3597300052642822,
      "step": 1910,
      "unclipped_grad_norm": 1.0634745359420776
    },
    {
      "bits_per_byte": 1.8880833273426032,
      "learning_rate": 0.00027421729424061787,
      "loss_nats": 1.3087196350097656,
      "step": 1920,
      "unclipped_grad_norm": 1.0458215475082397
    },
    {
      "bits_per_byte": 1.9484389188363973,
      "learning_rate": 0.0002738883508174385,
      "loss_nats": 1.3505549430847168,
      "step": 1930,
      "unclipped_grad_norm": 1.0484954118728638
    },
    {
      "bits_per_byte": 1.9728711181832694,
      "learning_rate": 0.0002735575467927768,
      "loss_nats": 1.3674900531768799,
      "step": 1940,
      "unclipped_grad_norm": 1.0497392416000366
    },
    {
      "bits_per_byte": 1.9061248233660644,
      "learning_rate": 0.0002732248878191609,
      "loss_nats": 1.3212250471115112,
      "step": 1950,
      "unclipped_grad_norm": 1.2195848226547241
    },
    {
      "bits_per_byte": 1.8858399856447488,
      "learning_rate": 0.0002728903795808148,
      "loss_nats": 1.3071646690368652,
      "step": 1960,
      "unclipped_grad_norm": 1.0152307748794556
    },
    {
      "bits_per_byte": 1.8880905506139394,
      "learning_rate": 0.0002725540277935614,
      "loss_nats": 1.3087246417999268,
      "step": 1970,
      "unclipped_grad_norm": 1.1332975625991821
    },
    {
      "bits_per_byte": 1.9225495104887642,
      "learning_rate": 0.00027221583820472456,
      "loss_nats": 1.33260977268219,
      "step": 1980,
      "unclipped_grad_norm": 1.1374293565750122
    },
    {
      "bits_per_byte": 1.8947043154355196,
      "learning_rate": 0.0002718758165930314,
      "loss_nats": 1.3133089542388916,
      "step": 1990,
      "unclipped_grad_norm": 1.0456688404083252
    },
    {
      "bits_per_byte": 1.977381019236845,
      "learning_rate": 0.0002715339687685131,
      "loss_nats": 1.37061607837677,
      "step": 2000,
      "unclipped_grad_norm": 1.1185020208358765
    },
    {
      "bits_per_byte": 1.9636655748131868,
      "learning_rate": 0.00027119030057240576,
      "loss_nats": 1.3611092567443848,
      "step": 2010,
      "unclipped_grad_norm": 1.1275893449783325
    },
    {
      "bits_per_byte": 1.8762688071589317,
      "learning_rate": 0.0002708448178770506,
      "loss_nats": 1.3005304336547852,
      "step": 2020,
      "unclipped_grad_norm": 1.0362930297851562
    },
    {
      "bits_per_byte": 1.8977705941177525,
      "learning_rate": 0.00027049752658579376,
      "loss_nats": 1.3154343366622925,
      "step": 2030,
      "unclipped_grad_norm": 1.0457504987716675
    },
    {
      "bits_per_byte": 1.8942691993288368,
      "learning_rate": 0.00027014843263288514,
      "loss_nats": 1.3130073547363281,
      "step": 2040,
      "unclipped_grad_norm": 0.9790334105491638
    },
    {
      "bits_per_byte": 1.854481013051726,
      "learning_rate": 0.00026979754198337724,
      "loss_nats": 1.2854282855987549,
      "step": 2050,
      "unclipped_grad_norm": 1.067064881324768
    },
    {
      "bits_per_byte": 1.8548413167052835,
      "learning_rate": 0.00026944486063302305,
      "loss_nats": 1.2856780290603638,
      "step": 2060,
      "unclipped_grad_norm": 1.0005096197128296
    },
    {
      "bits_per_byte": 1.9705749778116002,
      "learning_rate": 0.0002690903946081738,
      "loss_nats": 1.3658984899520874,
      "step": 2070,
      "unclipped_grad_norm": 1.1014602184295654
    },
    {
      "bits_per_byte": 1.9435265783798028,
      "learning_rate": 0.00026873414996567596,
      "loss_nats": 1.3471499681472778,
      "step": 2080,
      "unclipped_grad_norm": 1.1029977798461914
    },
    {
      "bits_per_byte": 1.8586646630166146,
      "learning_rate": 0.00026837613279276736,
      "loss_nats": 1.2883281707763672,
      "step": 2090,
      "unclipped_grad_norm": 1.0793615579605103
    },
    {
      "bits_per_byte": 1.8191048696998495,
      "learning_rate": 0.0002680163492069739,
      "loss_nats": 1.2609074115753174,
      "step": 2100,
      "unclipped_grad_norm": 1.0433158874511719
    },
    {
      "bits_per_byte": 1.851254274556238,
      "learning_rate": 0.00026765480535600414,
      "loss_nats": 1.2831916809082031,
      "step": 2110,
      "unclipped_grad_norm": 1.1101045608520508
    },
    {
      "bits_per_byte": 1.8581610978148881,
      "learning_rate": 0.0002672915074176452,
      "loss_nats": 1.2879791259765625,
      "step": 2120,
      "unclipped_grad_norm": 1.1086524724960327
    },
    {
      "bits_per_byte": 1.9049420986760808,
      "learning_rate": 0.00026692646159965624,
      "loss_nats": 1.3204052448272705,
      "step": 2130,
      "unclipped_grad_norm": 1.0729707479476929
    },
    {
      "bits_per_byte": 1.941282376768694,
      "learning_rate": 0.0002665596741396633,
      "loss_nats": 1.3455944061279297,
      "step": 2140,
      "unclipped_grad_norm": 1.0528757572174072
    },
    {
      "bits_per_byte": 1.843757550805128,
      "learning_rate": 0.000266191151305052,
      "loss_nats": 1.2779953479766846,
      "step": 2150,
      "unclipped_grad_norm": 1.1246051788330078
    },
    {
      "bits_per_byte": 1.9154739722496175,
      "learning_rate": 0.00026582089939286075,
      "loss_nats": 1.3277053833007812,
      "step": 2160,
      "unclipped_grad_norm": 1.0607515573501587
    },
    {
      "bits_per_byte": 1.850249895875199,
      "learning_rate": 0.0002654489247296732,
      "loss_nats": 1.2824954986572266,
      "step": 2170,
      "unclipped_grad_norm": 1.0301073789596558
    },
    {
      "bits_per_byte": 1.9307078514977434,
      "learning_rate": 0.00026507523367151007,
      "loss_nats": 1.3382647037506104,
      "step": 2180,
      "unclipped_grad_norm": 1.0744229555130005
    },
    {
      "bits_per_byte": 1.895962540509232,
      "learning_rate": 0.00026469983260372037,
      "loss_nats": 1.3141810894012451,
      "step": 2190,
      "unclipped_grad_norm": 1.1302496194839478
    },
    {
      "bits_per_byte": 1.8514286649642127,
      "learning_rate": 0.0002643227279408727,
      "loss_nats": 1.2833125591278076,
      "step": 2200,
      "unclipped_grad_norm": 1.043583869934082
    },
    {
      "bits_per_byte": 1.8687825743495246,
      "learning_rate": 0.0002639439261266453,
      "loss_nats": 1.2953413724899292,
      "step": 2210,
      "unclipped_grad_norm": 1.1252996921539307
    },
    {
      "bits_per_byte": 1.862911258631719,
      "learning_rate": 0.000263563433633716,
      "loss_nats": 1.291271686553955,
      "step": 2220,
      "unclipped_grad_norm": 1.0556633472442627
    },
    {
      "bits_per_byte": 1.8208848901362797,
      "learning_rate": 0.00026318125696365156,
      "loss_nats": 1.262141227722168,
      "step": 2230,
      "unclipped_grad_norm": 1.0693137645721436
    },
    {
      "bits_per_byte": 1.90406292336487,
      "learning_rate": 0.000262797402646797,
      "loss_nats": 1.319795846939087,
      "step": 2240,
      "unclipped_grad_norm": 1.0752713680267334
    },
    {
      "bits_per_byte": 1.8964874315596651,
      "learning_rate": 0.00026241187724216334,
      "loss_nats": 1.314544916152954,
      "step": 2250,
      "unclipped_grad_norm": 1.0330802202224731
    },
    {
      "bits_per_byte": 1.935825367256817,
      "learning_rate": 0.0002620246873373162,
      "loss_nats": 1.3418118953704834,
      "step": 2260,
      "unclipped_grad_norm": 1.078423023223877
    },
    {
      "bits_per_byte": 1.832309869615747,
      "learning_rate": 0.00026163583954826264,
      "loss_nats": 1.270060420036316,
      "step": 2270,
      "unclipped_grad_norm": 1.056177020072937
    },
    {
      "bits_per_byte": 1.887026837918353,
      "learning_rate": 0.0002612453405193386,
      "loss_nats": 1.3079873323440552,
      "step": 2280,
      "unclipped_grad_norm": 1.0121575593948364
    },
    {
      "bits_per_byte": 1.834445722156812,
      "learning_rate": 0.000260853196923095,
      "loss_nats": 1.271540880203247,
      "step": 2290,
      "unclipped_grad_norm": 1.0369646549224854
    },
    {
      "bits_per_byte": 1.8213642057842343,
      "learning_rate": 0.0002604594154601839,
      "loss_nats": 1.262473464012146,
      "step": 2300,
      "unclipped_grad_norm": 0.9950895309448242
    },
    {
      "bits_per_byte": 1.8373876573824712,
      "learning_rate": 0.0002600640028592441,
      "loss_nats": 1.2735800743103027,
      "step": 2310,
      "unclipped_grad_norm": 1.0178841352462769
    },
    {
      "bits_per_byte": 1.8979250345382275,
      "learning_rate": 0.0002596669658767859,
      "loss_nats": 1.315541386604309,
      "step": 2320,
      "unclipped_grad_norm": 0.9894719123840332
    },
    {
      "bits_per_byte": 1.8294843666447218,
      "learning_rate": 0.000259268311297076,
      "loss_nats": 1.2681019306182861,
      "step": 2330,
      "unclipped_grad_norm": 0.9816614389419556
    },
    {
      "bits_per_byte": 1.848164090285535,
      "learning_rate": 0.00025886804593202114,
      "loss_nats": 1.2810497283935547,
      "step": 2340,
      "unclipped_grad_norm": 1.0238531827926636
    },
    {
      "bits_per_byte": 1.905264050198496,
      "learning_rate": 0.00025846617662105193,
      "loss_nats": 1.3206284046173096,
      "step": 2350,
      "unclipped_grad_norm": 1.0559433698654175
    },
    {
      "bits_per_byte": 1.8307938425483912,
      "learning_rate": 0.00025806271023100637,
      "loss_nats": 1.2690095901489258,
      "step": 2360,
      "unclipped_grad_norm": 1.0712460279464722
    },
    {
      "bits_per_byte": 1.7703326537069675,
      "learning_rate": 0.00025765765365601165,
      "loss_nats": 1.2271010875701904,
      "step": 2370,
      "unclipped_grad_norm": 1.1017504930496216
    },
    {
      "bits_per_byte": 1.886526368404342,
      "learning_rate": 0.00025725101381736707,
      "loss_nats": 1.3076404333114624,
      "step": 2380,
      "unclipped_grad_norm": 1.1038753986358643
    },
    {
      "bits_per_byte": 1.8630619154338748,
      "learning_rate": 0.00025684279766342567,
      "loss_nats": 1.2913761138916016,
      "step": 2390,
      "unclipped_grad_norm": 1.0329313278198242
    },
    {
      "bits_per_byte": 1.9135943738583379,
      "learning_rate": 0.00025643301216947504,
      "loss_nats": 1.3264025449752808,
      "step": 2400,
      "unclipped_grad_norm": 1.1860445737838745
    },
    {
      "bits_per_byte": 1.835037686441082,
      "learning_rate": 0.00025602166433761866,
      "loss_nats": 1.2719511985778809,
      "step": 2410,
      "unclipped_grad_norm": 1.153725266456604
    },
    {
      "bits_per_byte": 1.871177432762789,
      "learning_rate": 0.0002556087611966561,
      "loss_nats": 1.2970013618469238,
      "step": 2420,
      "unclipped_grad_norm": 1.0672882795333862
    },
    {
      "bits_per_byte": 1.8586395535495885,
      "learning_rate": 0.00025519430980196277,
      "loss_nats": 1.2883107662200928,
      "step": 2430,
      "unclipped_grad_norm": 1.0162642002105713
    },
    {
      "bits_per_byte": 1.8218378460047104,
      "learning_rate": 0.0002547783172353695,
      "loss_nats": 1.2628017663955688,
      "step": 2440,
      "unclipped_grad_norm": 1.065905213356018
    },
    {
      "bits_per_byte": 1.9384147380482075,
      "learning_rate": 0.0002543607906050415,
      "loss_nats": 1.34360671043396,
      "step": 2450,
      "unclipped_grad_norm": 1.218255639076233
    },
    {
      "bits_per_byte": 1.8158471743272062,
      "learning_rate": 0.0002539417370453568,
      "loss_nats": 1.2586493492126465,
      "step": 2460,
      "unclipped_grad_norm": 1.0468804836273193
    },
    {
      "bits_per_byte": 1.8206396428761493,
      "learning_rate": 0.00025352116371678464,
      "loss_nats": 1.2619712352752686,
      "step": 2470,
      "unclipped_grad_norm": 1.1381754875183105
    },
    {
      "bits_per_byte": 1.8851444878046595,
      "learning_rate": 0.0002530990778057627,
      "loss_nats": 1.3066825866699219,
      "step": 2480,
      "unclipped_grad_norm": 1.035730242729187
    },
    {
      "bits_per_byte": 1.8941151028736636,
      "learning_rate": 0.00025267548652457475,
      "loss_nats": 1.3129005432128906,
      "step": 2490,
      "unclipped_grad_norm": 1.0220366716384888
    },
    {
      "bits_per_byte": 1.8318142156159603,
      "learning_rate": 0.00025225039711122696,
      "loss_nats": 1.2697168588638306,
      "step": 2500,
      "unclipped_grad_norm": 0.982593834400177
    },
    {
      "bits_per_byte": 1.8129355080480989,
      "learning_rate": 0.00025182381682932464,
      "loss_nats": 1.2566311359405518,
      "step": 2510,
      "unclipped_grad_norm": 1.091998815536499
    },
    {
      "bits_per_byte": 1.841321932503609,
      "learning_rate": 0.0002513957529679477,
      "loss_nats": 1.2763071060180664,
      "step": 2520,
      "unclipped_grad_norm": 1.000153660774231
    },
    {
      "bits_per_byte": 1.848013777448681,
      "learning_rate": 0.00025096621284152656,
      "loss_nats": 1.2809455394744873,
      "step": 2530,
      "unclipped_grad_norm": 1.0528815984725952
    },
    {
      "bits_per_byte": 1.8343394368785788,
      "learning_rate": 0.0002505352037897168,
      "loss_nats": 1.2714672088623047,
      "step": 2540,
      "unclipped_grad_norm": 0.9793087840080261
    },
    {
      "bits_per_byte": 1.7887870800751502,
      "learning_rate": 0.0002501027331772739,
      "loss_nats": 1.2398927211761475,
      "step": 2550,
      "unclipped_grad_norm": 1.0650514364242554
    },
    {
      "bits_per_byte": 1.7584782335482958,
      "learning_rate": 0.00024966880839392746,
      "loss_nats": 1.2188842296600342,
      "step": 2560,
      "unclipped_grad_norm": 0.9875280857086182
    },
    {
      "bits_per_byte": 1.81654955147333,
      "learning_rate": 0.0002492334368542548,
      "loss_nats": 1.2591361999511719,
      "step": 2570,
      "unclipped_grad_norm": 1.0060385465621948
    },
    {
      "bits_per_byte": 1.7945154782100887,
      "learning_rate": 0.00024879662599755416,
      "loss_nats": 1.2438633441925049,
      "step": 2580,
      "unclipped_grad_norm": 1.0824097394943237
    },
    {
      "bits_per_byte": 1.7956424805211926,
      "learning_rate": 0.00024835838328771803,
      "loss_nats": 1.2446445226669312,
      "step": 2590,
      "unclipped_grad_norm": 0.9988609552383423
    },
    {
      "bits_per_byte": 1.815006351147138,
      "learning_rate": 0.0002479187162131051,
      "loss_nats": 1.2580665349960327,
      "step": 2600,
      "unclipped_grad_norm": 0.9940101504325867
    },
    {
      "bits_per_byte": 1.840550934279791,
      "learning_rate": 0.00024747763228641264,
      "loss_nats": 1.2757726907730103,
      "step": 2610,
      "unclipped_grad_norm": 1.0804052352905273
    },
    {
      "bits_per_byte": 1.7607050649116673,
      "learning_rate": 0.00024703513904454795,
      "loss_nats": 1.2204277515411377,
      "step": 2620,
      "unclipped_grad_norm": 1.0194178819656372
    },
    {
      "bits_per_byte": 1.7479795526263755,
      "learning_rate": 0.00024659124404849956,
      "loss_nats": 1.2116070985794067,
      "step": 2630,
      "unclipped_grad_norm": 0.9564489126205444
    },
    {
      "bits_per_byte": 1.7532458333784458,
      "learning_rate": 0.00024614595488320833,
      "loss_nats": 1.2152574062347412,
      "step": 2640,
      "unclipped_grad_norm": 0.9848498106002808
    },
    {
      "bits_per_byte": 1.813315245741204,
      "learning_rate": 0.00024569927915743745,
      "loss_nats": 1.2568943500518799,
      "step": 2650,
      "unclipped_grad_norm": 0.9513642191886902
    },
    {
      "bits_per_byte": 1.7231679595517,
      "learning_rate": 0.0002452512245036427,
      "loss_nats": 1.1944090127944946,
      "step": 2660,
      "unclipped_grad_norm": 1.0000529289245605
    },
    {
      "bits_per_byte": 1.802969457395901,
      "learning_rate": 0.00024480179857784177,
      "loss_nats": 1.249723196029663,
      "step": 2670,
      "unclipped_grad_norm": 0.9922699928283691
    },
    {
      "bits_per_byte": 1.9163316497294702,
      "learning_rate": 0.00024435100905948387,
      "loss_nats": 1.328299880027771,
      "step": 2680,
      "unclipped_grad_norm": 1.192367672920227
    },
    {
      "bits_per_byte": 1.8243539521868333,
      "learning_rate": 0.00024389886365131793,
      "loss_nats": 1.2645457983016968,
      "step": 2690,
      "unclipped_grad_norm": 1.0074772834777832
    },
    {
      "bits_per_byte": 1.751833683832211,
      "learning_rate": 0.00024344537007926163,
      "loss_nats": 1.2142785787582397,
      "step": 2700,
      "unclipped_grad_norm": 0.9746342301368713
    },
    {
      "bits_per_byte": 1.826312318632208,
      "learning_rate": 0.00024299053609226878,
      "loss_nats": 1.2659032344818115,
      "step": 2710,
      "unclipped_grad_norm": 1.0221014022827148
    },
    {
      "bits_per_byte": 1.8721807795479226,
      "learning_rate": 0.00024253436946219733,
      "loss_nats": 1.297696828842163,
      "step": 2720,
      "unclipped_grad_norm": 1.0561023950576782
    },
    {
      "bits_per_byte": 1.742347808741222,
      "learning_rate": 0.0002420768779836764,
      "loss_nats": 1.2077034711837769,
      "step": 2730,
      "unclipped_grad_norm": 0.946500301361084
    },
    {
      "bits_per_byte": 1.826287381147833,
      "learning_rate": 0.00024161806947397305,
      "loss_nats": 1.2658859491348267,
      "step": 2740,
      "unclipped_grad_norm": 1.0504826307296753
    },
    {
      "bits_per_byte": 1.7872662374935704,
      "learning_rate": 0.00024115795177285886,
      "loss_nats": 1.23883855342865,
      "step": 2750,
      "unclipped_grad_norm": 1.0072195529937744
    },
    {
      "bits_per_byte": 1.9127833036768689,
      "learning_rate": 0.0002406965327424758,
      "loss_nats": 1.3258403539657593,
      "step": 2760,
      "unclipped_grad_norm": 1.1406866312026978
    },
    {
      "bits_per_byte": 1.8309943743192973,
      "learning_rate": 0.00024023382026720197,
      "loss_nats": 1.269148588180542,
      "step": 2770,
      "unclipped_grad_norm": 1.0432147979736328
    },
    {
      "bits_per_byte": 1.835324725485372,
      "learning_rate": 0.00023976982225351698,
      "loss_nats": 1.2721501588821411,
      "step": 2780,
      "unclipped_grad_norm": 1.0946613550186157
    },
    {
      "bits_per_byte": 1.8014186898330709,
      "learning_rate": 0.0002393045466298666,
      "loss_nats": 1.2486482858657837,
      "step": 2790,
      "unclipped_grad_norm": 1.0307797193527222
    },
    {
      "bits_per_byte": 1.8122052697125355,
      "learning_rate": 0.00023883800134652763,
      "loss_nats": 1.2561249732971191,
      "step": 2800,
      "unclipped_grad_norm": 1.0519623756408691
    },
    {
      "bits_per_byte": 1.7642310532196568,
      "learning_rate": 0.0002383701943754715,
      "loss_nats": 1.2228717803955078,
      "step": 2810,
      "unclipped_grad_norm": 1.0939170122146606
    },
    {
      "bits_per_byte": 1.8221561858914574,
      "learning_rate": 0.00023790113371022884,
      "loss_nats": 1.2630224227905273,
      "step": 2820,
      "unclipped_grad_norm": 1.006143569946289
    },
    {
      "bits_per_byte": 1.7646739085456284,
      "learning_rate": 0.0002374308273657521,
      "loss_nats": 1.223178744316101,
      "step": 2830,
      "unclipped_grad_norm": 0.9335127472877502
    },
    {
      "bits_per_byte": 1.8147093371090979,
      "learning_rate": 0.00023695928337827911,
      "loss_nats": 1.2578606605529785,
      "step": 2840,
      "unclipped_grad_norm": 0.9985573887825012
    },
    {
      "bits_per_byte": 1.758499215431701,
      "learning_rate": 0.00023648650980519564,
      "loss_nats": 1.2188987731933594,
      "step": 2850,
      "unclipped_grad_norm": 0.9980565905570984
    },
    {
      "bits_per_byte": 1.783597331602714,
      "learning_rate": 0.00023601251472489757,
      "loss_nats": 1.236295461654663,
      "step": 2860,
      "unclipped_grad_norm": 1.0390338897705078
    },
    {
      "bits_per_byte": 1.7758223399225082,
      "learning_rate": 0.00023553730623665305,
      "loss_nats": 1.2309062480926514,
      "step": 2870,
      "unclipped_grad_norm": 1.0219858884811401
    },
    {
      "bits_per_byte": 1.7873183482367818,
      "learning_rate": 0.00023506089246046403,
      "loss_nats": 1.2388746738433838,
      "step": 2880,
      "unclipped_grad_norm": 1.002677083015442
    },
    {
      "bits_per_byte": 1.7586128959639213,
      "learning_rate": 0.00023458328153692736,
      "loss_nats": 1.2189775705337524,
      "step": 2890,
      "unclipped_grad_norm": 1.0140609741210938
    },
    {
      "bits_per_byte": 1.7806794739481757,
      "learning_rate": 0.0002341044816270961,
      "loss_nats": 1.2342729568481445,
      "step": 2900,
      "unclipped_grad_norm": 0.9781115651130676
    },
    {
      "bits_per_byte": 1.7540770255300655,
      "learning_rate": 0.00023362450091233953,
      "loss_nats": 1.2158335447311401,
      "step": 2910,
      "unclipped_grad_norm": 0.9601871967315674
    },
    {
      "bits_per_byte": 1.798447689539416,
      "learning_rate": 0.00023314334759420388,
      "loss_nats": 1.246588945388794,
      "step": 2920,
      "unclipped_grad_norm": 1.0120164155960083
    },
    {
      "bits_per_byte": 1.749404256906123,
      "learning_rate": 0.0002326610298942717,
      "loss_nats": 1.2125946283340454,
      "step": 2930,
      "unclipped_grad_norm": 0.9387297630310059
    },
    {
      "bits_per_byte": 1.7688945347804526,
      "learning_rate": 0.00023217755605402175,
      "loss_nats": 1.2261042594909668,
      "step": 2940,
      "unclipped_grad_norm": 1.073227047920227
    },
    {
      "bits_per_byte": 1.8185380148826058,
      "learning_rate": 0.00023169293433468802,
      "loss_nats": 1.260514497756958,
      "step": 2950,
      "unclipped_grad_norm": 1.033491849899292
    },
    {
      "bits_per_byte": 1.8193900169349801,
      "learning_rate": 0.00023120717301711857,
      "loss_nats": 1.2611050605773926,
      "step": 2960,
      "unclipped_grad_norm": 1.0577077865600586
    },
    {
      "bits_per_byte": 1.799499191466791,
      "learning_rate": 0.00023072028040163408,
      "loss_nats": 1.2473177909851074,
      "step": 2970,
      "unclipped_grad_norm": 1.0431829690933228
    },
    {
      "bits_per_byte": 1.8318936716006589,
      "learning_rate": 0.00023023226480788593,
      "loss_nats": 1.269771933555603,
      "step": 2980,
      "unclipped_grad_norm": 1.3554320335388184
    },
    {
      "bits_per_byte": 1.7557352822496848,
      "learning_rate": 0.00022974313457471416,
      "loss_nats": 1.2169829607009888,
      "step": 2990,
      "unclipped_grad_norm": 1.0419063568115234
    },
    {
      "bits_per_byte": 1.8299745171996809,
      "learning_rate": 0.0002292528980600049,
      "loss_nats": 1.2684416770935059,
      "step": 3000,
      "unclipped_grad_norm": 1.0811892747879028
    },
    {
      "bits_per_byte": 1.7247794369902845,
      "learning_rate": 0.00022876156364054758,
      "loss_nats": 1.1955260038375854,
      "step": 3010,
      "unclipped_grad_norm": 1.0030642747879028
    },
    {
      "bits_per_byte": 1.7923558920632043,
      "learning_rate": 0.0002282691397118918,
      "loss_nats": 1.2423664331436157,
      "step": 3020,
      "unclipped_grad_norm": 1.002471685409546
    },
    {
      "bits_per_byte": 1.7403096423458468,
      "learning_rate": 0.00022777563468820376,
      "loss_nats": 1.2062907218933105,
      "step": 3030,
      "unclipped_grad_norm": 1.0230085849761963
    },
    {
      "bits_per_byte": 1.7392983843587735,
      "learning_rate": 0.00022728105700212282,
      "loss_nats": 1.205589771270752,
      "step": 3040,
      "unclipped_grad_norm": 1.0195822715759277
    },
    {
      "bits_per_byte": 1.8747281865725025,
      "learning_rate": 0.00022678541510461698,
      "loss_nats": 1.2994625568389893,
      "step": 3050,
      "unclipped_grad_norm": 1.0621304512023926
    },
    {
      "bits_per_byte": 1.80563828417199,
      "learning_rate": 0.00022628871746483876,
      "loss_nats": 1.251573085784912,
      "step": 3060,
      "unclipped_grad_norm": 1.0812959671020508
    },
    {
      "bits_per_byte": 1.7309672007856773,
      "learning_rate": 0.0002257909725699804,
      "loss_nats": 1.199815034866333,
      "step": 3070,
      "unclipped_grad_norm": 1.048651099205017
    },
    {
      "bits_per_byte": 1.80954315025983,
      "learning_rate": 0.00022529218892512886,
      "loss_nats": 1.2542797327041626,
      "step": 3080,
      "unclipped_grad_norm": 1.0254682302474976
    },
    {
      "bits_per_byte": 1.787755528135275,
      "learning_rate": 0.00022479237505312034,
      "loss_nats": 1.2391777038574219,
      "step": 3090,
      "unclipped_grad_norm": 1.0376405715942383
    },
    {
      "bits_per_byte": 1.748834134418513,
      "learning_rate": 0.00022429153949439496,
      "loss_nats": 1.2121994495391846,
      "step": 3100,
      "unclipped_grad_norm": 1.0169665813446045
    },
    {
      "bits_per_byte": 1.8895020122295705,
      "learning_rate": 0.00022378969080685053,
      "loss_nats": 1.30970299243927,
      "step": 3110,
      "unclipped_grad_norm": 1.009265661239624
    },
    {
      "bits_per_byte": 1.772648744066137,
      "learning_rate": 0.0002232868375656965,
      "loss_nats": 1.2287064790725708,
      "step": 3120,
      "unclipped_grad_norm": 1.0459308624267578
    },
    {
      "bits_per_byte": 1.7184131552032957,
      "learning_rate": 0.00022278298836330733,
      "loss_nats": 1.1911132335662842,
      "step": 3130,
      "unclipped_grad_norm": 0.9758033752441406
    },
    {
      "bits_per_byte": 1.7503415623533254,
      "learning_rate": 0.00022227815180907572,
      "loss_nats": 1.2132443189620972,
      "step": 3140,
      "unclipped_grad_norm": 1.0032713413238525
    },
    {
      "bits_per_byte": 1.7368366246943234,
      "learning_rate": 0.00022177233652926546,
      "loss_nats": 1.203883409500122,
      "step": 3150,
      "unclipped_grad_norm": 0.9544663429260254
    },
    {
      "bits_per_byte": 1.7448823170669872,
      "learning_rate": 0.00022126555116686416,
      "loss_nats": 1.2094602584838867,
      "step": 3160,
      "unclipped_grad_norm": 0.9832491278648376
    },
    {
      "bits_per_byte": 1.8183364512157945,
      "learning_rate": 0.00022075780438143532,
      "loss_nats": 1.2603747844696045,
      "step": 3170,
      "unclipped_grad_norm": 1.080575942993164
    },
    {
      "bits_per_byte": 1.796097030667423,
      "learning_rate": 0.00022024910484897065,
      "loss_nats": 1.2449595928192139,
      "step": 3180,
      "unclipped_grad_norm": 1.0869723558425903
    },
    {
      "bits_per_byte": 1.7798367589589479,
      "learning_rate": 0.00021973946126174158,
      "loss_nats": 1.2336888313293457,
      "step": 3190,
      "unclipped_grad_norm": 1.0730317831039429
    },
    {
      "bits_per_byte": 1.7660776309419706,
      "learning_rate": 0.00021922888232815086,
      "loss_nats": 1.2241517305374146,
      "step": 3200,
      "unclipped_grad_norm": 1.0053741931915283
    },
    {
      "bits_per_byte": 1.736418362887425,
      "learning_rate": 0.0002187173767725837,
      "loss_nats": 1.2035934925079346,
      "step": 3210,
      "unclipped_grad_norm": 0.9884828925132751
    },
    {
      "bits_per_byte": 1.7094793444215763,
      "learning_rate": 0.0002182049533352588,
      "loss_nats": 1.1849207878112793,
      "step": 3220,
      "unclipped_grad_norm": 0.9791789054870605
    },
    {
      "bits_per_byte": 1.7767474346014993,
      "learning_rate": 0.00021769162077207884,
      "loss_nats": 1.231547474861145,
      "step": 3230,
      "unclipped_grad_norm": 1.0538209676742554
    },
    {
      "bits_per_byte": 1.783878351254224,
      "learning_rate": 0.00021717738785448102,
      "loss_nats": 1.236490249633789,
      "step": 3240,
      "unclipped_grad_norm": 1.0962731838226318
    },
    {
      "bits_per_byte": 1.7619694813608107,
      "learning_rate": 0.00021666226336928708,
      "loss_nats": 1.221304178237915,
      "step": 3250,
      "unclipped_grad_norm": 0.9704980850219727
    },
    {
      "bits_per_byte": 1.7647327266122235,
      "learning_rate": 0.0002161462561185532,
      "loss_nats": 1.2232195138931274,
      "step": 3260,
      "unclipped_grad_norm": 0.9566147923469543
    },
    {
      "bits_per_byte": 1.793783004100064,
      "learning_rate": 0.00021562937491941957,
      "loss_nats": 1.243355631828308,
      "step": 3270,
      "unclipped_grad_norm": 1.0367501974105835
    },
    {
      "bits_per_byte": 1.7882781834112471,
      "learning_rate": 0.00021511162860395975,
      "loss_nats": 1.2395399808883667,
      "step": 3280,
      "unclipped_grad_norm": 1.1419988870620728
    },
    {
      "bits_per_byte": 1.7570841421804018,
      "learning_rate": 0.0002145930260190298,
      "loss_nats": 1.2179179191589355,
      "step": 3290,
      "unclipped_grad_norm": 1.0371057987213135
    },
    {
      "bits_per_byte": 1.7704131415875712,
      "learning_rate": 0.00021407357602611698,
      "loss_nats": 1.2271568775177002,
      "step": 3300,
      "unclipped_grad_norm": 1.0890001058578491
    },
    {
      "bits_per_byte": 1.73421681297373,
      "learning_rate": 0.0002135532875011885,
      "loss_nats": 1.202067494392395,
      "step": 3310,
      "unclipped_grad_norm": 1.0024704933166504
    },
    {
      "bits_per_byte": 1.7556792159055035,
      "learning_rate": 0.0002130321693345397,
      "loss_nats": 1.2169440984725952,
      "step": 3320,
      "unclipped_grad_norm": 1.0351438522338867
    },
    {
      "bits_per_byte": 1.751791376100099,
      "learning_rate": 0.00021251023043064224,
      "loss_nats": 1.2142492532730103,
      "step": 3330,
      "unclipped_grad_norm": 0.967126727104187
    },
    {
      "bits_per_byte": 1.7023927992927743,
      "learning_rate": 0.00021198747970799188,
      "loss_nats": 1.1800087690353394,
      "step": 3340,
      "unclipped_grad_norm": 0.9899824857711792
    },
    {
      "bits_per_byte": 1.7427576433982281,
      "learning_rate": 0.00021146392609895618,
      "loss_nats": 1.2079875469207764,
      "step": 3350,
      "unclipped_grad_norm": 1.044270396232605
    },
    {
      "bits_per_byte": 1.7489617455454531,
      "learning_rate": 0.00021093957854962178,
      "loss_nats": 1.2122879028320312,
      "step": 3360,
      "unclipped_grad_norm": 1.0146241188049316
    },
    {
      "bits_per_byte": 1.6929069242017853,
      "learning_rate": 0.00021041444601964157,
      "loss_nats": 1.1734336614608765,
      "step": 3370,
      "unclipped_grad_norm": 0.9934349656105042
    },
    {
      "bits_per_byte": 1.7374774320514381,
      "learning_rate": 0.00020988853748208164,
      "loss_nats": 1.2043275833129883,
      "step": 3380,
      "unclipped_grad_norm": 1.0039739608764648
    },
    {
      "bits_per_byte": 1.7582206035373031,
      "learning_rate": 0.00020936186192326782,
      "loss_nats": 1.218705654144287,
      "step": 3390,
      "unclipped_grad_norm": 1.007129430770874
    },
    {
      "bits_per_byte": 1.7465608677394082,
      "learning_rate": 0.00020883442834263225,
      "loss_nats": 1.2106237411499023,
      "step": 3400,
      "unclipped_grad_norm": 0.9504732489585876
    },
    {
      "bits_per_byte": 1.744059208099958,
      "learning_rate": 0.0002083062457525596,
      "loss_nats": 1.2088897228240967,
      "step": 3410,
      "unclipped_grad_norm": 1.002007007598877
    },
    {
      "bits_per_byte": 1.7186439559207538,
      "learning_rate": 0.00020777732317823306,
      "loss_nats": 1.1912732124328613,
      "step": 3420,
      "unclipped_grad_norm": 0.9931132197380066
    },
    {
      "bits_per_byte": 1.762099672227514,
      "learning_rate": 0.00020724766965748005,
      "loss_nats": 1.221394419670105,
      "step": 3430,
      "unclipped_grad_norm": 0.943431556224823
    },
    {
      "bits_per_byte": 1.763225298677411,
      "learning_rate": 0.00020671729424061788,
      "loss_nats": 1.2221746444702148,
      "step": 3440,
      "unclipped_grad_norm": 0.9422610402107239
    },
    {
      "bits_per_byte": 1.8310394337738234,
      "learning_rate": 0.00020618620599029906,
      "loss_nats": 1.2691798210144043,
      "step": 3450,
      "unclipped_grad_norm": 1.1188790798187256
    },
    {
      "bits_per_byte": 1.822499291279929,
      "learning_rate": 0.00020565441398135649,
      "loss_nats": 1.2632602453231812,
      "step": 3460,
      "unclipped_grad_norm": 1.14117431640625
    },
    {
      "bits_per_byte": 1.7955301758501792,
      "learning_rate": 0.00020512192730064827,
      "loss_nats": 1.2445666790008545,
      "step": 3470,
      "unclipped_grad_norm": 1.0871022939682007
    },
    {
      "bits_per_byte": 1.7607453088519693,
      "learning_rate": 0.00020458875504690273,
      "loss_nats": 1.2204556465148926,
      "step": 3480,
      "unclipped_grad_norm": 1.0999677181243896
    },
    {
      "bits_per_byte": 1.6814413568167144,
      "learning_rate": 0.00020405490633056245,
      "loss_nats": 1.1654863357543945,
      "step": 3490,
      "unclipped_grad_norm": 1.0303022861480713
    },
    {
      "bits_per_byte": 1.6926489502254911,
      "learning_rate": 0.00020352039027362917,
      "loss_nats": 1.1732548475265503,
      "step": 3500,
      "unclipped_grad_norm": 1.0132246017456055
    },
    {
      "bits_per_byte": 1.7370531508517597,
      "learning_rate": 0.0002029852160095075,
      "loss_nats": 1.2040334939956665,
      "step": 3510,
      "unclipped_grad_norm": 1.0184191465377808
    },
    {
      "bits_per_byte": 1.7040419409318979,
      "learning_rate": 0.000202449392682849,
      "loss_nats": 1.1811518669128418,
      "step": 3520,
      "unclipped_grad_norm": 0.9375280141830444
    },
    {
      "bits_per_byte": 1.7216300906876847,
      "learning_rate": 0.0002019129294493959,
      "loss_nats": 1.1933430433273315,
      "step": 3530,
      "unclipped_grad_norm": 1.0306724309921265
    },
    {
      "bits_per_byte": 1.7320362449434406,
      "learning_rate": 0.00020137583547582478,
      "loss_nats": 1.2005560398101807,
      "step": 3540,
      "unclipped_grad_norm": 0.9943498373031616
    },
    {
      "bits_per_byte": 1.7834315403272825,
      "learning_rate": 0.00020083811993958973,
      "loss_nats": 1.2361805438995361,
      "step": 3550,
      "unclipped_grad_norm": 1.066209316253662
    },
    {
      "bits_per_byte": 1.7059953198803977,
      "learning_rate": 0.00020029979202876563,
      "loss_nats": 1.1825058460235596,
      "step": 3560,
      "unclipped_grad_norm": 1.0326995849609375
    },
    {
      "bits_per_byte": 1.768657198722262,
      "learning_rate": 0.00019976086094189118,
      "loss_nats": 1.2259397506713867,
      "step": 3570,
      "unclipped_grad_norm": 1.0416048765182495
    },
    {
      "bits_per_byte": 1.8330426876910733,
      "learning_rate": 0.00019922133588781175,
      "loss_nats": 1.2705683708190918,
      "step": 3580,
      "unclipped_grad_norm": 1.1256591081619263
    },
    {
      "bits_per_byte": 1.759539710469421,
      "learning_rate": 0.0001986812260855219,
      "loss_nats": 1.2196199893951416,
      "step": 3590,
      "unclipped_grad_norm": 1.0512219667434692
    },
    {
      "bits_per_byte": 1.727328735824024,
      "learning_rate": 0.00019814054076400787,
      "loss_nats": 1.1972930431365967,
      "step": 3600,
      "unclipped_grad_norm": 0.9710429906845093
    },
    {
      "bits_per_byte": 1.7248793589104356,
      "learning_rate": 0.00019759928916209004,
      "loss_nats": 1.1955952644348145,
      "step": 3610,
      "unclipped_grad_norm": 1.0369946956634521
    },
    {
      "bits_per_byte": 1.7248757472747676,
      "learning_rate": 0.00019705748052826493,
      "loss_nats": 1.1955927610397339,
      "step": 3620,
      "unclipped_grad_norm": 1.0838762521743774
    },
    {
      "bits_per_byte": 1.7412607064051182,
      "learning_rate": 0.00019651512412054723,
      "loss_nats": 1.2069499492645264,
      "step": 3630,
      "unclipped_grad_norm": 0.9884288311004639
    },
    {
      "bits_per_byte": 1.6889442719432552,
      "learning_rate": 0.00019597222920631144,
      "loss_nats": 1.170686960220337,
      "step": 3640,
      "unclipped_grad_norm": 1.0365123748779297
    },
    {
      "bits_per_byte": 1.7701531038194667,
      "learning_rate": 0.0001954288050621338,
      "loss_nats": 1.2269766330718994,
      "step": 3650,
      "unclipped_grad_norm": 1.1143370866775513
    },
    {
      "bits_per_byte": 1.6641216719789245,
      "learning_rate": 0.00019488486097363363,
      "loss_nats": 1.1534812450408936,
      "step": 3660,
      "unclipped_grad_norm": 1.0202206373214722
    },
    {
      "bits_per_byte": 1.6856676584790171,
      "learning_rate": 0.00019434040623531452,
      "loss_nats": 1.1684157848358154,
      "step": 3670,
      "unclipped_grad_norm": 1.0151705741882324
    },
    {
      "bits_per_byte": 1.7531821997976265,
      "learning_rate": 0.0001937954501504058,
      "loss_nats": 1.2152132987976074,
      "step": 3680,
      "unclipped_grad_norm": 1.013437271118164
    },
    {
      "bits_per_byte": 1.7126154480600597,
      "learning_rate": 0.00019325000203070333,
      "loss_nats": 1.1870945692062378,
      "step": 3690,
      "unclipped_grad_norm": 1.0570411682128906
    },
    {
      "bits_per_byte": 1.6629353356532728,
      "learning_rate": 0.0001927040711964106,
      "loss_nats": 1.1526589393615723,
      "step": 3700,
      "unclipped_grad_norm": 1.0999462604522705
    },
    {
      "bits_per_byte": 1.7333302424085324,
      "learning_rate": 0.00019215766697597923,
      "loss_nats": 1.2014529705047607,
      "step": 3710,
      "unclipped_grad_norm": 1.0060557126998901
    },
    {
      "bits_per_byte": 1.714764887230543,
      "learning_rate": 0.00019161079870594978,
      "loss_nats": 1.1885844469070435,
      "step": 3720,
      "unclipped_grad_norm": 1.061936378479004
    },
    {
      "bits_per_byte": 1.761044214699169,
      "learning_rate": 0.0001910634757307922,
      "loss_nats": 1.2206628322601318,
      "step": 3730,
      "unclipped_grad_norm": 1.0892959833145142
    },
    {
      "bits_per_byte": 1.704870553343755,
      "learning_rate": 0.0001905157074027459,
      "loss_nats": 1.1817262172698975,
      "step": 3740,
      "unclipped_grad_norm": 1.0097936391830444
    },
    {
      "bits_per_byte": 1.6862241943372092,
      "learning_rate": 0.00018996750308166037,
      "loss_nats": 1.1688015460968018,
      "step": 3750,
      "unclipped_grad_norm": 1.0313822031021118
    },
    {
      "bits_per_byte": 1.6824576023006628,
      "learning_rate": 0.00018941887213483482,
      "loss_nats": 1.16619074344635,
      "step": 3760,
      "unclipped_grad_norm": 1.0350043773651123
    },
    {
      "bits_per_byte": 1.7407710717981117,
      "learning_rate": 0.00018886982393685845,
      "loss_nats": 1.2066105604171753,
      "step": 3770,
      "unclipped_grad_norm": 1.0363857746124268
    },
    {
      "bits_per_byte": 1.6889002443846344,
      "learning_rate": 0.00018832036786945008,
      "loss_nats": 1.170656442642212,
      "step": 3780,
      "unclipped_grad_norm": 0.9539183974266052
    },
    {
      "bits_per_byte": 1.7337478162848272,
      "learning_rate": 0.0001877705133212979,
      "loss_nats": 1.20174241065979,
      "step": 3790,
      "unclipped_grad_norm": 0.9895081520080566
    },
    {
      "bits_per_byte": 1.6773657119565675,
      "learning_rate": 0.00018722026968789906,
      "loss_nats": 1.1626613140106201,
      "step": 3800,
      "unclipped_grad_norm": 0.9700848460197449
    },
    {
      "bits_per_byte": 1.6665356204664348,
      "learning_rate": 0.00018666964637139907,
      "loss_nats": 1.1551544666290283,
      "step": 3810,
      "unclipped_grad_norm": 1.059192180633545
    },
    {
      "bits_per_byte": 1.7701960994821824,
      "learning_rate": 0.00018611865278043115,
      "loss_nats": 1.227006435394287,
      "step": 3820,
      "unclipped_grad_norm": 0.9839220643043518
    },
    {
      "bits_per_byte": 1.6973205149708774,
      "learning_rate": 0.00018556729832995555,
      "loss_nats": 1.1764929294586182,
      "step": 3830,
      "unclipped_grad_norm": 1.0503867864608765
    },
    {
      "bits_per_byte": 1.6894750104038179,
      "learning_rate": 0.00018501559244109864,
      "loss_nats": 1.1710548400878906,
      "step": 3840,
      "unclipped_grad_norm": 1.0203979015350342
    },
    {
      "bits_per_byte": 1.7223587811793903,
      "learning_rate": 0.00018446354454099178,
      "loss_nats": 1.1938481330871582,
      "step": 3850,
      "unclipped_grad_norm": 1.0148154497146606
    },
    {
      "bits_per_byte": 1.6589467140144625,
      "learning_rate": 0.00018391116406261048,
      "loss_nats": 1.1498942375183105,
      "step": 3860,
      "unclipped_grad_norm": 0.9799250364303589
    },
    {
      "bits_per_byte": 1.719725382829379,
      "learning_rate": 0.00018335846044461305,
      "loss_nats": 1.1920228004455566,
      "step": 3870,
      "unclipped_grad_norm": 0.9761002659797668
    },
    {
      "bits_per_byte": 1.70201718918329,
      "learning_rate": 0.0001828054431311793,
      "loss_nats": 1.1797484159469604,
      "step": 3880,
      "unclipped_grad_norm": 1.047680139541626
    },
    {
      "bits_per_byte": 1.7301960305792086,
      "learning_rate": 0.00018225212157184934,
      "loss_nats": 1.1992805004119873,
      "step": 3890,
      "unclipped_grad_norm": 1.0218281745910645
    },
    {
      "bits_per_byte": 1.728929550338255,
      "learning_rate": 0.00018169850522136192,
      "loss_nats": 1.1984026432037354,
      "step": 3900,
      "unclipped_grad_norm": 1.1127806901931763
    },
    {
      "bits_per_byte": 1.7357486624449652,
      "learning_rate": 0.000181144603539493,
      "loss_nats": 1.2031292915344238,
      "step": 3910,
      "unclipped_grad_norm": 0.9941173791885376
    },
    {
      "bits_per_byte": 1.7061793413168207,
      "learning_rate": 0.00018059042599089412,
      "loss_nats": 1.182633399963379,
      "step": 3920,
      "unclipped_grad_norm": 0.9868062734603882
    },
    {
      "bits_per_byte": 1.6839027725158628,
      "learning_rate": 0.00018003598204493048,
      "loss_nats": 1.1671924591064453,
      "step": 3930,
      "unclipped_grad_norm": 1.0294983386993408
    },
    {
      "bits_per_byte": 1.6929858642385314,
      "learning_rate": 0.00017948128117551936,
      "loss_nats": 1.1734883785247803,
      "step": 3940,
      "unclipped_grad_norm": 1.0079408884048462
    },
    {
      "bits_per_byte": 1.7648030675164263,
      "learning_rate": 0.00017892633286096816,
      "loss_nats": 1.2232682704925537,
      "step": 3950,
      "unclipped_grad_norm": 1.091759204864502
    },
    {
      "bits_per_byte": 1.7722222270919972,
      "learning_rate": 0.00017837114658381247,
      "loss_nats": 1.2284108400344849,
      "step": 3960,
      "unclipped_grad_norm": 1.075370192527771
    },
    {
      "bits_per_byte": 1.7441799399208635,
      "learning_rate": 0.00017781573183065386,
      "loss_nats": 1.2089734077453613,
      "step": 3970,
      "unclipped_grad_norm": 1.0237821340560913
    },
    {
      "bits_per_byte": 1.7199797451700052,
      "learning_rate": 0.0001772600980919981,
      "loss_nats": 1.1921991109848022,
      "step": 3980,
      "unclipped_grad_norm": 1.0290383100509644
    },
    {
      "bits_per_byte": 1.7549034021674614,
      "learning_rate": 0.0001767042548620927,
      "loss_nats": 1.2164063453674316,
      "step": 3990,
      "unclipped_grad_norm": 1.0765279531478882
    },
    {
      "bits_per_byte": 1.7501554771250918,
      "learning_rate": 0.00017614821163876486,
      "loss_nats": 1.2131153345108032,
      "step": 4000,
      "unclipped_grad_norm": 1.037840485572815
    },
    {
      "bits_per_byte": 1.783230320625773,
      "learning_rate": 0.00017559197792325906,
      "loss_nats": 1.2360410690307617,
      "step": 4010,
      "unclipped_grad_norm": 1.1067007780075073
    },
    {
      "bits_per_byte": 1.634854696350997,
      "learning_rate": 0.0001750355632200748,
      "loss_nats": 1.133194923400879,
      "step": 4020,
      "unclipped_grad_norm": 0.9555695652961731
    },
    {
      "bits_per_byte": 1.6858748975733067,
      "learning_rate": 0.00017447897703680418,
      "loss_nats": 1.1685594320297241,
      "step": 4030,
      "unclipped_grad_norm": 0.9827634692192078
    },
    {
      "bits_per_byte": 1.6858767893824662,
      "learning_rate": 0.0001739222288839694,
      "loss_nats": 1.1685607433319092,
      "step": 4040,
      "unclipped_grad_norm": 1.0647450685501099
    },
    {
      "bits_per_byte": 1.6753074235910415,
      "learning_rate": 0.00017336532827486026,
      "loss_nats": 1.1612346172332764,
      "step": 4050,
      "unclipped_grad_norm": 1.0498672723770142
    },
    {
      "bits_per_byte": 1.7132395731000407,
      "learning_rate": 0.00017280828472537163,
      "loss_nats": 1.1875271797180176,
      "step": 4060,
      "unclipped_grad_norm": 1.0244723558425903
    },
    {
      "bits_per_byte": 1.659084988065756,
      "learning_rate": 0.0001722511077538408,
      "loss_nats": 1.1499900817871094,
      "step": 4070,
      "unclipped_grad_norm": 0.954911470413208
    },
    {
      "bits_per_byte": 1.7001991605810192,
      "learning_rate": 0.00017169380688088497,
      "loss_nats": 1.1784882545471191,
      "step": 4080,
      "unclipped_grad_norm": 1.0178123712539673
    },
    {
      "bits_per_byte": 1.6512532421107655,
      "learning_rate": 0.0001711363916292384,
      "loss_nats": 1.144561529159546,
      "step": 4090,
      "unclipped_grad_norm": 0.988084614276886
    },
    {
      "bits_per_byte": 1.6312373852553994,
      "learning_rate": 0.00017057887152358974,
      "loss_nats": 1.1306875944137573,
      "step": 4100,
      "unclipped_grad_norm": 0.9640962481498718
    },
    {
      "bits_per_byte": 1.7389465078551083,
      "learning_rate": 0.0001700212560904194,
      "loss_nats": 1.205345869064331,
      "step": 4110,
      "unclipped_grad_norm": 1.0591799020767212
    },
    {
      "bits_per_byte": 1.670625711869254,
      "learning_rate": 0.00016946355485783656,
      "loss_nats": 1.157989501953125,
      "step": 4120,
      "unclipped_grad_norm": 1.0143104791641235
    },
    {
      "bits_per_byte": 1.6531302607622822,
      "learning_rate": 0.0001689057773554166,
      "loss_nats": 1.1458625793457031,
      "step": 4130,
      "unclipped_grad_norm": 1.0070092678070068
    },
    {
      "bits_per_byte": 1.6853878427060633,
      "learning_rate": 0.00016834793311403806,
      "loss_nats": 1.1682218313217163,
      "step": 4140,
      "unclipped_grad_norm": 1.0397000312805176
    },
    {
      "bits_per_byte": 1.716280570332597,
      "learning_rate": 0.00016779003166571981,
      "loss_nats": 1.1896350383758545,
      "step": 4150,
      "unclipped_grad_norm": 1.0394762754440308
    },
    {
      "bits_per_byte": 1.7032425655706875,
      "learning_rate": 0.00016723208254345838,
      "loss_nats": 1.1805977821350098,
      "step": 4160,
      "unclipped_grad_norm": 1.0539857149124146
    },
    {
      "bits_per_byte": 1.6362312454885026,
      "learning_rate": 0.00016667409528106481,
      "loss_nats": 1.1341490745544434,
      "step": 4170,
      "unclipped_grad_norm": 1.0312221050262451
    },
    {
      "bits_per_byte": 1.6176134356020002,
      "learning_rate": 0.00016611607941300184,
      "loss_nats": 1.121244192123413,
      "step": 4180,
      "unclipped_grad_norm": 0.9745424389839172
    },
    {
      "bits_per_byte": 1.7779335989445002,
      "learning_rate": 0.0001655580444742211,
      "loss_nats": 1.2323696613311768,
      "step": 4190,
      "unclipped_grad_norm": 1.067311406135559
    },
    {
      "bits_per_byte": 1.675829562919061,
      "learning_rate": 0.00016499999999999997,
      "loss_nats": 1.1615965366363525,
      "step": 4200,
      "unclipped_grad_norm": 1.045929193496704
    },
    {
      "bits_per_byte": 1.681460618873611,
      "learning_rate": 0.0001644419555257789,
      "loss_nats": 1.1654996871948242,
      "step": 4210,
      "unclipped_grad_norm": 1.0254682302474976
    },
    {
      "bits_per_byte": 1.7232076875440492,
      "learning_rate": 0.0001638839205869981,
      "loss_nats": 1.1944365501403809,
      "step": 4220,
      "unclipped_grad_norm": 1.0498327016830444
    },
    {
      "bits_per_byte": 1.6892706950145928,
      "learning_rate": 0.0001633259047189352,
      "loss_nats": 1.1709132194519043,
      "step": 4230,
      "unclipped_grad_norm": 1.0495660305023193
    },
    {
      "bits_per_byte": 1.7317139494557237,
      "learning_rate": 0.0001627679174565416,
      "loss_nats": 1.2003326416015625,
      "step": 4240,
      "unclipped_grad_norm": 1.1651132106781006
    },
    {
      "bits_per_byte": 1.744415040204593,
      "learning_rate": 0.00016220996833428016,
      "loss_nats": 1.2091363668441772,
      "step": 4250,
      "unclipped_grad_norm": 1.0255383253097534
    },
    {
      "bits_per_byte": 1.73589312787169,
      "learning_rate": 0.00016165206688596191,
      "loss_nats": 1.2032294273376465,
      "step": 4260,
      "unclipped_grad_norm": 1.1120972633361816
    },
    {
      "bits_per_byte": 1.7234627378152787,
      "learning_rate": 0.00016109422264458335,
      "loss_nats": 1.1946133375167847,
      "step": 4270,
      "unclipped_grad_norm": 1.016249656677246
    },
    {
      "bits_per_byte": 1.7430716837187037,
      "learning_rate": 0.00016053644514216336,
      "loss_nats": 1.208205223083496,
      "step": 4280,
      "unclipped_grad_norm": 1.0722230672836304
    },
    {
      "bits_per_byte": 1.7030208799337254,
      "learning_rate": 0.00015997874390958058,
      "loss_nats": 1.1804441213607788,
      "step": 4290,
      "unclipped_grad_norm": 1.03997004032135
    },
    {
      "bits_per_byte": 1.6244537015747667,
      "learning_rate": 0.00015942112847641018,
      "loss_nats": 1.1259855031967163,
      "step": 4300,
      "unclipped_grad_norm": 1.037449836730957
    },
    {
      "bits_per_byte": 1.7698973656176336,
      "learning_rate": 0.00015886360837076156,
      "loss_nats": 1.2267993688583374,
      "step": 4310,
      "unclipped_grad_norm": 1.235188603401184
    },
    {
      "bits_per_byte": 1.6511122163370582,
      "learning_rate": 0.00015830619311911495,
      "loss_nats": 1.1444637775421143,
      "step": 4320,
      "unclipped_grad_norm": 1.1049147844314575
    },
    {
      "bits_per_byte": 1.6449793150072904,
      "learning_rate": 0.00015774889224615915,
      "loss_nats": 1.1402127742767334,
      "step": 4330,
      "unclipped_grad_norm": 1.0132696628570557
    },
    {
      "bits_per_byte": 1.6583323919855806,
      "learning_rate": 0.00015719171527462834,
      "loss_nats": 1.1494684219360352,
      "step": 4340,
      "unclipped_grad_norm": 1.0222140550613403
    },
    {
      "bits_per_byte": 1.5761074866080718,
      "learning_rate": 0.0001566346717251397,
      "loss_nats": 1.0924744606018066,
      "step": 4350,
      "unclipped_grad_norm": 1.0156723260879517
    },
    {
      "bits_per_byte": 1.7219245249859618,
      "learning_rate": 0.00015607777111603056,
      "loss_nats": 1.1935471296310425,
      "step": 4360,
      "unclipped_grad_norm": 1.0623725652694702
    },
    {
      "bits_per_byte": 1.6842240361076746,
      "learning_rate": 0.00015552102296319576,
      "loss_nats": 1.1674151420593262,
      "step": 4370,
      "unclipped_grad_norm": 1.070603847503662
    },
    {
      "bits_per_byte": 1.6398215553079147,
      "learning_rate": 0.00015496443677992516,
      "loss_nats": 1.1366376876831055,
      "step": 4380,
      "unclipped_grad_norm": 1.0265616178512573
    },
    {
      "bits_per_byte": 1.6765849107216504,
      "learning_rate": 0.0001544080220767409,
      "loss_nats": 1.1621201038360596,
      "step": 4390,
      "unclipped_grad_norm": 1.020019769668579
    },
    {
      "bits_per_byte": 1.6365060737645816,
      "learning_rate": 0.00015385178836123511,
      "loss_nats": 1.1343395709991455,
      "step": 4400,
      "unclipped_grad_norm": 1.0334678888320923
    },
    {
      "bits_per_byte": 1.6461109608499676,
      "learning_rate": 0.00015329574513790726,
      "loss_nats": 1.1409971714019775,
      "step": 4410,
      "unclipped_grad_norm": 1.064432144165039
    },
    {
      "bits_per_byte": 1.6319834459948421,
      "learning_rate": 0.00015273990190800186,
      "loss_nats": 1.1312047243118286,
      "step": 4420,
      "unclipped_grad_norm": 1.0356578826904297
    },
    {
      "bits_per_byte": 1.6589714795161867,
      "learning_rate": 0.0001521842681693461,
      "loss_nats": 1.1499114036560059,
      "step": 4430,
      "unclipped_grad_norm": 1.0542991161346436
    },
    {
      "bits_per_byte": 1.6678407968038327,
      "learning_rate": 0.00015162885341618752,
      "loss_nats": 1.1560591459274292,
      "step": 4440,
      "unclipped_grad_norm": 1.1220062971115112
    },
    {
      "bits_per_byte": 1.6463864770566499,
      "learning_rate": 0.00015107366713903178,
      "loss_nats": 1.141188144683838,
      "step": 4450,
      "unclipped_grad_norm": 1.069725751876831
    },
    {
      "bits_per_byte": 1.6193611233000682,
      "learning_rate": 0.0001505187188244806,
      "loss_nats": 1.1224555969238281,
      "step": 4460,
      "unclipped_grad_norm": 1.0096163749694824
    },
    {
      "bits_per_byte": 1.7261988098078551,
      "learning_rate": 0.00014996401795506952,
      "loss_nats": 1.196509838104248,
      "step": 4470,
      "unclipped_grad_norm": 1.0757471323013306
    },
    {
      "bits_per_byte": 1.6253762165139949,
      "learning_rate": 0.00014940957400910585,
      "loss_nats": 1.1266249418258667,
      "step": 4480,
      "unclipped_grad_norm": 1.0172395706176758
    },
    {
      "bits_per_byte": 1.5785597872267245,
      "learning_rate": 0.00014885539646050699,
      "loss_nats": 1.0941742658615112,
      "step": 4490,
      "unclipped_grad_norm": 1.081101894378662
    },
    {
      "bits_per_byte": 1.6683827141367014,
      "learning_rate": 0.00014830149477863805,
      "loss_nats": 1.1564347743988037,
      "step": 4500,
      "unclipped_grad_norm": 1.027239441871643
    },
    {
      "bits_per_byte": 1.6472935135573004,
      "learning_rate": 0.00014774787842815066,
      "loss_nats": 1.1418168544769287,
      "step": 4510,
      "unclipped_grad_norm": 1.0604325532913208
    },
    {
      "bits_per_byte": 1.6464067710094517,
      "learning_rate": 0.00014719455686882066,
      "loss_nats": 1.1412022113800049,
      "step": 4520,
      "unclipped_grad_norm": 1.0223703384399414
    },
    {
      "bits_per_byte": 1.6663331968863693,
      "learning_rate": 0.00014664153955538695,
      "loss_nats": 1.155014157295227,
      "step": 4530,
      "unclipped_grad_norm": 1.096794843673706
    },
    {
      "bits_per_byte": 1.5983706407622622,
      "learning_rate": 0.00014608883593738946,
      "loss_nats": 1.1079061031341553,
      "step": 4540,
      "unclipped_grad_norm": 0.991118311882019
    },
    {
      "bits_per_byte": 1.6821887934173643,
      "learning_rate": 0.00014553645545900817,
      "loss_nats": 1.1660044193267822,
      "step": 4550,
      "unclipped_grad_norm": 1.0131337642669678
    },
    {
      "bits_per_byte": 1.6779839895864195,
      "learning_rate": 0.0001449844075589013,
      "loss_nats": 1.1630898714065552,
      "step": 4560,
      "unclipped_grad_norm": 1.0416655540466309
    },
    {
      "bits_per_byte": 1.6489940780090317,
      "learning_rate": 0.0001444327016700444,
      "loss_nats": 1.1429955959320068,
      "step": 4570,
      "unclipped_grad_norm": 1.0762525796890259
    },
    {
      "bits_per_byte": 1.5938887728807776,
      "learning_rate": 0.0001438813472195688,
      "loss_nats": 1.104799509048462,
      "step": 4580,
      "unclipped_grad_norm": 1.039039969444275
    },
    {
      "bits_per_byte": 1.7129355077733153,
      "learning_rate": 0.00014333035362860093,
      "loss_nats": 1.1873164176940918,
      "step": 4590,
      "unclipped_grad_norm": 1.1662163734436035
    },
    {
      "bits_per_byte": 1.6753789683738005,
      "learning_rate": 0.00014277973031210088,
      "loss_nats": 1.1612842082977295,
      "step": 4600,
      "unclipped_grad_norm": 1.0834790468215942
    },
    {
      "bits_per_byte": 1.6489213293477167,
      "learning_rate": 0.00014222948667870207,
      "loss_nats": 1.1429451704025269,
      "step": 4610,
      "unclipped_grad_norm": 1.0408029556274414
    },
    {
      "bits_per_byte": 1.5772676815707922,
      "learning_rate": 0.00014167963213054991,
      "loss_nats": 1.0932786464691162,
      "step": 4620,
      "unclipped_grad_norm": 0.9947658777236938
    },
    {
      "bits_per_byte": 1.6629637127906651,
      "learning_rate": 0.00014113017606314152,
      "loss_nats": 1.1526786088943481,
      "step": 4630,
      "unclipped_grad_norm": 1.0645713806152344
    },
    {
      "bits_per_byte": 1.6351708004632826,
      "learning_rate": 0.00014058112786516518,
      "loss_nats": 1.1334140300750732,
      "step": 4640,
      "unclipped_grad_norm": 1.0213215351104736
    },
    {
      "bits_per_byte": 1.6143973600308656,
      "learning_rate": 0.0001400324969183396,
      "loss_nats": 1.1190149784088135,
      "step": 4650,
      "unclipped_grad_norm": 1.0158205032348633
    },
    {
      "bits_per_byte": 1.6442494206370286,
      "learning_rate": 0.00013948429259725406,
      "loss_nats": 1.1397068500518799,
      "step": 4660,
      "unclipped_grad_norm": 1.032319188117981
    },
    {
      "bits_per_byte": 1.6788175894951514,
      "learning_rate": 0.00013893652426920776,
      "loss_nats": 1.1636676788330078,
      "step": 4670,
      "unclipped_grad_norm": 1.1467437744140625
    },
    {
      "bits_per_byte": 1.6228953667752988,
      "learning_rate": 0.00013838920129405016,
      "loss_nats": 1.1249053478240967,
      "step": 4680,
      "unclipped_grad_norm": 1.0684930086135864
    },
    {
      "bits_per_byte": 1.6116814100084402,
      "learning_rate": 0.0001378423330240207,
      "loss_nats": 1.1171324253082275,
      "step": 4690,
      "unclipped_grad_norm": 1.0818438529968262
    },
    {
      "bits_per_byte": 1.7118176205427071,
      "learning_rate": 0.00013729592880358936,
      "loss_nats": 1.1865415573120117,
      "step": 4700,
      "unclipped_grad_norm": 1.0913734436035156
    },
    {
      "bits_per_byte": 1.6314229545356804,
      "learning_rate": 0.00013674999796929659,
      "loss_nats": 1.1308162212371826,
      "step": 4710,
      "unclipped_grad_norm": 1.0516728162765503
    },
    {
      "bits_per_byte": 1.6067035441618671,
      "learning_rate": 0.00013620454984959417,
      "loss_nats": 1.1136820316314697,
      "step": 4720,
      "unclipped_grad_norm": 1.027355670928955
    },
    {
      "bits_per_byte": 1.723685455348146,
      "learning_rate": 0.00013565959376468542,
      "loss_nats": 1.194767713546753,
      "step": 4730,
      "unclipped_grad_norm": 1.0737906694412231
    },
    {
      "bits_per_byte": 1.6626813172779484,
      "learning_rate": 0.00013511513902636634,
      "loss_nats": 1.1524828672409058,
      "step": 4740,
      "unclipped_grad_norm": 1.093795657157898
    },
    {
      "bits_per_byte": 1.6942468410346574,
      "learning_rate": 0.0001345711949378662,
      "loss_nats": 1.1743624210357666,
      "step": 4750,
      "unclipped_grad_norm": 1.098819375038147
    },
    {
      "bits_per_byte": 1.567997816689287,
      "learning_rate": 0.00013402777079368853,
      "loss_nats": 1.086853265762329,
      "step": 4760,
      "unclipped_grad_norm": 0.9886804819107056
    },
    {
      "bits_per_byte": 1.674730077832095,
      "learning_rate": 0.0001334848758794528,
      "loss_nats": 1.1608344316482544,
      "step": 4770,
      "unclipped_grad_norm": 1.0352309942245483
    },
    {
      "bits_per_byte": 1.6338455021557339,
      "learning_rate": 0.00013294251947173504,
      "loss_nats": 1.132495403289795,
      "step": 4780,
      "unclipped_grad_norm": 1.037882685661316
    },
    {
      "bits_per_byte": 1.6704422063807836,
      "learning_rate": 0.00013240071083790996,
      "loss_nats": 1.1578623056411743,
      "step": 4790,
      "unclipped_grad_norm": 1.117195725440979
    },
    {
      "bits_per_byte": 1.6306944360266256,
      "learning_rate": 0.0001318594592359921,
      "loss_nats": 1.1303112506866455,
      "step": 4800,
      "unclipped_grad_norm": 1.0563780069351196
    },
    {
      "bits_per_byte": 1.6131430905581232,
      "learning_rate": 0.0001313187739144781,
      "loss_nats": 1.1181455850601196,
      "step": 4810,
      "unclipped_grad_norm": 1.0533764362335205
    },
    {
      "bits_per_byte": 1.675093477173368,
      "learning_rate": 0.0001307786641121882,
      "loss_nats": 1.1610863208770752,
      "step": 4820,
      "unclipped_grad_norm": 1.0993541479110718
    },
    {
      "bits_per_byte": 1.6710443176414542,
      "learning_rate": 0.00013023913905810876,
      "loss_nats": 1.1582796573638916,
      "step": 4830,
      "unclipped_grad_norm": 1.2536643743515015
    },
    {
      "bits_per_byte": 1.6771435103716528,
      "learning_rate": 0.00012970020797123432,
      "loss_nats": 1.1625072956085205,
      "step": 4840,
      "unclipped_grad_norm": 1.074759840965271
    },
    {
      "bits_per_byte": 1.6143529885069432,
      "learning_rate": 0.00012916188006041027,
      "loss_nats": 1.1189842224121094,
      "step": 4850,
      "unclipped_grad_norm": 1.0702130794525146
    },
    {
      "bits_per_byte": 1.6332604171774985,
      "learning_rate": 0.00012862416452417516,
      "loss_nats": 1.1320898532867432,
      "step": 4860,
      "unclipped_grad_norm": 1.0851078033447266
    },
    {
      "bits_per_byte": 1.6331224870915066,
      "learning_rate": 0.00012808707055060405,
      "loss_nats": 1.1319942474365234,
      "step": 4870,
      "unclipped_grad_norm": 1.08431875705719
    },
    {
      "bits_per_byte": 1.7854224114936703,
      "learning_rate": 0.000127550607317151,
      "loss_nats": 1.237560510635376,
      "step": 4880,
      "unclipped_grad_norm": 1.1104563474655151
    },
    {
      "bits_per_byte": 1.5755340964500952,
      "learning_rate": 0.00012701478399049248,
      "loss_nats": 1.0920770168304443,
      "step": 4890,
      "unclipped_grad_norm": 1.042919635772705
    },
    {
      "bits_per_byte": 1.6730817961062259,
      "learning_rate": 0.0001264796097263708,
      "loss_nats": 1.1596919298171997,
      "step": 4900,
      "unclipped_grad_norm": 1.0552027225494385
    },
    {
      "bits_per_byte": 1.6497614645971816,
      "learning_rate": 0.0001259450936694375,
      "loss_nats": 1.1435275077819824,
      "step": 4910,
      "unclipped_grad_norm": 1.0571706295013428
    },
    {
      "bits_per_byte": 1.5965801293841295,
      "learning_rate": 0.00012541124495309727,
      "loss_nats": 1.106665015220642,
      "step": 4920,
      "unclipped_grad_norm": 1.0640807151794434
    },
    {
      "bits_per_byte": 1.6007018655947076,
      "learning_rate": 0.00012487807269935167,
      "loss_nats": 1.1095219850540161,
      "step": 4930,
      "unclipped_grad_norm": 1.0152149200439453
    },
    {
      "bits_per_byte": 1.6331477685411835,
      "learning_rate": 0.00012434558601864349,
      "loss_nats": 1.1320117712020874,
      "step": 4940,
      "unclipped_grad_norm": 1.100314736366272
    },
    {
      "bits_per_byte": 1.6360496318091915,
      "learning_rate": 0.0001238137940097009,
      "loss_nats": 1.1340231895446777,
      "step": 4950,
      "unclipped_grad_norm": 1.079222559928894
    },
    {
      "bits_per_byte": 1.6569551549174713,
      "learning_rate": 0.0001232827057593821,
      "loss_nats": 1.1485137939453125,
      "step": 4960,
      "unclipped_grad_norm": 1.0604397058486938
    },
    {
      "bits_per_byte": 1.638398398872025,
      "learning_rate": 0.00012275233034251992,
      "loss_nats": 1.1356512308120728,
      "step": 4970,
      "unclipped_grad_norm": 1.0775542259216309
    },
    {
      "bits_per_byte": 1.680847672705936,
      "learning_rate": 0.0001222226768217669,
      "loss_nats": 1.1650748252868652,
      "step": 4980,
      "unclipped_grad_norm": 1.1589142084121704
    },
    {
      "bits_per_byte": 1.633999426628256,
      "learning_rate": 0.00012169375424744034,
      "loss_nats": 1.1326020956039429,
      "step": 4990,
      "unclipped_grad_norm": 1.075969934463501
    },
    {
      "bits_per_byte": 1.6444650868812105,
      "learning_rate": 0.00012116557165736772,
      "loss_nats": 1.1398563385009766,
      "step": 5000,
      "unclipped_grad_norm": 1.0691334009170532
    },
    {
      "bits_per_byte": 1.6738108305632333,
      "learning_rate": 0.00012063813807673214,
      "loss_nats": 1.1601972579956055,
      "step": 5010,
      "unclipped_grad_norm": 1.0487086772918701
    },
    {
      "bits_per_byte": 1.6477535671483583,
      "learning_rate": 0.00012011146251791832,
      "loss_nats": 1.142135739326477,
      "step": 5020,
      "unclipped_grad_norm": 1.1224150657653809
    },
    {
      "bits_per_byte": 1.5805776596692978,
      "learning_rate": 0.0001195855539803584,
      "loss_nats": 1.0955729484558105,
      "step": 5030,
      "unclipped_grad_norm": 0.9987351894378662
    },
    {
      "bits_per_byte": 1.6501410303076358,
      "learning_rate": 0.00011906042145037817,
      "loss_nats": 1.143790602684021,
      "step": 5040,
      "unclipped_grad_norm": 1.069442868232727
    },
    {
      "bits_per_byte": 1.6202205206064295,
      "learning_rate": 0.0001185360739010438,
      "loss_nats": 1.1230512857437134,
      "step": 5050,
      "unclipped_grad_norm": 1.1084743738174438
    },
    {
      "bits_per_byte": 1.6928149134835737,
      "learning_rate": 0.00011801252029200808,
      "loss_nats": 1.1733698844909668,
      "step": 5060,
      "unclipped_grad_norm": 1.0974880456924438
    },
    {
      "bits_per_byte": 1.6259845191500966,
      "learning_rate": 0.00011748976956935777,
      "loss_nats": 1.1270465850830078,
      "step": 5070,
      "unclipped_grad_norm": 1.0709881782531738
    },
    {
      "bits_per_byte": 1.6998562271751987,
      "learning_rate": 0.00011696783066546024,
      "loss_nats": 1.1782505512237549,
      "step": 5080,
      "unclipped_grad_norm": 1.1272931098937988
    },
    {
      "bits_per_byte": 1.6260173678364114,
      "learning_rate": 0.00011644671249881147,
      "loss_nats": 1.127069354057312,
      "step": 5090,
      "unclipped_grad_norm": 1.0618640184402466
    },
    {
      "bits_per_byte": 1.6234274810970684,
      "learning_rate": 0.00011592642397388293,
      "loss_nats": 1.1252741813659668,
      "step": 5100,
      "unclipped_grad_norm": 1.1292778253555298
    },
    {
      "bits_per_byte": 1.6568031222541084,
      "learning_rate": 0.00011540697398097015,
      "loss_nats": 1.1484084129333496,
      "step": 5110,
      "unclipped_grad_norm": 1.1496297121047974
    },
    {
      "bits_per_byte": 1.5598939941806313,
      "learning_rate": 0.00011488837139604016,
      "loss_nats": 1.0812361240386963,
      "step": 5120,
      "unclipped_grad_norm": 1.035429835319519
    },
    {
      "bits_per_byte": 1.6914442117561972,
      "learning_rate": 0.00011437062508058039,
      "loss_nats": 1.172419786453247,
      "step": 5130,
      "unclipped_grad_norm": 1.0999172925949097
    },
    {
      "bits_per_byte": 1.6623124144918477,
      "learning_rate": 0.00011385374388144673,
      "loss_nats": 1.1522271633148193,
      "step": 5140,
      "unclipped_grad_norm": 1.0919309854507446
    },
    {
      "bits_per_byte": 1.6100444791375281,
      "learning_rate": 0.00011333773663071286,
      "loss_nats": 1.1159977912902832,
      "step": 5150,
      "unclipped_grad_norm": 1.0898085832595825
    },
    {
      "bits_per_byte": 1.6321538808018472,
      "learning_rate": 0.00011282261214551897,
      "loss_nats": 1.1313228607177734,
      "step": 5160,
      "unclipped_grad_norm": 1.0807921886444092
    },
    {
      "bits_per_byte": 1.6844840738757791,
      "learning_rate": 0.00011230837922792112,
      "loss_nats": 1.167595386505127,
      "step": 5170,
      "unclipped_grad_norm": 1.176076889038086
    },
    {
      "bits_per_byte": 1.65267932225172,
      "learning_rate": 0.0001117950466647412,
      "loss_nats": 1.145550012588501,
      "step": 5180,
      "unclipped_grad_norm": 1.110230803489685
    },
    {
      "bits_per_byte": 1.5426296877564192,
      "learning_rate": 0.00011128262322741627,
      "loss_nats": 1.0692694187164307,
      "step": 5190,
      "unclipped_grad_norm": 1.0403128862380981
    },
    {
      "bits_per_byte": 1.6380544335702993,
      "learning_rate": 0.00011077111767184913,
      "loss_nats": 1.1354128122329712,
      "step": 5200,
      "unclipped_grad_norm": 1.1156731843948364
    },
    {
      "bits_per_byte": 1.6025732088187459,
      "learning_rate": 0.00011026053873825837,
      "loss_nats": 1.1108191013336182,
      "step": 5210,
      "unclipped_grad_norm": 1.112536907196045
    },
    {
      "bits_per_byte": 1.595609287320009,
      "learning_rate": 0.0001097508951510293,
      "loss_nats": 1.105992078781128,
      "step": 5220,
      "unclipped_grad_norm": 1.0583246946334839
    },
    {
      "bits_per_byte": 1.6907153492818405,
      "learning_rate": 0.0001092421956185646,
      "loss_nats": 1.1719145774841309,
      "step": 5230,
      "unclipped_grad_norm": 1.1788045167922974
    },
    {
      "bits_per_byte": 1.6472453584150588,
      "learning_rate": 0.00010873444883313581,
      "loss_nats": 1.1417834758758545,
      "step": 5240,
      "unclipped_grad_norm": 1.1392273902893066
    },
    {
      "bits_per_byte": 1.6431225903085755,
      "learning_rate": 0.00010822766347073447,
      "loss_nats": 1.1389257907867432,
      "step": 5250,
      "unclipped_grad_norm": 1.087467074394226
    },
    {
      "bits_per_byte": 1.5745493237912547,
      "learning_rate": 0.00010772184819092424,
      "loss_nats": 1.0913944244384766,
      "step": 5260,
      "unclipped_grad_norm": 1.0672780275344849
    },
    {
      "bits_per_byte": 1.625834722261195,
      "learning_rate": 0.0001072170116366926,
      "loss_nats": 1.126942753791809,
      "step": 5270,
      "unclipped_grad_norm": 1.0955538749694824
    },
    {
      "bits_per_byte": 1.5722745092682924,
      "learning_rate": 0.00010671316243430345,
      "loss_nats": 1.0898176431655884,
      "step": 5280,
      "unclipped_grad_norm": 1.1191643476486206
    },
    {
      "bits_per_byte": 1.581137635180507,
      "learning_rate": 0.00010621030919314944,
      "loss_nats": 1.095961093902588,
      "step": 5290,
      "unclipped_grad_norm": 1.0905182361602783
    },
    {
      "bits_per_byte": 1.5625956696430352,
      "learning_rate": 0.000105708460505605,
      "loss_nats": 1.0831087827682495,
      "step": 5300,
      "unclipped_grad_norm": 1.0970019102096558
    },
    {
      "bits_per_byte": 1.575699715742876,
      "learning_rate": 0.00010520762494687964,
      "loss_nats": 1.0921918153762817,
      "step": 5310,
      "unclipped_grad_norm": 1.085243582725525
    },
    {
      "bits_per_byte": 1.5934223559316376,
      "learning_rate": 0.00010470781107487111,
      "loss_nats": 1.1044762134552002,
      "step": 5320,
      "unclipped_grad_norm": 1.1693148612976074
    },
    {
      "bits_per_byte": 1.6214211314921028,
      "learning_rate": 0.00010420902743001955,
      "loss_nats": 1.1238834857940674,
      "step": 5330,
      "unclipped_grad_norm": 1.1232852935791016
    },
    {
      "bits_per_byte": 1.6839337293930183,
      "learning_rate": 0.00010371128253516117,
      "loss_nats": 1.1672139167785645,
      "step": 5340,
      "unclipped_grad_norm": 1.1369119882583618
    },
    {
      "bits_per_byte": 1.5416975417887426,
      "learning_rate": 0.000103214584895383,
      "loss_nats": 1.0686233043670654,
      "step": 5350,
      "unclipped_grad_norm": 1.0887452363967896
    },
    {
      "bits_per_byte": 1.5866414239734188,
      "learning_rate": 0.00010271894299787715,
      "loss_nats": 1.099776029586792,
      "step": 5360,
      "unclipped_grad_norm": 1.0925503969192505
    },
    {
      "bits_per_byte": 1.6646420914804352,
      "learning_rate": 0.00010222436531179622,
      "loss_nats": 1.1538419723510742,
      "step": 5370,
      "unclipped_grad_norm": 1.1924686431884766
    },
    {
      "bits_per_byte": 1.586491799067168,
      "learning_rate": 0.0001017308602881082,
      "loss_nats": 1.0996723175048828,
      "step": 5380,
      "unclipped_grad_norm": 1.0599968433380127
    },
    {
      "bits_per_byte": 1.5783984675002152,
      "learning_rate": 0.00010123843635945242,
      "loss_nats": 1.0940624475479126,
      "step": 5390,
      "unclipped_grad_norm": 1.0973831415176392
    },
    {
      "bits_per_byte": 1.642996527025493,
      "learning_rate": 0.00010074710193999505,
      "loss_nats": 1.1388384103775024,
      "step": 5400,
      "unclipped_grad_norm": 1.1477714776992798
    },
    {
      "bits_per_byte": 1.5308346016122951,
      "learning_rate": 0.0001002568654252858,
      "loss_nats": 1.0610936880111694,
      "step": 5410,
      "unclipped_grad_norm": 1.0651572942733765
    },
    {
      "bits_per_byte": 1.6877225071915258,
      "learning_rate": 9.976773519211404e-05,
      "loss_nats": 1.1698400974273682,
      "step": 5420,
      "unclipped_grad_norm": 1.230097770690918
    },
    {
      "bits_per_byte": 1.606558218821888,
      "learning_rate": 9.92797195983659e-05,
      "loss_nats": 1.1135812997817993,
      "step": 5430,
      "unclipped_grad_norm": 1.1141976118087769
    },
    {
      "bits_per_byte": 1.6079992614534675,
      "learning_rate": 9.879282698288142e-05,
      "loss_nats": 1.1145801544189453,
      "step": 5440,
      "unclipped_grad_norm": 1.0613634586334229
    },
    {
      "bits_per_byte": 1.6121270170568258,
      "learning_rate": 9.830706566531197e-05,
      "loss_nats": 1.1174412965774536,
      "step": 5450,
      "unclipped_grad_norm": 1.1033883094787598
    },
    {
      "bits_per_byte": 1.6549515570349194,
      "learning_rate": 9.782244394597826e-05,
      "loss_nats": 1.147125005722046,
      "step": 5460,
      "unclipped_grad_norm": 1.0616251230239868
    },
    {
      "bits_per_byte": 1.6138821000088808,
      "learning_rate": 9.733897010572829e-05,
      "loss_nats": 1.1186578273773193,
      "step": 5470,
      "unclipped_grad_norm": 1.1187942028045654
    },
    {
      "bits_per_byte": 1.566549894751673,
      "learning_rate": 9.685665240579612e-05,
      "loss_nats": 1.085849642753601,
      "step": 5480,
      "unclipped_grad_norm": 1.095421314239502
    },
    {
      "bits_per_byte": 1.6100902265226575,
      "learning_rate": 9.637549908766041e-05,
      "loss_nats": 1.1160295009613037,
      "step": 5490,
      "unclipped_grad_norm": 1.1169681549072266
    },
    {
      "bits_per_byte": 1.6345392801693144,
      "learning_rate": 9.589551837290389e-05,
      "loss_nats": 1.1329762935638428,
      "step": 5500,
      "unclipped_grad_norm": 1.0606133937835693
    },
    {
      "bits_per_byte": 1.6112232482265416,
      "learning_rate": 9.541671846307257e-05,
      "loss_nats": 1.1168148517608643,
      "step": 5510,
      "unclipped_grad_norm": 1.082831859588623
    },
    {
      "bits_per_byte": 1.6200284159854157,
      "learning_rate": 9.493910753953597e-05,
      "loss_nats": 1.1229181289672852,
      "step": 5520,
      "unclipped_grad_norm": 1.1288034915924072
    },
    {
      "bits_per_byte": 1.6500037881522474,
      "learning_rate": 9.446269376334689e-05,
      "loss_nats": 1.1436954736709595,
      "step": 5530,
      "unclipped_grad_norm": 1.1526131629943848
    },
    {
      "bits_per_byte": 1.6647621353707376,
      "learning_rate": 9.39874852751024e-05,
      "loss_nats": 1.1539251804351807,
      "step": 5540,
      "unclipped_grad_norm": 1.1722888946533203
    },
    {
      "bits_per_byte": 1.5974680758105342,
      "learning_rate": 9.351349019480433e-05,
      "loss_nats": 1.1072804927825928,
      "step": 5550,
      "unclipped_grad_norm": 1.073315143585205
    },
    {
      "bits_per_byte": 1.594942510582614,
      "learning_rate": 9.304071662172086e-05,
      "loss_nats": 1.1055299043655396,
      "step": 5560,
      "unclipped_grad_norm": 1.0879392623901367
    },
    {
      "bits_per_byte": 1.6466642290377933,
      "learning_rate": 9.256917263424787e-05,
      "loss_nats": 1.1413806676864624,
      "step": 5570,
      "unclipped_grad_norm": 1.21271812915802
    },
    {
      "bits_per_byte": 1.5387917229197647,
      "learning_rate": 9.209886628977114e-05,
      "loss_nats": 1.0666091442108154,
      "step": 5580,
      "unclipped_grad_norm": 1.0752229690551758
    },
    {
      "bits_per_byte": 1.5950678859350929,
      "learning_rate": 9.162980562452843e-05,
      "loss_nats": 1.105616807937622,
      "step": 5590,
      "unclipped_grad_norm": 1.115165114402771
    },
    {
      "bits_per_byte": 1.60302053569364,
      "learning_rate": 9.116199865347238e-05,
      "loss_nats": 1.1111291646957397,
      "step": 5600,
      "unclipped_grad_norm": 1.0947556495666504
    },
    {
      "bits_per_byte": 1.6150159816260192,
      "learning_rate": 9.069545337013333e-05,
      "loss_nats": 1.1194437742233276,
      "step": 5610,
      "unclipped_grad_norm": 1.0436956882476807
    },
    {
      "bits_per_byte": 1.5445598490470525,
      "learning_rate": 9.0230177746483e-05,
      "loss_nats": 1.070607304573059,
      "step": 5620,
      "unclipped_grad_norm": 1.1328603029251099
    },
    {
      "bits_per_byte": 1.5627000631121089,
      "learning_rate": 8.976617973279797e-05,
      "loss_nats": 1.0831811428070068,
      "step": 5630,
      "unclipped_grad_norm": 1.0480417013168335
    },
    {
      "bits_per_byte": 1.5462631652211978,
      "learning_rate": 8.930346725752419e-05,
      "loss_nats": 1.07178795337677,
      "step": 5640,
      "unclipped_grad_norm": 1.1326069831848145
    },
    {
      "bits_per_byte": 1.6566736193180087,
      "learning_rate": 8.88420482271411e-05,
      "loss_nats": 1.1483186483383179,
      "step": 5650,
      "unclipped_grad_norm": 1.174943447113037
    },
    {
      "bits_per_byte": 1.6152889180929386,
      "learning_rate": 8.838193052602691e-05,
      "loss_nats": 1.1196329593658447,
      "step": 5660,
      "unclipped_grad_norm": 1.1866281032562256
    },
    {
      "bits_per_byte": 1.6043573568387968,
      "learning_rate": 8.792312201632352e-05,
      "loss_nats": 1.112055778503418,
      "step": 5670,
      "unclipped_grad_norm": 1.1592957973480225
    },
    {
      "bits_per_byte": 1.6221047625292826,
      "learning_rate": 8.746563053780261e-05,
      "loss_nats": 1.1243573427200317,
      "step": 5680,
      "unclipped_grad_norm": 1.1609989404678345
    },
    {
      "bits_per_byte": 1.6421648189259206,
      "learning_rate": 8.700946390773114e-05,
      "loss_nats": 1.1382619142532349,
      "step": 5690,
      "unclipped_grad_norm": 1.1130645275115967
    },
    {
      "bits_per_byte": 1.598431178655366,
      "learning_rate": 8.655462992073833e-05,
      "loss_nats": 1.1079480648040771,
      "step": 5700,
      "unclipped_grad_norm": 1.1522045135498047
    },
    {
      "bits_per_byte": 1.6208697551134366,
      "learning_rate": 8.610113634868199e-05,
      "loss_nats": 1.1235013008117676,
      "step": 5710,
      "unclipped_grad_norm": 1.1070868968963623
    },
    {
      "bits_per_byte": 1.648464543427025,
      "learning_rate": 8.564899094051613e-05,
      "loss_nats": 1.14262855052948,
      "step": 5720,
      "unclipped_grad_norm": 1.2136235237121582
    },
    {
      "bits_per_byte": 1.5559942875723172,
      "learning_rate": 8.519820142215816e-05,
      "loss_nats": 1.0785330533981323,
      "step": 5730,
      "unclipped_grad_norm": 1.1237554550170898
    },
    {
      "bits_per_byte": 1.6165538504900345,
      "learning_rate": 8.474877549635728e-05,
      "loss_nats": 1.1205097436904907,
      "step": 5740,
      "unclipped_grad_norm": 1.1293457746505737
    },
    {
      "bits_per_byte": 1.5847578699811693,
      "learning_rate": 8.430072084256247e-05,
      "loss_nats": 1.0984704494476318,
      "step": 5750,
      "unclipped_grad_norm": 1.1838749647140503
    },
    {
      "bits_per_byte": 1.5751051717188433,
      "learning_rate": 8.385404511679161e-05,
      "loss_nats": 1.0917797088623047,
      "step": 5760,
      "unclipped_grad_norm": 1.1379411220550537
    },
    {
      "bits_per_byte": 1.5366526027083331,
      "learning_rate": 8.340875595150034e-05,
      "loss_nats": 1.0651264190673828,
      "step": 5770,
      "unclipped_grad_norm": 1.1725109815597534
    },
    {
      "bits_per_byte": 1.6123841311198657,
      "learning_rate": 8.296486095545203e-05,
      "loss_nats": 1.117619514465332,
      "step": 5780,
      "unclipped_grad_norm": 1.1070845127105713
    },
    {
      "bits_per_byte": 1.5625800192218067,
      "learning_rate": 8.252236771358729e-05,
      "loss_nats": 1.0830979347229004,
      "step": 5790,
      "unclipped_grad_norm": 1.0731046199798584
    },
    {
      "bits_per_byte": 1.581076581339451,
      "learning_rate": 8.208128378689484e-05,
      "loss_nats": 1.0959187746047974,
      "step": 5800,
      "unclipped_grad_norm": 1.1906232833862305
    },
    {
      "bits_per_byte": 1.6024834338749956,
      "learning_rate": 8.16416167122819e-05,
      "loss_nats": 1.1107568740844727,
      "step": 5810,
      "unclipped_grad_norm": 1.198943853378296
    },
    {
      "bits_per_byte": 1.6068543729466738,
      "learning_rate": 8.120337400244578e-05,
      "loss_nats": 1.1137865781784058,
      "step": 5820,
      "unclipped_grad_norm": 1.1112759113311768
    },
    {
      "bits_per_byte": 1.567694095327863,
      "learning_rate": 8.076656314574521e-05,
      "loss_nats": 1.0866427421569824,
      "step": 5830,
      "unclipped_grad_norm": 1.2037192583084106
    },
    {
      "bits_per_byte": 1.5486041930647423,
      "learning_rate": 8.03311916060725e-05,
      "loss_nats": 1.0734106302261353,
      "step": 5840,
      "unclipped_grad_norm": 1.1501859426498413
    },
    {
      "bits_per_byte": 1.540087784176667,
      "learning_rate": 7.989726682272606e-05,
      "loss_nats": 1.0675075054168701,
      "step": 5850,
      "unclipped_grad_norm": 1.1106252670288086
    },
    {
      "bits_per_byte": 1.591153904766757,
      "learning_rate": 7.946479621028318e-05,
      "loss_nats": 1.1029038429260254,
      "step": 5860,
      "unclipped_grad_norm": 1.267377257347107
    },
    {
      "bits_per_byte": 1.6187461133405827,
      "learning_rate": 7.903378715847343e-05,
      "loss_nats": 1.1220293045043945,
      "step": 5870,
      "unclipped_grad_norm": 1.1557857990264893
    },
    {
      "bits_per_byte": 1.6194158137830426,
      "learning_rate": 7.860424703205224e-05,
      "loss_nats": 1.1224935054779053,
      "step": 5880,
      "unclipped_grad_norm": 1.1605706214904785
    },
    {
      "bits_per_byte": 1.6297550667876128,
      "learning_rate": 7.817618317067534e-05,
      "loss_nats": 1.1296601295471191,
      "step": 5890,
      "unclipped_grad_norm": 1.1718353033065796
    },
    {
      "bits_per_byte": 1.5804839291245776,
      "learning_rate": 7.774960288877295e-05,
      "loss_nats": 1.0955079793930054,
      "step": 5900,
      "unclipped_grad_norm": 1.1900224685668945
    },
    {
      "bits_per_byte": 1.5770434161940672,
      "learning_rate": 7.732451347542522e-05,
      "loss_nats": 1.093123197555542,
      "step": 5910,
      "unclipped_grad_norm": 1.1988046169281006
    },
    {
      "bits_per_byte": 1.5761890063845807,
      "learning_rate": 7.690092219423722e-05,
      "loss_nats": 1.0925309658050537,
      "step": 5920,
      "unclipped_grad_norm": 1.1799134016036987
    },
    {
      "bits_per_byte": 1.6240858306845711,
      "learning_rate": 7.647883628321536e-05,
      "loss_nats": 1.1257305145263672,
      "step": 5930,
      "unclipped_grad_norm": 1.1704758405685425
    },
    {
      "bits_per_byte": 1.6230097352381225,
      "learning_rate": 7.605826295464316e-05,
      "loss_nats": 1.124984622001648,
      "step": 5940,
      "unclipped_grad_norm": 1.1741182804107666
    },
    {
      "bits_per_byte": 1.5942451209333652,
      "learning_rate": 7.563920939495851e-05,
      "loss_nats": 1.1050465106964111,
      "step": 5950,
      "unclipped_grad_norm": 1.223347544670105
    },
    {
      "bits_per_byte": 1.587844614598855,
      "learning_rate": 7.522168276463047e-05,
      "loss_nats": 1.1006100177764893,
      "step": 5960,
      "unclipped_grad_norm": 1.1676011085510254
    },
    {
      "bits_per_byte": 1.6162547726601841,
      "learning_rate": 7.480569019803719e-05,
      "loss_nats": 1.120302438735962,
      "step": 5970,
      "unclipped_grad_norm": 1.2063930034637451
    },
    {
      "bits_per_byte": 1.5550433954956966,
      "learning_rate": 7.439123880334388e-05,
      "loss_nats": 1.077873945236206,
      "step": 5980,
      "unclipped_grad_norm": 1.1823816299438477
    },
    {
      "bits_per_byte": 1.5403349232459567,
      "learning_rate": 7.39783356623813e-05,
      "loss_nats": 1.0676788091659546,
      "step": 5990,
      "unclipped_grad_norm": 1.1445503234863281
    },
    {
      "bits_per_byte": 1.6087774829486217,
      "learning_rate": 7.356698783052497e-05,
      "loss_nats": 1.1151195764541626,
      "step": 6000,
      "unclipped_grad_norm": 1.182093858718872
    },
    {
      "bits_per_byte": 1.585380103211991,
      "learning_rate": 7.315720233657431e-05,
      "loss_nats": 1.0989017486572266,
      "step": 6010,
      "unclipped_grad_norm": 1.1548501253128052
    },
    {
      "bits_per_byte": 1.57159036228316,
      "learning_rate": 7.274898618263286e-05,
      "loss_nats": 1.0893434286117554,
      "step": 6020,
      "unclipped_grad_norm": 1.132990837097168
    },
    {
      "bits_per_byte": 1.6225496816470646,
      "learning_rate": 7.234234634398831e-05,
      "loss_nats": 1.1246657371520996,
      "step": 6030,
      "unclipped_grad_norm": 1.1933883428573608
    },
    {
      "bits_per_byte": 1.5568839538252304,
      "learning_rate": 7.193728976899362e-05,
      "loss_nats": 1.0791497230529785,
      "step": 6040,
      "unclipped_grad_norm": 1.1697132587432861
    },
    {
      "bits_per_byte": 1.5810753774608948,
      "learning_rate": 7.153382337894797e-05,
      "loss_nats": 1.0959179401397705,
      "step": 6050,
      "unclipped_grad_norm": 1.1444475650787354
    },
    {
      "bits_per_byte": 1.6146938581209531,
      "learning_rate": 7.113195406797887e-05,
      "loss_nats": 1.119220495223999,
      "step": 6060,
      "unclipped_grad_norm": 1.1966716051101685
    },
    {
      "bits_per_byte": 1.5827436091742642,
      "learning_rate": 7.073168870292397e-05,
      "loss_nats": 1.097074270248413,
      "step": 6070,
      "unclipped_grad_norm": 1.1267356872558594
    },
    {
      "bits_per_byte": 1.5673551175230125,
      "learning_rate": 7.033303412321404e-05,
      "loss_nats": 1.0864077806472778,
      "step": 6080,
      "unclipped_grad_norm": 1.097261905670166
    },
    {
      "bits_per_byte": 1.5774123189801679,
      "learning_rate": 6.993599714075586e-05,
      "loss_nats": 1.0933789014816284,
      "step": 6090,
      "unclipped_grad_norm": 1.1282033920288086
    },
    {
      "bits_per_byte": 1.5779040173789847,
      "learning_rate": 6.954058453981609e-05,
      "loss_nats": 1.093719720840454,
      "step": 6100,
      "unclipped_grad_norm": 1.1281672716140747
    },
    {
      "bits_per_byte": 1.5504769121499873,
      "learning_rate": 6.914680307690504e-05,
      "loss_nats": 1.0747087001800537,
      "step": 6110,
      "unclipped_grad_norm": 1.1312857866287231
    },
    {
      "bits_per_byte": 1.5543487575688617,
      "learning_rate": 6.875465948066139e-05,
      "loss_nats": 1.0773924589157104,
      "step": 6120,
      "unclipped_grad_norm": 1.157880187034607
    },
    {
      "bits_per_byte": 1.5931889754744168,
      "learning_rate": 6.836416045173733e-05,
      "loss_nats": 1.1043144464492798,
      "step": 6130,
      "unclipped_grad_norm": 1.1723217964172363
    },
    {
      "bits_per_byte": 1.6084328297162926,
      "learning_rate": 6.797531266268377e-05,
      "loss_nats": 1.1148806810379028,
      "step": 6140,
      "unclipped_grad_norm": 1.1961045265197754
    },
    {
      "bits_per_byte": 1.542341444833573,
      "learning_rate": 6.758812275783663e-05,
      "loss_nats": 1.0690696239471436,
      "step": 6150,
      "unclipped_grad_norm": 1.1969319581985474
    },
    {
      "bits_per_byte": 1.6357159854665178,
      "learning_rate": 6.720259735320298e-05,
      "loss_nats": 1.1337919235229492,
      "step": 6160,
      "unclipped_grad_norm": 1.2412800788879395
    },
    {
      "bits_per_byte": 1.4931134748159045,
      "learning_rate": 6.681874303634842e-05,
      "loss_nats": 1.034947395324707,
      "step": 6170,
      "unclipped_grad_norm": 1.0997314453125
    },
    {
      "bits_per_byte": 1.5659319610871227,
      "learning_rate": 6.643656636628402e-05,
      "loss_nats": 1.0854213237762451,
      "step": 6180,
      "unclipped_grad_norm": 1.128227710723877
    },
    {
      "bits_per_byte": 1.581150533879322,
      "learning_rate": 6.605607387335466e-05,
      "loss_nats": 1.0959700345993042,
      "step": 6190,
      "unclipped_grad_norm": 1.1780462265014648
    },
    {
      "bits_per_byte": 1.5351257407339731,
      "learning_rate": 6.567727205912724e-05,
      "loss_nats": 1.064068078994751,
      "step": 6200,
      "unclipped_grad_norm": 1.13433837890625
    },
    {
      "bits_per_byte": 1.5793001725386888,
      "learning_rate": 6.530016739627959e-05,
      "loss_nats": 1.0946874618530273,
      "step": 6210,
      "unclipped_grad_norm": 1.1643598079681396
    },
    {
      "bits_per_byte": 1.5609047362197521,
      "learning_rate": 6.492476632848993e-05,
      "loss_nats": 1.0819367170333862,
      "step": 6220,
      "unclipped_grad_norm": 1.131072998046875
    },
    {
      "bits_per_byte": 1.5253499028936293,
      "learning_rate": 6.455107527032678e-05,
      "loss_nats": 1.0572919845581055,
      "step": 6230,
      "unclipped_grad_norm": 1.1126173734664917
    },
    {
      "bits_per_byte": 1.5400140036194467,
      "learning_rate": 6.417910060713926e-05,
      "loss_nats": 1.0674563646316528,
      "step": 6240,
      "unclipped_grad_norm": 1.1826318502426147
    },
    {
      "bits_per_byte": 1.5344780540708238,
      "learning_rate": 6.380884869494799e-05,
      "loss_nats": 1.0636191368103027,
      "step": 6250,
      "unclipped_grad_norm": 1.1302508115768433
    },
    {
      "bits_per_byte": 1.600364263651064,
      "learning_rate": 6.34403258603367e-05,
      "loss_nats": 1.109287977218628,
      "step": 6260,
      "unclipped_grad_norm": 1.1724376678466797
    },
    {
      "bits_per_byte": 1.5544997583363194,
      "learning_rate": 6.30735384003437e-05,
      "loss_nats": 1.077497124671936,
      "step": 6270,
      "unclipped_grad_norm": 1.081700086593628
    },
    {
      "bits_per_byte": 1.6754886933050508,
      "learning_rate": 6.270849258235484e-05,
      "loss_nats": 1.161360263824463,
      "step": 6280,
      "unclipped_grad_norm": 1.2664587497711182
    },
    {
      "bits_per_byte": 1.5413214157313058,
      "learning_rate": 6.234519464399581e-05,
      "loss_nats": 1.0683625936508179,
      "step": 6290,
      "unclipped_grad_norm": 1.2083820104599
    },
    {
      "bits_per_byte": 1.5888871734283854,
      "learning_rate": 6.198365079302614e-05,
      "loss_nats": 1.101332664489746,
      "step": 6300,
      "unclipped_grad_norm": 1.2475577592849731
    },
    {
      "bits_per_byte": 1.6228484155116132,
      "learning_rate": 6.16238672072326e-05,
      "loss_nats": 1.1248728036880493,
      "step": 6310,
      "unclipped_grad_norm": 1.2506906986236572
    },
    {
      "bits_per_byte": 1.5728649257087044,
      "learning_rate": 6.126585003432406e-05,
      "loss_nats": 1.0902268886566162,
      "step": 6320,
      "unclipped_grad_norm": 1.1333374977111816
    },
    {
      "bits_per_byte": 1.5253509347895344,
      "learning_rate": 6.090960539182614e-05,
      "loss_nats": 1.0572926998138428,
      "step": 6330,
      "unclipped_grad_norm": 1.180550217628479
    },
    {
      "bits_per_byte": 1.5807446548232857,
      "learning_rate": 6.0555139366976935e-05,
      "loss_nats": 1.0956887006759644,
      "step": 6340,
      "unclipped_grad_norm": 1.1644010543823242
    },
    {
      "bits_per_byte": 1.5717518539923203,
      "learning_rate": 6.020245801662274e-05,
      "loss_nats": 1.0894553661346436,
      "step": 6350,
      "unclipped_grad_norm": 1.142453908920288
    },
    {
      "bits_per_byte": 1.5231024336121541,
      "learning_rate": 5.985156736711483e-05,
      "loss_nats": 1.0557341575622559,
      "step": 6360,
      "unclipped_grad_norm": 1.1504549980163574
    },
    {
      "bits_per_byte": 1.6077399116159663,
      "learning_rate": 5.950247341420618e-05,
      "loss_nats": 1.1144003868103027,
      "step": 6370,
      "unclipped_grad_norm": 1.2130403518676758
    },
    {
      "bits_per_byte": 1.5316929670227515,
      "learning_rate": 5.915518212294935e-05,
      "loss_nats": 1.0616886615753174,
      "step": 6380,
      "unclipped_grad_norm": 1.1643673181533813
    },
    {
      "bits_per_byte": 1.5801346323606753,
      "learning_rate": 5.880969942759425e-05,
      "loss_nats": 1.0952658653259277,
      "step": 6390,
      "unclipped_grad_norm": 1.1411337852478027
    },
    {
      "bits_per_byte": 1.5548263533903077,
      "learning_rate": 5.846603123148688e-05,
      "loss_nats": 1.077723503112793,
      "step": 6400,
      "unclipped_grad_norm": 1.1154334545135498
    },
    {
      "bits_per_byte": 1.5450890396637573,
      "learning_rate": 5.812418340696858e-05,
      "loss_nats": 1.0709741115570068,
      "step": 6410,
      "unclipped_grad_norm": 1.137816071510315
    },
    {
      "bits_per_byte": 1.5323659351355776,
      "learning_rate": 5.778416179527541e-05,
      "loss_nats": 1.0621551275253296,
      "step": 6420,
      "unclipped_grad_norm": 1.1425901651382446
    },
    {
      "bits_per_byte": 1.5643414655319436,
      "learning_rate": 5.7445972206438635e-05,
      "loss_nats": 1.0843188762664795,
      "step": 6430,
      "unclipped_grad_norm": 1.2278902530670166
    },
    {
      "bits_per_byte": 1.5636070996127593,
      "learning_rate": 5.710962041918516e-05,
      "loss_nats": 1.0838098526000977,
      "step": 6440,
      "unclipped_grad_norm": 1.204923152923584
    },
    {
      "bits_per_byte": 1.5427707135301267,
      "learning_rate": 5.677511218083908e-05,
      "loss_nats": 1.0693671703338623,
      "step": 6450,
      "unclipped_grad_norm": 1.1532135009765625
    },
    {
      "bits_per_byte": 1.5522318231193915,
      "learning_rate": 5.644245320722316e-05,
      "loss_nats": 1.0759251117706299,
      "step": 6460,
      "unclipped_grad_norm": 1.5724140405654907
    },
    {
      "bits_per_byte": 1.5677630603708592,
      "learning_rate": 5.61116491825615e-05,
      "loss_nats": 1.0866905450820923,
      "step": 6470,
      "unclipped_grad_norm": 1.182875394821167
    },
    {
      "bits_per_byte": 1.5839062118940967,
      "learning_rate": 5.5782705759382104e-05,
      "loss_nats": 1.0978801250457764,
      "step": 6480,
      "unclipped_grad_norm": 1.211335301399231
    },
    {
      "bits_per_byte": 1.561875234318571,
      "learning_rate": 5.5455628558420546e-05,
      "loss_nats": 1.0826094150543213,
      "step": 6490,
      "unclipped_grad_norm": 1.2880244255065918
    },
    {
      "bits_per_byte": 1.577221590220361,
      "learning_rate": 5.513042316852368e-05,
      "loss_nats": 1.0932466983795166,
      "step": 6500,
      "unclipped_grad_norm": 1.2181490659713745
    },
    {
      "bits_per_byte": 1.5067461955844976,
      "learning_rate": 5.480709514655443e-05,
      "loss_nats": 1.0443968772888184,
      "step": 6510,
      "unclipped_grad_norm": 1.1337521076202393
    },
    {
      "bits_per_byte": 1.4936307986296997,
      "learning_rate": 5.448565001729653e-05,
      "loss_nats": 1.0353059768676758,
      "step": 6520,
      "unclipped_grad_norm": 1.2117457389831543
    },
    {
      "bits_per_byte": 1.5784743118492457,
      "learning_rate": 5.41660932733604e-05,
      "loss_nats": 1.0941150188446045,
      "step": 6530,
      "unclipped_grad_norm": 1.2337595224380493
    },
    {
      "bits_per_byte": 1.5448394928373554,
      "learning_rate": 5.384843037508902e-05,
      "loss_nats": 1.0708011388778687,
      "step": 6540,
      "unclipped_grad_norm": 1.1313438415527344
    },
    {
      "bits_per_byte": 1.5964084906985685,
      "learning_rate": 5.353266675046498e-05,
      "loss_nats": 1.1065460443496704,
      "step": 6550,
      "unclipped_grad_norm": 1.2166260480880737
    },
    {
      "bits_per_byte": 1.5631484218829081,
      "learning_rate": 5.321880779501729e-05,
      "loss_nats": 1.0834919214248657,
      "step": 6560,
      "unclipped_grad_norm": 1.1645575761795044
    },
    {
      "bits_per_byte": 1.4810062401604644,
      "learning_rate": 5.290685887172962e-05,
      "loss_nats": 1.0265552997589111,
      "step": 6570,
      "unclipped_grad_norm": 1.2005696296691895
    },
    {
      "bits_per_byte": 1.5898447728283895,
      "learning_rate": 5.2596825310948295e-05,
      "loss_nats": 1.1019964218139648,
      "step": 6580,
      "unclipped_grad_norm": 1.1795482635498047
    },
    {
      "bits_per_byte": 1.5634753609021985,
      "learning_rate": 5.228871241029154e-05,
      "loss_nats": 1.0837185382843018,
      "step": 6590,
      "unclipped_grad_norm": 1.1887481212615967
    },
    {
      "bits_per_byte": 1.47844799822888,
      "learning_rate": 5.198252543455865e-05,
      "loss_nats": 1.0247820615768433,
      "step": 6600,
      "unclipped_grad_norm": 1.1429756879806519
    },
    {
      "bits_per_byte": 1.6001293353499855,
      "learning_rate": 5.167826961564027e-05,
      "loss_nats": 1.1091251373291016,
      "step": 6610,
      "unclipped_grad_norm": 1.1761175394058228
    },
    {
      "bits_per_byte": 1.6007649832275743,
      "learning_rate": 5.137595015242884e-05,
      "loss_nats": 1.1095657348632812,
      "step": 6620,
      "unclipped_grad_norm": 1.2725090980529785
    },
    {
      "bits_per_byte": 1.5866947385951862,
      "learning_rate": 5.1075572210729925e-05,
      "loss_nats": 1.0998129844665527,
      "step": 6630,
      "unclipped_grad_norm": 1.242309808731079
    },
    {
      "bits_per_byte": 1.557243053600232,
      "learning_rate": 5.07771409231737e-05,
      "loss_nats": 1.0793986320495605,
      "step": 6640,
      "unclipped_grad_norm": 1.2104532718658447
    },
    {
      "bits_per_byte": 1.5462134622350983,
      "learning_rate": 5.048066138912756e-05,
      "loss_nats": 1.0717535018920898,
      "step": 6650,
      "unclipped_grad_norm": 1.1630686521530151
    },
    {
      "bits_per_byte": 1.588834890702523,
      "learning_rate": 5.018613867460863e-05,
      "loss_nats": 1.1012964248657227,
      "step": 6660,
      "unclipped_grad_norm": 1.204805612564087
    },
    {
      "bits_per_byte": 1.587033544417386,
      "learning_rate": 4.989357781219755e-05,
      "loss_nats": 1.1000478267669678,
      "step": 6670,
      "unclipped_grad_norm": 1.2499597072601318
    },
    {
      "bits_per_byte": 1.5912992301067361,
      "learning_rate": 4.960298380095222e-05,
      "loss_nats": 1.1030045747756958,
      "step": 6680,
      "unclipped_grad_norm": 1.2160732746124268
    },
    {
      "bits_per_byte": 1.56477726956923,
      "learning_rate": 4.931436160632256e-05,
      "loss_nats": 1.0846209526062012,
      "step": 6690,
      "unclipped_grad_norm": 1.2467725276947021
    },
    {
      "bits_per_byte": 1.6040180350686444,
      "learning_rate": 4.9027716160065485e-05,
      "loss_nats": 1.1118205785751343,
      "step": 6700,
      "unclipped_grad_norm": 1.267776370048523
    },
    {
      "bits_per_byte": 1.537844786444114,
      "learning_rate": 4.874305236016084e-05,
      "loss_nats": 1.0659527778625488,
      "step": 6710,
      "unclipped_grad_norm": 1.2301596403121948
    },
    {
      "bits_per_byte": 1.5327245189626266,
      "learning_rate": 4.846037507072752e-05,
      "loss_nats": 1.062403678894043,
      "step": 6720,
      "unclipped_grad_norm": 1.180388331413269
    },
    {
      "bits_per_byte": 1.5337890915714674,
      "learning_rate": 4.817968912194049e-05,
      "loss_nats": 1.0631415843963623,
      "step": 6730,
      "unclipped_grad_norm": 1.2269387245178223
    },
    {
      "bits_per_byte": 1.5501341507268178,
      "learning_rate": 4.790099930994813e-05,
      "loss_nats": 1.074471116065979,
      "step": 6740,
      "unclipped_grad_norm": 1.1682806015014648
    },
    {
      "bits_per_byte": 1.5184903748639662,
      "learning_rate": 4.762431039679047e-05,
      "loss_nats": 1.0525373220443726,
      "step": 6750,
      "unclipped_grad_norm": 1.23702871799469
    },
    {
      "bits_per_byte": 1.500455070215936,
      "learning_rate": 4.734962711031755e-05,
      "loss_nats": 1.0400362014770508,
      "step": 6760,
      "unclipped_grad_norm": 1.1704752445220947
    },
    {
      "bits_per_byte": 1.5177950490065277,
      "learning_rate": 4.707695414410892e-05,
      "loss_nats": 1.0520553588867188,
      "step": 6770,
      "unclipped_grad_norm": 1.209053874015808
    },
    {
      "bits_per_byte": 1.5265771710901863,
      "learning_rate": 4.680629615739322e-05,
      "loss_nats": 1.0581426620483398,
      "step": 6780,
      "unclipped_grad_norm": 1.2882649898529053
    },
    {
      "bits_per_byte": 1.6213150181965206,
      "learning_rate": 4.653765777496863e-05,
      "loss_nats": 1.1238099336624146,
      "step": 6790,
      "unclipped_grad_norm": 1.1679272651672363
    },
    {
      "bits_per_byte": 1.5863846538756805,
      "learning_rate": 4.6271043587123986e-05,
      "loss_nats": 1.0995980501174927,
      "step": 6800,
      "unclipped_grad_norm": 1.280809998512268
    },
    {
      "bits_per_byte": 1.578056394007649,
      "learning_rate": 4.600645814956006e-05,
      "loss_nats": 1.093825340270996,
      "step": 6810,
      "unclipped_grad_norm": 1.24360191822052
    },
    {
      "bits_per_byte": 1.5169633409069554,
      "learning_rate": 4.574390598331204e-05,
      "loss_nats": 1.0514788627624512,
      "step": 6820,
      "unclipped_grad_norm": 1.2292150259017944
    },
    {
      "bits_per_byte": 1.5347601056182387,
      "learning_rate": 4.548339157467199e-05,
      "loss_nats": 1.063814640045166,
      "step": 6830,
      "unclipped_grad_norm": 1.242634892463684
    },
    {
      "bits_per_byte": 1.5555476486280264,
      "learning_rate": 4.52249193751124e-05,
      "loss_nats": 1.078223466873169,
      "step": 6840,
      "unclipped_grad_norm": 1.2094796895980835
    },
    {
      "bits_per_byte": 1.5217322478327302,
      "learning_rate": 4.4968493801209985e-05,
      "loss_nats": 1.0547844171524048,
      "step": 6850,
      "unclipped_grad_norm": 1.1779916286468506
    },
    {
      "bits_per_byte": 1.549958384457636,
      "learning_rate": 4.471411923457034e-05,
      "loss_nats": 1.074349284172058,
      "step": 6860,
      "unclipped_grad_norm": 1.2639429569244385
    },
    {
      "bits_per_byte": 1.5571720247654257,
      "learning_rate": 4.446180002175291e-05,
      "loss_nats": 1.079349398612976,
      "step": 6870,
      "unclipped_grad_norm": 1.2982949018478394
    },
    {
      "bits_per_byte": 1.597770421310751,
      "learning_rate": 4.421154047419693e-05,
      "loss_nats": 1.107490062713623,
      "step": 6880,
      "unclipped_grad_norm": 1.2857638597488403
    },
    {
      "bits_per_byte": 1.4860104193526196,
      "learning_rate": 4.396334486814753e-05,
      "loss_nats": 1.0300239324569702,
      "step": 6890,
      "unclipped_grad_norm": 1.2555214166641235
    },
    {
      "bits_per_byte": 1.5064882216082034,
      "learning_rate": 4.371721744458285e-05,
      "loss_nats": 1.0442180633544922,
      "step": 6900,
      "unclipped_grad_norm": 1.202001690864563
    },
    {
      "bits_per_byte": 1.685866814388716,
      "learning_rate": 4.347316240914145e-05,
      "loss_nats": 1.1685538291931152,
      "step": 6910,
      "unclipped_grad_norm": 1.3379162549972534
    },
    {
      "bits_per_byte": 1.5053493524941899,
      "learning_rate": 4.3231183932050546e-05,
      "loss_nats": 1.043428659439087,
      "step": 6920,
      "unclipped_grad_norm": 1.1770343780517578
    },
    {
      "bits_per_byte": 1.516240325842728,
      "learning_rate": 4.299128614805464e-05,
      "loss_nats": 1.0509777069091797,
      "step": 6930,
      "unclipped_grad_norm": 1.2326538562774658
    },
    {
      "bits_per_byte": 1.5463919802266939,
      "learning_rate": 4.2753473156344965e-05,
      "loss_nats": 1.0718772411346436,
      "step": 6940,
      "unclipped_grad_norm": 1.2218632698059082
    },
    {
      "bits_per_byte": 1.5410003241221448,
      "learning_rate": 4.251774902048942e-05,
      "loss_nats": 1.0681400299072266,
      "step": 6950,
      "unclipped_grad_norm": 1.2248505353927612
    },
    {
      "bits_per_byte": 1.574234251574874,
      "learning_rate": 4.228411776836306e-05,
      "loss_nats": 1.0911760330200195,
      "step": 6960,
      "unclipped_grad_norm": 1.2728900909423828
    },
    {
      "bits_per_byte": 1.556295085228676,
      "learning_rate": 4.2052583392079427e-05,
      "loss_nats": 1.0787415504455566,
      "step": 6970,
      "unclipped_grad_norm": 1.2423036098480225
    },
    {
      "bits_per_byte": 1.6109532354646872,
      "learning_rate": 4.182314984792214e-05,
      "loss_nats": 1.1166276931762695,
      "step": 6980,
      "unclipped_grad_norm": 1.2731002569198608
    },
    {
      "bits_per_byte": 1.5507145921734797,
      "learning_rate": 4.159582105627748e-05,
      "loss_nats": 1.074873447418213,
      "step": 6990,
      "unclipped_grad_norm": 1.2125478982925415
    },
    {
      "bits_per_byte": 1.506560110356264,
      "learning_rate": 4.137060090156724e-05,
      "loss_nats": 1.0442678928375244,
      "step": 7000,
      "unclipped_grad_norm": 1.2366911172866821
    },
    {
      "bits_per_byte": 1.5357397187975532,
      "learning_rate": 4.114749323218249e-05,
      "loss_nats": 1.0644936561584473,
      "step": 7010,
      "unclipped_grad_norm": 1.2639423608779907
    },
    {
      "bits_per_byte": 1.5037485379799589,
      "learning_rate": 4.0926501860417706e-05,
      "loss_nats": 1.0423190593719482,
      "step": 7020,
      "unclipped_grad_norm": 1.2011712789535522
    },
    {
      "bits_per_byte": 1.528263632964547,
      "learning_rate": 4.070763056240571e-05,
      "loss_nats": 1.0593116283416748,
      "step": 7030,
      "unclipped_grad_norm": 1.1849545240402222
    },
    {
      "bits_per_byte": 1.5983039114937276,
      "learning_rate": 4.049088307805306e-05,
      "loss_nats": 1.1078598499298096,
      "step": 7040,
      "unclipped_grad_norm": 1.3057087659835815
    },
    {
      "bits_per_byte": 1.5441888824691414,
      "learning_rate": 4.027626311097629e-05,
      "loss_nats": 1.070350170135498,
      "step": 7050,
      "unclipped_grad_norm": 1.2155181169509888
    },
    {
      "bits_per_byte": 1.5741694141154987,
      "learning_rate": 4.006377432843846e-05,
      "loss_nats": 1.0911310911178589,
      "step": 7060,
      "unclipped_grad_norm": 1.3050142526626587
    },
    {
      "bits_per_byte": 1.5094996378248111,
      "learning_rate": 3.985342036128656e-05,
      "loss_nats": 1.0463054180145264,
      "step": 7070,
      "unclipped_grad_norm": 1.1835323572158813
    },
    {
      "bits_per_byte": 1.4442604269771167,
      "learning_rate": 3.964520480388956e-05,
      "loss_nats": 1.0010850429534912,
      "step": 7080,
      "unclipped_grad_norm": 1.167786717414856
    },
    {
      "bits_per_byte": 1.5378045425038123,
      "learning_rate": 3.943913121407678e-05,
      "loss_nats": 1.065924882888794,
      "step": 7090,
      "unclipped_grad_norm": 1.240479588508606
    },
    {
      "bits_per_byte": 1.5019334330827527,
      "learning_rate": 3.923520311307735e-05,
      "loss_nats": 1.0410609245300293,
      "step": 7100,
      "unclipped_grad_norm": 1.1977860927581787
    },
    {
      "bits_per_byte": 1.5780336922977352,
      "learning_rate": 3.9033423985459816e-05,
      "loss_nats": 1.0938096046447754,
      "step": 7110,
      "unclipped_grad_norm": 1.2797318696975708
    },
    {
      "bits_per_byte": 1.56051364767169,
      "learning_rate": 3.88337972790728e-05,
      "loss_nats": 1.0816656351089478,
      "step": 7120,
      "unclipped_grad_norm": 1.297847032546997
    },
    {
      "bits_per_byte": 1.4993894657111901,
      "learning_rate": 3.8636326404985845e-05,
      "loss_nats": 1.0392975807189941,
      "step": 7130,
      "unclipped_grad_norm": 1.224601149559021
    },
    {
      "bits_per_byte": 1.6077283887783587,
      "learning_rate": 3.844101473743143e-05,
      "loss_nats": 1.1143923997879028,
      "step": 7140,
      "unclipped_grad_norm": 1.3082928657531738
    },
    {
      "bits_per_byte": 1.624090990164097,
      "learning_rate": 3.824786561374701e-05,
      "loss_nats": 1.1257340908050537,
      "step": 7150,
      "unclipped_grad_norm": 1.2189027070999146
    },
    {
      "bits_per_byte": 1.5477993142587043,
      "learning_rate": 3.805688233431824e-05,
      "loss_nats": 1.0728527307510376,
      "step": 7160,
      "unclipped_grad_norm": 1.2660733461380005
    },
    {
      "bits_per_byte": 1.5516317756505311,
      "learning_rate": 3.786806816252236e-05,
      "loss_nats": 1.0755091905593872,
      "step": 7170,
      "unclipped_grad_norm": 1.2526509761810303
    },
    {
      "bits_per_byte": 1.4692106380683878,
      "learning_rate": 3.768142632467269e-05,
      "loss_nats": 1.0183792114257812,
      "step": 7180,
      "unclipped_grad_norm": 1.1470739841461182
    },
    {
      "bits_per_byte": 1.4673153892558797,
      "learning_rate": 3.7496960009963236e-05,
      "loss_nats": 1.0170655250549316,
      "step": 7190,
      "unclipped_grad_norm": 1.2286535501480103
    },
    {
      "bits_per_byte": 1.553365876719181,
      "learning_rate": 3.7314672370414325e-05,
      "loss_nats": 1.0767111778259277,
      "step": 7200,
      "unclipped_grad_norm": 1.2306180000305176
    },
    {
      "bits_per_byte": 1.561453876823957,
      "learning_rate": 3.713456652081882e-05,
      "loss_nats": 1.0823173522949219,
      "step": 7210,
      "unclipped_grad_norm": 1.2458006143569946
    },
    {
      "bits_per_byte": 1.5086355969868763,
      "learning_rate": 3.695664553868873e-05,
      "loss_nats": 1.0457065105438232,
      "step": 7220,
      "unclipped_grad_norm": 1.1648746728897095
    },
    {
      "bits_per_byte": 1.5304512522835219,
      "learning_rate": 3.678091246420277e-05,
      "loss_nats": 1.0608279705047607,
      "step": 7230,
      "unclipped_grad_norm": 1.2855236530303955
    },
    {
      "bits_per_byte": 1.493701999447157,
      "learning_rate": 3.660737030015427e-05,
      "loss_nats": 1.0353553295135498,
      "step": 7240,
      "unclipped_grad_norm": 1.2490638494491577
    },
    {
      "bits_per_byte": 1.5430964486708607,
      "learning_rate": 3.6436022011900056e-05,
      "loss_nats": 1.0695929527282715,
      "step": 7250,
      "unclipped_grad_norm": 1.286012887954712
    },
    {
      "bits_per_byte": 1.5196966611771179,
      "learning_rate": 3.626687052730954e-05,
      "loss_nats": 1.0533734560012817,
      "step": 7260,
      "unclipped_grad_norm": 1.2369425296783447
    },
    {
      "bits_per_byte": 1.5993731276341416,
      "learning_rate": 3.6099918736714955e-05,
      "loss_nats": 1.1086009740829468,
      "step": 7270,
      "unclipped_grad_norm": 1.2697807550430298
    },
    {
      "bits_per_byte": 1.5305549578219921,
      "learning_rate": 3.5935169492861716e-05,
      "loss_nats": 1.0608998537063599,
      "step": 7280,
      "unclipped_grad_norm": 1.2743209600448608
    },
    {
      "bits_per_byte": 1.5476247518680786,
      "learning_rate": 3.5772625610859886e-05,
      "loss_nats": 1.0727317333221436,
      "step": 7290,
      "unclipped_grad_norm": 1.2424356937408447
    },
    {
      "bits_per_byte": 1.5365045256459402,
      "learning_rate": 3.5612289868135887e-05,
      "loss_nats": 1.0650237798690796,
      "step": 7300,
      "unclipped_grad_norm": 1.298574686050415
    },
    {
      "bits_per_byte": 1.5764911799021466,
      "learning_rate": 3.545416500438522e-05,
      "loss_nats": 1.0927404165267944,
      "step": 7310,
      "unclipped_grad_norm": 1.3189024925231934
    },
    {
      "bits_per_byte": 1.5490052566065542,
      "learning_rate": 3.52982537215255e-05,
      "loss_nats": 1.0736886262893677,
      "step": 7320,
      "unclipped_grad_norm": 1.2497724294662476
    },
    {
      "bits_per_byte": 1.5554974296939745,
      "learning_rate": 3.5144558683650396e-05,
      "loss_nats": 1.0781886577606201,
      "step": 7330,
      "unclipped_grad_norm": 1.279703140258789
    },
    {
      "bits_per_byte": 1.5241250424541843,
      "learning_rate": 3.499308251698404e-05,
      "loss_nats": 1.0564429759979248,
      "step": 7340,
      "unclipped_grad_norm": 1.2488007545471191
    },
    {
      "bits_per_byte": 1.57090466745417,
      "learning_rate": 3.484382780983616e-05,
      "loss_nats": 1.0888681411743164,
      "step": 7350,
      "unclipped_grad_norm": 1.238014578819275
    },
    {
      "bits_per_byte": 1.5500565865512788,
      "learning_rate": 3.469679711255795e-05,
      "loss_nats": 1.0744173526763916,
      "step": 7360,
      "unclipped_grad_norm": 1.2180218696594238
    },
    {
      "bits_per_byte": 1.5597980278614498,
      "learning_rate": 3.4551992937498284e-05,
      "loss_nats": 1.081169605255127,
      "step": 7370,
      "unclipped_grad_norm": 1.2368392944335938
    },
    {
      "bits_per_byte": 1.6215841710451209,
      "learning_rate": 3.440941775896104e-05,
      "loss_nats": 1.1239964962005615,
      "step": 7380,
      "unclipped_grad_norm": 1.3172329664230347
    },
    {
      "bits_per_byte": 1.5928735592927343,
      "learning_rate": 3.4269074013162596e-05,
      "loss_nats": 1.1040958166122437,
      "step": 7390,
      "unclipped_grad_norm": 1.3043875694274902
    },
    {
      "bits_per_byte": 1.5057738056765193,
      "learning_rate": 3.41309640981904e-05,
      "loss_nats": 1.0437228679656982,
      "step": 7400,
      "unclipped_grad_norm": 1.2024799585342407
    },
    {
      "bits_per_byte": 1.5152624324899222,
      "learning_rate": 3.3995090373961795e-05,
      "loss_nats": 1.050299882888794,
      "step": 7410,
      "unclipped_grad_norm": 1.2088910341262817
    },
    {
      "bits_per_byte": 1.597047062281222,
      "learning_rate": 3.3861455162183876e-05,
      "loss_nats": 1.1069886684417725,
      "step": 7420,
      "unclipped_grad_norm": 1.3177212476730347
    },
    {
      "bits_per_byte": 1.5299019396966662,
      "learning_rate": 3.3730060746313664e-05,
      "loss_nats": 1.0604472160339355,
      "step": 7430,
      "unclipped_grad_norm": 1.290786862373352
    },
    {
      "bits_per_byte": 1.5250764504787575,
      "learning_rate": 3.3600909371519225e-05,
      "loss_nats": 1.0571024417877197,
      "step": 7440,
      "unclipped_grad_norm": 1.2367006540298462
    },
    {
      "bits_per_byte": 1.4892672548120085,
      "learning_rate": 3.3474003244641194e-05,
      "loss_nats": 1.0322813987731934,
      "step": 7450,
      "unclipped_grad_norm": 1.3495734930038452
    },
    {
      "bits_per_byte": 1.5046927227331957,
      "learning_rate": 3.334934453415514e-05,
      "loss_nats": 1.042973518371582,
      "step": 7460,
      "unclipped_grad_norm": 1.2188435792922974
    },
    {
      "bits_per_byte": 1.5481360562890936,
      "learning_rate": 3.3226935370134486e-05,
      "loss_nats": 1.073086142539978,
      "step": 7470,
      "unclipped_grad_norm": 1.1978274583816528
    },
    {
      "bits_per_byte": 1.4779339420854511,
      "learning_rate": 3.31067778442141e-05,
      "loss_nats": 1.024425745010376,
      "step": 7480,
      "unclipped_grad_norm": 1.25648832321167
    },
    {
      "bits_per_byte": 1.5058897219832008,
      "learning_rate": 3.298887400955457e-05,
      "loss_nats": 1.0438032150268555,
      "step": 7490,
      "unclipped_grad_norm": 1.2401517629623413
    },
    {
      "bits_per_byte": 1.5728836718176484,
      "learning_rate": 3.287322588080714e-05,
      "loss_nats": 1.0902398824691772,
      "step": 7500,
      "unclipped_grad_norm": 1.2475262880325317
    },
    {
      "bits_per_byte": 1.5378353273983165,
      "learning_rate": 3.275983543407924e-05,
      "loss_nats": 1.0659462213516235,
      "step": 7510,
      "unclipped_grad_norm": 1.2484221458435059
    },
    {
      "bits_per_byte": 1.5312337733449477,
      "learning_rate": 3.2648704606900735e-05,
      "loss_nats": 1.0613703727722168,
      "step": 7520,
      "unclipped_grad_norm": 1.1939175128936768
    },
    {
      "bits_per_byte": 1.5599504044901142,
      "learning_rate": 3.253983529819086e-05,
      "loss_nats": 1.081275224685669,
      "step": 7530,
      "unclipped_grad_norm": 1.2631168365478516
    },
    {
      "bits_per_byte": 1.5976060058965262,
      "learning_rate": 3.243322936822571e-05,
      "loss_nats": 1.1073760986328125,
      "step": 7540,
      "unclipped_grad_norm": 1.3131186962127686
    },
    {
      "bits_per_byte": 1.5182279293387495,
      "learning_rate": 3.232888863860648e-05,
      "loss_nats": 1.052355408668518,
      "step": 7550,
      "unclipped_grad_norm": 1.2190253734588623
    },
    {
      "bits_per_byte": 1.5653272700866891,
      "learning_rate": 3.222681489222838e-05,
      "loss_nats": 1.0850021839141846,
      "step": 7560,
      "unclipped_grad_norm": 1.2437095642089844
    },
    {
      "bits_per_byte": 1.566524613301996,
      "learning_rate": 3.212700987325009e-05,
      "loss_nats": 1.085832118988037,
      "step": 7570,
      "unclipped_grad_norm": 1.2195208072662354
    },
    {
      "bits_per_byte": 1.514368810636039,
      "learning_rate": 3.202947528706403e-05,
      "loss_nats": 1.049680471420288,
      "step": 7580,
      "unclipped_grad_norm": 1.2027956247329712
    },
    {
      "bits_per_byte": 1.5218865162705542,
      "learning_rate": 3.1934212800267116e-05,
      "loss_nats": 1.0548913478851318,
      "step": 7590,
      "unclipped_grad_norm": 1.22147798538208
    },
    {
      "bits_per_byte": 1.5124059726417418,
      "learning_rate": 3.1841224040632484e-05,
      "loss_nats": 1.048319935798645,
      "step": 7600,
      "unclipped_grad_norm": 1.2925957441329956
    },
    {
      "bits_per_byte": 1.451400458710337,
      "learning_rate": 3.175051059708143e-05,
      "loss_nats": 1.0060341358184814,
      "step": 7610,
      "unclipped_grad_norm": 1.22130286693573
    },
    {
      "bits_per_byte": 1.554240580481469,
      "learning_rate": 3.1662074019656414e-05,
      "loss_nats": 1.077317476272583,
      "step": 7620,
      "unclipped_grad_norm": 1.2743861675262451
    },
    {
      "bits_per_byte": 1.5089792183233002,
      "learning_rate": 3.157591581949455e-05,
      "loss_nats": 1.0459446907043457,
      "step": 7630,
      "unclipped_grad_norm": 1.1950352191925049
    },
    {
      "bits_per_byte": 1.523324463214418,
      "learning_rate": 3.149203746880175e-05,
      "loss_nats": 1.055888056755066,
      "step": 7640,
      "unclipped_grad_norm": 1.2816646099090576
    },
    {
      "bits_per_byte": 1.6003928127711071,
      "learning_rate": 3.141044040082763e-05,
      "loss_nats": 1.1093077659606934,
      "step": 7650,
      "unclipped_grad_norm": 1.3542457818984985
    },
    {
      "bits_per_byte": 1.5396506042781735,
      "learning_rate": 3.133112600984091e-05,
      "loss_nats": 1.067204475402832,
      "step": 7660,
      "unclipped_grad_norm": 1.277113437652588
    },
    {
      "bits_per_byte": 1.515347907867401,
      "learning_rate": 3.125409565110571e-05,
      "loss_nats": 1.0503591299057007,
      "step": 7670,
      "unclipped_grad_norm": 1.3265182971954346
    },
    {
      "bits_per_byte": 1.4941618810555641,
      "learning_rate": 3.117935064085835e-05,
      "loss_nats": 1.0356740951538086,
      "step": 7680,
      "unclipped_grad_norm": 1.2858244180679321
    },
    {
      "bits_per_byte": 1.5394590156051124,
      "learning_rate": 3.110689225628479e-05,
      "loss_nats": 1.0670716762542725,
      "step": 7690,
      "unclipped_grad_norm": 1.2726463079452515
    },
    {
      "bits_per_byte": 1.5472200766905984,
      "learning_rate": 3.1036721735498926e-05,
      "loss_nats": 1.0724512338638306,
      "step": 7700,
      "unclipped_grad_norm": 1.3285934925079346
    },
    {
      "bits_per_byte": 1.4615679010466955,
      "learning_rate": 3.096884027752132e-05,
      "loss_nats": 1.013081669807434,
      "step": 7710,
      "unclipped_grad_norm": 1.2261079549789429
    },
    {
      "bits_per_byte": 1.4667946257890672,
      "learning_rate": 3.090324904225885e-05,
      "loss_nats": 1.0167045593261719,
      "step": 7720,
      "unclipped_grad_norm": 1.24894380569458
    },
    {
      "bits_per_byte": 1.5217016349208765,
      "learning_rate": 3.083994915048472e-05,
      "loss_nats": 1.0547631978988647,
      "step": 7730,
      "unclipped_grad_norm": 1.2570104598999023
    },
    {
      "bits_per_byte": 1.5186754281962946,
      "learning_rate": 3.077894168381942e-05,
      "loss_nats": 1.0526655912399292,
      "step": 7740,
      "unclipped_grad_norm": 1.2608413696289062
    },
    {
      "bits_per_byte": 1.530832365837834,
      "learning_rate": 3.0720227684712254e-05,
      "loss_nats": 1.0610921382904053,
      "step": 7750,
      "unclipped_grad_norm": 1.268287181854248
    },
    {
      "bits_per_byte": 1.5045843736631521,
      "learning_rate": 3.0663808156423456e-05,
      "loss_nats": 1.042898416519165,
      "step": 7760,
      "unclipped_grad_norm": 1.2538435459136963
    },
    {
      "bits_per_byte": 1.5028014295216574,
      "learning_rate": 3.060968406300711e-05,
      "loss_nats": 1.041662573814392,
      "step": 7770,
      "unclipped_grad_norm": 1.3365142345428467
    },
    {
      "bits_per_byte": 1.6460007199707645,
      "learning_rate": 3.055785632929463e-05,
      "loss_nats": 1.1409207582473755,
      "step": 7780,
      "unclipped_grad_norm": 1.3043659925460815
    },
    {
      "bits_per_byte": 1.5196737874845532,
      "learning_rate": 3.0508325840878984e-05,
      "loss_nats": 1.0533576011657715,
      "step": 7790,
      "unclipped_grad_norm": 1.2850605249404907
    },
    {
      "bits_per_byte": 1.559599903847656,
      "learning_rate": 3.046109344409957e-05,
      "loss_nats": 1.0810322761535645,
      "step": 7800,
      "unclipped_grad_norm": 1.3824384212493896
    },
    {
      "bits_per_byte": 1.4742115495901766,
      "learning_rate": 3.0416159946027727e-05,
      "loss_nats": 1.0218455791473389,
      "step": 7810,
      "unclipped_grad_norm": 1.2828388214111328
    },
    {
      "bits_per_byte": 1.5144011433744011,
      "learning_rate": 3.0373526114452946e-05,
      "loss_nats": 1.0497028827667236,
      "step": 7820,
      "unclipped_grad_norm": 1.3298958539962769
    },
    {
      "bits_per_byte": 1.549842984098907,
      "learning_rate": 3.0333192677869803e-05,
      "loss_nats": 1.0742692947387695,
      "step": 7830,
      "unclipped_grad_norm": 1.2448348999023438
    },
    {
      "bits_per_byte": 1.5177059619933808,
      "learning_rate": 3.029516032546544e-05,
      "loss_nats": 1.0519936084747314,
      "step": 7840,
      "unclipped_grad_norm": 1.234107255935669
    },
    {
      "bits_per_byte": 1.486972662284197,
      "learning_rate": 3.02594297071078e-05,
      "loss_nats": 1.0306909084320068,
      "step": 7850,
      "unclipped_grad_norm": 1.2269891500473022
    },
    {
      "bits_per_byte": 1.4944172752920954,
      "learning_rate": 3.0226001433334603e-05,
      "loss_nats": 1.0358511209487915,
      "step": 7860,
      "unclipped_grad_norm": 1.266727328300476
    },
    {
      "bits_per_byte": 1.549148518154723,
      "learning_rate": 3.019487607534278e-05,
      "loss_nats": 1.0737879276275635,
      "step": 7870,
      "unclipped_grad_norm": 1.3428150415420532
    },
    {
      "bits_per_byte": 1.522268833703422,
      "learning_rate": 3.016605416497886e-05,
      "loss_nats": 1.0551563501358032,
      "step": 7880,
      "unclipped_grad_norm": 1.305385947227478
    },
    {
      "bits_per_byte": 1.5751703531435204,
      "learning_rate": 3.0139536194729734e-05,
      "loss_nats": 1.0918248891830444,
      "step": 7890,
      "unclipped_grad_norm": 1.2760140895843506
    },
    {
      "bits_per_byte": 1.5609104116472305,
      "learning_rate": 3.0115322617714362e-05,
      "loss_nats": 1.0819406509399414,
      "step": 7900,
      "unclipped_grad_norm": 1.2964600324630737
    },
    {
      "bits_per_byte": 1.5527064952357728,
      "learning_rate": 3.0093413847675965e-05,
      "loss_nats": 1.07625412940979,
      "step": 7910,
      "unclipped_grad_norm": 1.3213040828704834
    },
    {
      "bits_per_byte": 1.4860267577044515,
      "learning_rate": 3.0073810258974985e-05,
      "loss_nats": 1.0300352573394775,
      "step": 7920,
      "unclipped_grad_norm": 1.2681679725646973
    },
    {
      "bits_per_byte": 1.482107273091288,
      "learning_rate": 3.0056512186582658e-05,
      "loss_nats": 1.0273184776306152,
      "step": 7930,
      "unclipped_grad_norm": 1.2925180196762085
    },
    {
      "bits_per_byte": 1.5584990428994832,
      "learning_rate": 3.0041519926075316e-05,
      "loss_nats": 1.08026921749115,
      "step": 7940,
      "unclipped_grad_norm": 1.4040197134017944
    },
    {
      "bits_per_byte": 1.488459280318255,
      "learning_rate": 3.0028833733629313e-05,
      "loss_nats": 1.0317213535308838,
      "step": 7950,
      "unclipped_grad_norm": 1.3141639232635498
    },
    {
      "bits_per_byte": 1.530331896323823,
      "learning_rate": 3.0018453826016686e-05,
      "loss_nats": 1.0607452392578125,
      "step": 7960,
      "unclipped_grad_norm": 1.263742446899414
    },
    {
      "bits_per_byte": 1.5034190192209056,
      "learning_rate": 3.001038038060143e-05,
      "loss_nats": 1.042090654373169,
      "step": 7970,
      "unclipped_grad_norm": 1.2354859113693237
    },
    {
      "bits_per_byte": 1.5362255697862408,
      "learning_rate": 3.0004613535336433e-05,
      "loss_nats": 1.0648304224014282,
      "step": 7980,
      "unclipped_grad_norm": 1.2929013967514038
    },
    {
      "bits_per_byte": 1.5535658925421343,
      "learning_rate": 3.000115338876117e-05,
      "loss_nats": 1.0768498182296753,
      "step": 7990,
      "unclipped_grad_norm": 1.2544541358947754
    },
    {
      "bits_per_byte": 1.4716885641020192,
      "learning_rate": 3e-05,
      "loss_nats": 1.020096778869629,
      "step": 8000,
      "unclipped_grad_norm": 1.2711973190307617
    }
  ]
}