wuyang/llm-atlas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
badf498598622b05c950b485ab100f265cf1df18
llm-atlas/experiments/k3/attnres_gradient/results/raw/formal-depth-32-block-seed-2026073001.json
T
wuyang 26fc824409 research: audit AttnRes gradient scale study
2026-07-30 09:49:53 +08:00

13073 lines
366 KiB
JSON
Raw Blame History

{
"architecture": "block",
"batch_size": 32,
"canonical_sha256_without_self": "c7554d9beea6fe9e63617aafd88403f7fc93b4b305a0c6d603a98a8572e5b5f1",
"depth": 32,
"diagnostics": [
{
"activation_grad_rms_by_block": [
0.0002677410375326872,
0.0002473295899108052,
0.00023034414334688336,
0.00021438254043459892,
0.00020044577831868082,
0.00018819731485564262,
0.00017973134526982903,
0.000171217747265473,
0.0001640921109355986,
0.00016061548376455903,
0.00015463090676348656,
0.00015024171443656087,
0.0001478576596127823,
0.00014403100067283958,
0.00013951722939964384,
0.00013661097909789532,
0.00013379570737015456,
0.0001311724045081064,
0.0001286817860091105,
0.00012687484559137374,
0.0001249351626029238,
0.00012369138130452484,
0.00012227990373503417,
0.00012101328320568427,
0.00011953213106608018,
0.0001181897969217971,
0.0001171672047348693,
0.00011609455395955592,
0.00011492414341773838,
0.0001140913664130494,
0.00011346775136189535,
0.00011285000800853595
],
"activation_grad_statistics": {
"first_quartile_mean": 0.00021242368711682502,
"first_to_last_ratio": 1.8345658968465297,
"imbalance_abs_log_ratio": 0.6068078850437921,
"last_quartile_mean": 0.0001157896194854402,
"mean": 0.0001511171253696375,
"normalized": [
1.7717451736709768,
1.636674793183293,
1.5242755762025908,
1.4186515254985965,
1.3264266232459345,
1.2453738409548603,
1.1893512719369177,
1.1330135273991528,
1.085860457802012,
1.0628542818802846,
1.023252039669589,
0.9942070699734702,
0.9784308644775869,
0.9531083940389613,
0.9232390376562555,
0.9040072643238835,
0.8853775311228679,
0.8680181295617843,
0.8515367513400655,
0.8395795332993111,
0.8267439067367663,
0.8185133286645813,
0.8091730400239778,
0.800791325997512,
0.7909899739933552,
0.7821072339266707,
0.7753403490721149,
0.7682422073314642,
0.7604971517068638,
0.7549863467424895,
0.7508596466770359,
0.7467718018887739
],
"population_cv": 0.2714186609524326
},
"activation_output_rms_by_block": [
0.0034885562490671873,
0.004831995815038681,
0.005952076520770788,
0.006925336085259914,
0.00385080324485898,
0.005542940925806761,
0.006729288958013058,
0.007927131839096546,
0.004081308841705322,
0.005811207927763462,
0.007534808944910765,
0.008581922389566898,
0.00499920267611742,
0.007102519739419222,
0.008542952127754688,
0.010447698645293713,
0.005041283555328846,
0.007636935915797949,
0.009387043304741383,
0.011020516976714134,
0.005807018838822842,
0.008367395959794521,
0.01024632342159748,
0.012378672137856483,
0.006453148555010557,
0.009127611294388771,
0.011037957854568958,
0.012611051090061665,
0.007010402157902718,
0.009794782847166061,
0.012445803731679916,
0.014087481424212456
],
"activation_output_statistics": {
"first_quartile_mean": 0.0056560162047389895,
"first_to_last_ratio": 0.5480088979804595,
"imbalance_abs_log_ratio": 0.6014637549753238,
"last_quartile_mean": 0.010321029869373888,
"mean": 0.007962599374877755,
"normalized": [
0.4381177659237371,
0.606836484864952,
0.747504205668035,
0.8697330807712821,
0.4836113263475722,
0.6961220406611074,
0.845112084785299,
0.9955457339856744,
0.512559862622497,
0.7298129234151854,
0.9462750278110585,
1.077778999737589,
0.6278355146046978,
0.8919850673170736,
1.072884836415099,
1.3120964842531853,
0.6331203314377778,
0.9591008609440675,
1.1788918244911069,
1.384035094304029,
0.7292868277593074,
1.0508372411895934,
1.2868063479276561,
1.554601902603762,
0.8104324041933391,
1.1463105029730196,
1.3862254440922965,
1.5837857082010076,
0.8804162846646224,
1.2300986632667847,
1.5630327667805075,
1.7692063559870779
],
"population_cv": 0.3451071044649734
},
"bits_per_byte": 7.959341947457508,
"branch_output_rms_by_sublayer": [
0.0022265110164880753,
0.0027073738165199757,
0.0021613112185150385,
0.002722575329244137,
0.0025601957459002733,
0.0026850115973502398,
0.0025323182344436646,
0.0026861224323511124,
0.0027253651060163975,
0.00271838391199708,
0.002904455177485943,
0.0026645169127732515,
0.0029208383057266474,
0.0026859452482312918,
0.00299624796025455,
0.0027359756641089916,
0.0030937574338167906,
0.0026767742820084095,
0.0033804215490818024,
0.0027451682835817337,
0.0034370317589491606,
0.0026999011170119047,
0.0038969130255281925,
0.002663193503394723,
0.004255065228790045,
0.002626648638397455,
0.004119740333408117,
0.0026531771291047335,
0.003970506135374308,
0.0026517712976783514,
0.004385598469525576,
0.002676903735846281,
0.0041999719105660915,
0.002670086920261383,
0.005232699681073427,
0.0026355870068073273,
0.004664790350943804,
0.0026610493659973145,
0.004898969549685717,
0.0026388936676084995,
0.005131533369421959,
0.0026469926815479994,
0.005069608334451914,
0.00264862016774714,
0.0051318020559847355,
0.002641494618728757,
0.0058372304774820805,
0.002674198243767023,
0.0059690922498703,
0.002557571977376938,
0.00554921617731452,
0.00252733426168561,
0.0058550527319312096,
0.0025468147359788418,
0.006255472078919411,
0.0026856842450797558,
0.006502739619463682,
0.002551089273765683,
0.006122888531535864,
0.0025778315030038357,
0.007071305997669697,
0.0026001501828432083,
0.006241494789719582,
0.0025662274565547705
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.0056993408498569815,
0.005586029321860134,
0.005830344684528519,
0.00540288712216694,
0.00538025811909819,
0.0052226633069546126,
0.005297605878918106,
0.0052023753769565,
0.0048107019260181325,
0.005248672753719287,
0.004962353375385338,
0.005501319898161136,
0.005588445553926757,
0.005229606380765812,
0.004884302670286647,
0.005031817046498285,
0.005000470961685071,
0.0053937604587735305,
0.004980398173225761,
0.004658922297714218,
0.00496598185096497,
0.004578385934052906,
0.004596161734371138,
0.004990442293511754,
0.005201771295872185,
0.00459813521112035,
0.004706502430387731,
0.004781878207244352,
0.0048378722082161795,
0.004473110720597102,
0.005084432926394394,
0.004685705425265166
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.005452688082542498,
"first_to_last_ratio": 1.1368823875795573,
"imbalance_abs_log_ratio": 0.12828976841498158,
"last_quartile_mean": 0.004796176053137182,
"mean": 0.005075395512328069,
"normalized": [
1.122935313319594,
1.1006096585560166,
1.1487468652180286,
1.064525337787646,
1.060066768398564,
1.0290160233362764,
1.0437818818356701,
1.025018713185997,
0.947847692723257,
1.0341406380981206,
0.9777274230809889,
1.0839194472230789,
1.1010857262951226,
1.0303840100861434,
0.9623491722808084,
0.9914137793352396,
0.9852376922229987,
1.0627271206100406,
0.9812827711906272,
0.9179427074003902,
0.9784423379227619,
0.9020747098294245,
0.9055770576316904,
0.9832617539638114,
1.024899691706223,
0.905965889742295,
0.9273173724009683,
0.9421685848185097,
0.9532010257062827,
0.8813324419214177,
1.00178063247374,
0.9232197596982638
],
"population_cv": 0.06921897980134169
},
"depth_weights": [
{
"entropy_mean": 0.0,
"mean_weights": [
1.0
],
"sources": 1
},
{
"entropy_mean": 0.6931471824645996,
"mean_weights": [
0.5,
0.5
],
"sources": 2
},
{
"entropy_mean": 0.6931471824645996,
"mean_weights": [
0.5,
0.5
],
"sources": 2
},
{
"entropy_mean": 0.6931471824645996,
"mean_weights": [
0.5,
0.5
],
"sources": 2
},
{
"entropy_mean": 0.6931471824645996,
"mean_weights": [
0.5,
0.5
],
"sources": 2
},
{
"entropy_mean": 0.6931471824645996,
"mean_weights": [
0.5,
0.5
],
"sources": 2
},
{
"entropy_mean": 0.6931471824645996,
"mean_weights": [
0.5,
0.5
],
"sources": 2
},
{
"entropy_mean": 0.6931471824645996,
"mean_weights": [
0.5,
0.5
],
"sources": 2
},
{
"entropy_mean": 0.6931471824645996,
"mean_weights": [
0.5,
0.5
],
"sources": 2
},
{
"entropy_mean": 1.0986123085021973,
"mean_weights": [
0.3333333432674408,
0.3333333432674408,
0.3333333432674408
],
"sources": 3
},
{
"entropy_mean": 1.0986123085021973,
"mean_weights": [
0.3333333432674408,
0.3333333432674408,
0.3333333432674408
],
"sources": 3
},
{
"entropy_mean": 1.0986123085021973,
"mean_weights": [
0.3333333432674408,
0.3333333432674408,
0.3333333432674408
],
"sources": 3
},
{
"entropy_mean": 1.0986123085021973,
"mean_weights": [
0.3333333432674408,
0.3333333432674408,
0.3333333432674408
],
"sources": 3
},
{
"entropy_mean": 1.0986123085021973,
"mean_weights": [
0.3333333432674408,
0.3333333432674408,
0.3333333432674408
],
"sources": 3
},
{
"entropy_mean": 1.0986123085021973,
"mean_weights": [
0.3333333432674408,
0.3333333432674408,
0.3333333432674408
],
"sources": 3
},
{
"entropy_mean": 1.0986123085021973,
"mean_weights": [
0.3333333432674408,
0.3333333432674408,
0.3333333432674408
],
"sources": 3
},
{
"entropy_mean": 1.0986123085021973,
"mean_weights": [
0.3333333432674408,
0.3333333432674408,
0.3333333432674408
],
"sources": 3
},
{
"entropy_mean": 1.3862943649291992,
"mean_weights": [
0.25,
0.25,
0.25,
0.25
],
"sources": 4
},
{
"entropy_mean": 1.3862943649291992,
"mean_weights": [
0.25,
0.25,
0.25,
0.25
],
"sources": 4
},
{
"entropy_mean": 1.3862943649291992,
"mean_weights": [
0.25,
0.25,
0.25,
0.25
],
"sources": 4
},
{
"entropy_mean": 1.3862943649291992,
"mean_weights": [
0.25,
0.25,
0.25,
0.25
],
"sources": 4
},
{
"entropy_mean": 1.3862943649291992,
"mean_weights": [
0.25,
0.25,
0.25,
0.25
],
"sources": 4
},
{
"entropy_mean": 1.3862943649291992,
"mean_weights": [
0.25,
0.25,
0.25,
0.25
],
"sources": 4
},
{
"entropy_mean": 1.3862943649291992,
"mean_weights": [
0.25,
0.25,
0.25,
0.25
],
"sources": 4
},
{
"entropy_mean": 1.3862943649291992,
"mean_weights": [
0.25,
0.25,
0.25,
0.25
],
"sources": 4
},
{
"entropy_mean": 1.6094379425048828,
"mean_weights": [
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343
],
"sources": 5
},
{
"entropy_mean": 1.6094379425048828,
"mean_weights": [
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343
],
"sources": 5
},
{
"entropy_mean": 1.6094379425048828,
"mean_weights": [
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343
],
"sources": 5
},
{
"entropy_mean": 1.6094379425048828,
"mean_weights": [
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343
],
"sources": 5
},
{
"entropy_mean": 1.6094379425048828,
"mean_weights": [
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343
],
"sources": 5
},
{
"entropy_mean": 1.6094379425048828,
"mean_weights": [
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343
],
"sources": 5
},
{
"entropy_mean": 1.6094379425048828,
"mean_weights": [
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343
],
"sources": 5
},
{
"entropy_mean": 1.6094379425048828,
"mean_weights": [
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343
],
"sources": 5
},
{
"entropy_mean": 1.791759729385376,
"mean_weights": [
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592
],
"sources": 6
},
{
"entropy_mean": 1.791759729385376,
"mean_weights": [
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592
],
"sources": 6
},
{
"entropy_mean": 1.791759729385376,
"mean_weights": [
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592
],
"sources": 6
},
{
"entropy_mean": 1.791759729385376,
"mean_weights": [
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592
],
"sources": 6
},
{
"entropy_mean": 1.791759729385376,
"mean_weights": [
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592
],
"sources": 6
},
{
"entropy_mean": 1.791759729385376,
"mean_weights": [
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592
],
"sources": 6
},
{
"entropy_mean": 1.791759729385376,
"mean_weights": [
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592
],
"sources": 6
},
{
"entropy_mean": 1.791759729385376,
"mean_weights": [
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592
],
"sources": 6
},
{
"entropy_mean": 1.9459102153778076,
"mean_weights": [
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548
],
"sources": 7
},
{
"entropy_mean": 1.9459102153778076,
"mean_weights": [
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548
],
"sources": 7
},
{
"entropy_mean": 1.9459102153778076,
"mean_weights": [
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548
],
"sources": 7
},
{
"entropy_mean": 1.9459102153778076,
"mean_weights": [
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548
],
"sources": 7
},
{
"entropy_mean": 1.9459102153778076,
"mean_weights": [
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548
],
"sources": 7
},
{
"entropy_mean": 1.9459102153778076,
"mean_weights": [
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548
],
"sources": 7
},
{
"entropy_mean": 1.9459102153778076,
"mean_weights": [
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548
],
"sources": 7
},
{
"entropy_mean": 1.9459102153778076,
"mean_weights": [
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548
],
"sources": 7
},
{
"entropy_mean": 2.079441547393799,
"mean_weights": [
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125
],
"sources": 8
},
{
"entropy_mean": 2.079441547393799,
"mean_weights": [
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125
],
"sources": 8
},
{
"entropy_mean": 2.079441547393799,
"mean_weights": [
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125
],
"sources": 8
},
{
"entropy_mean": 2.079441547393799,
"mean_weights": [
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125
],
"sources": 8
},
{
"entropy_mean": 2.079441547393799,
"mean_weights": [
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125
],
"sources": 8
},
{
"entropy_mean": 2.079441547393799,
"mean_weights": [
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125
],
"sources": 8
},
{
"entropy_mean": 2.079441547393799,
"mean_weights": [
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125
],
"sources": 8
},
{
"entropy_mean": 2.079441547393799,
"mean_weights": [
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125
],
"sources": 8
},
{
"entropy_mean": 2.1972246170043945,
"mean_weights": [
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519
],
"sources": 9
},
{
"entropy_mean": 2.1972246170043945,
"mean_weights": [
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519
],
"sources": 9
},
{
"entropy_mean": 2.1972246170043945,
"mean_weights": [
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519
],
"sources": 9
},
{
"entropy_mean": 2.1972246170043945,
"mean_weights": [
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519
],
"sources": 9
},
{
"entropy_mean": 2.1972246170043945,
"mean_weights": [
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519
],
"sources": 9
},
{
"entropy_mean": 2.1972246170043945,
"mean_weights": [
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519
],
"sources": 9
},
{
"entropy_mean": 2.1972246170043945,
"mean_weights": [
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519
],
"sources": 9
}
],
"layer_input_rms_by_sublayer": [
0.02802751213312149,
0.014061377383768559,
0.01413465291261673,
0.014165290631353855,
0.014225807972252369,
0.014281988143920898,
0.01433455292135477,
0.014350161887705326,
0.01440478302538395,
0.00966146495193243,
0.009703479707241058,
0.009741888381540775,
0.00977881159633398,
0.009820535778999329,
0.009858808480203152,
0.00991273857653141,
0.009950309991836548,
0.0074927168898284435,
0.00752287870272994,
0.007546746172010899,
0.007571155205368996,
0.00766373984515667,
0.007693085819482803,
0.007762390188872814,
0.007788818795233965,
0.006325984839349985,
0.006353048607707024,
0.006375366821885109,
0.006389216054230928,
0.006445344537496567,
0.0064707002602517605,
0.006555322092026472,
0.00656962301582098,
0.005531130358576775,
0.005553483963012695,
0.005611324217170477,
0.0056249531917274,
0.0056777093559503555,
0.005688921548426151,
0.00574630219489336,
0.005768724251538515,
0.004946626722812653,
0.004966245498508215,
0.0050392793491482735,
0.005057469941675663,
0.005084683652967215,
0.005098348017781973,
0.005203457549214363,
0.005216081161051989,
0.004667261149734259,
0.0046758754178881645,
0.004717131145298481,
0.004727107938379049,
0.004768412560224533,
0.004777040798217058,
0.004826710093766451,
0.004837613552808762,
0.0044052014127373695,
0.0044186655431985855,
0.004504534415900707,
0.004510571248829365,
0.004574235528707504,
0.0045849340967834,
0.004644096828997135
],
"loss_nats": 5.516995429992676,
"loss_scale": 1.0,
"output_weights": {
"entropy_mean": 2.1972246170043945,
"mean_weights": [
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519
],
"sources": 9
},
"step": 0,
"stream_state_rms_by_sublayer": [
0.0022265110164880753,
0.0034885562490671873,
0.0040087271481752396,
0.004831995815038681,
0.005379301495850086,
0.005952076520770788,
0.006410556845366955,
0.006925336085259914,
0.0027253651060163975,
0.00385080324485898,
0.004838948138058186,
0.005542940925806761,
0.006175771821290255,
0.006729288958013058,
0.00742313964292407,
0.007927131839096546,
0.0030937574338167906,
0.004081308841705322,
0.005150026176124811,
0.005811207927763462,
0.007019025273621082,
0.007534808944910765,
0.008143599145114422,
0.008581922389566898,
0.004255065228790045,
0.00499920267611742,
0.0066224560141563416,
0.007102519739419222,
0.008129785768687725,
0.008542952127754688,
0.010099578648805618,
0.010447698645293713,
0.0041999719105660915,
0.005041283555328846,
0.00719039561226964,
0.007636935915797949,
0.009030314162373543,
0.009387043304741383,
0.010736005380749702,
0.011020516976714134,
0.005131533369421959,
0.005807018838822842,
0.00787449348717928,
0.008367395959794521,
0.009912566281855106,
0.01024632342159748,
0.012113703414797783,
0.012378672137856483,
0.0059690922498703,
0.006453148555010557,
0.00876442901790142,
0.009127611294388771,
0.010806876234710217,
0.011037957854568958,
0.012353328987956047,
0.012611051090061665,
0.006502739619463682,
0.007010402157902718,
0.009470963850617409,
0.009794782847166061,
0.012123330496251583,
0.012445803731679916,
0.01384514570236206,
0.014087481424212456
]
},
{
"activation_grad_rms_by_block": [
7.003638165770099e-05,
6.680699152639136e-05,
6.333583587547764e-05,
6.047846545698121e-05,
5.581479490501806e-05,
5.2793002396356314e-05,
4.981113306712359e-05,
4.656345481635071e-05,
4.2050978663610294e-05,
3.9379403460770845e-05,
3.663913594209589e-05,
3.422236841288395e-05,
3.138875399599783e-05,
2.928113826783374e-05,
2.7496294933371246e-05,
2.5903584173647687e-05,
2.449963358230889e-05,
2.3339889594353735e-05,
2.2383910618373193e-05,
2.1501236915355548e-05,
2.0739182218676433e-05,
2.0070132450200617e-05,
1.950136174855288e-05,
1.9030989278689958e-05,
1.87676414498128e-05,
1.8413003999739885e-05,
1.8116044884664007e-05,
1.7846368791651912e-05,
1.7435917470720597e-05,
1.7234746337635443e-05,
1.7042602848960087e-05,
1.689592500042636e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 5.8205007462674985e-05,
"first_to_last_ratio": 3.284886533563498,
"imbalance_abs_log_ratio": 1.189332110567881,
"last_quartile_mean": 1.7719031347951386e-05,
"mean": 3.265063452317918e-05,
"normalized": [
2.145023601546889,
2.046116178200582,
1.9398041355219169,
1.8522906626530224,
1.7094551367874427,
1.616905863157966,
1.5255793277696308,
1.4261117891382666,
1.2879069358899493,
1.2060838643982497,
1.1221569343801026,
1.0481379278736214,
0.9613520366262555,
0.8968015077026028,
0.8421366180143057,
0.7933562257498653,
0.7503570432885839,
0.714837243906681,
0.6855582118161445,
0.6585243205638621,
0.6351846609275962,
0.6146934889106834,
0.5972735915655352,
0.5828673640378894,
0.5748017373594797,
0.5639401582431183,
0.5548451094205582,
0.5465856652489405,
0.5340146592968225,
0.5278533354505021,
0.5219685037625007,
0.5174761607904339
],
"population_cv": 0.5072011380661324
},
"activation_output_rms_by_block": [
0.005866481456905603,
0.010142817161977291,
0.01482025533914566,
0.01956234686076641,
0.0067119961604475975,
0.011908270418643951,
0.017094247043132782,
0.02295343205332756,
0.007494525518268347,
0.013766046613454819,
0.02038806490600109,
0.02682213857769966,
0.009193498641252518,
0.016781754791736603,
0.024297412484884262,
0.03140362724661827,
0.009884056635200977,
0.018740201368927956,
0.02799132652580738,
0.03797563910484314,
0.012623735703527927,
0.02351362258195877,
0.034899674355983734,
0.047332651913166046,
0.01744839735329151,
0.029514454305171967,
0.044660378247499466,
0.0601838044822216,
0.018972838297486305,
0.0376126803457737,
0.0565216988325119,
0.07583927363157272
],
"activation_output_statistics": {
"first_quartile_mean": 0.013632480811793357,
"first_to_last_ratio": 0.32005493218522185,
"imbalance_abs_log_ratio": 1.1392626348419668,
"last_quartile_mean": 0.04259419068694115,
"mean": 0.025403792154975235,
"normalized": [
0.23092935972382672,
0.3992639012357397,
0.5833875213879504,
0.7700561688360058,
0.2642123711098415,
0.4687595594389148,
0.6729013896370168,
0.9035435305603466,
0.2950160146386087,
0.5418894364067922,
0.8025599005701267,
1.0558320747576517,
0.36189473544610173,
0.6606003816028687,
0.9564482474371727,
1.2361787191078082,
0.389077999650742,
0.7376930678145923,
1.1018562250488806,
1.4948807199009364,
0.4969232792693754,
0.9255949835565679,
1.3737978229029388,
1.8632120600111322,
0.6868422339014575,
1.1618129342705898,
1.7580201402629136,
2.3690874226600394,
0.7468506348084945,
1.4805931380763324,
2.2249315569778956,
2.985352468990339
],
"population_cv": 0.6518934572295025
},
"bits_per_byte": 5.150682269327325,
"branch_output_rms_by_sublayer": [
0.002919268561527133,
0.005079356953501701,
0.0026904908008873463,
0.0052208867855370045,
0.0028405722696334124,
0.005689430516213179,
0.0029144894797354937,
0.005868426524102688,
0.0030634324066340923,
0.00590060418471694,
0.0032732728868722916,
0.0059102908708155155,
0.003848863998427987,
0.0058776624500751495,
0.003392934100702405,
0.006663382984697819,
0.0037359180860221386,
0.006277414970099926,
0.004302032291889191,
0.006413500756025314,
0.004344160668551922,
0.006856496445834637,
0.004835603758692741,
0.006499431096017361,
0.005531937349587679,
0.006718362681567669,
0.005483300890773535,
0.0070298356004059315,
0.005397035740315914,
0.006969383917748928,
0.005739198997616768,
0.0068367342464625835,
0.005659298039972782,
0.007033169735223055,
0.007230010814964771,
0.007157569285482168,
0.006938732694834471,
0.0075136395171284676,
0.007343437522649765,
0.007708695251494646,
0.007882392033934593,
0.007887008599936962,
0.00830139685422182,
0.008205465972423553,
0.008992243558168411,
0.00814912747591734,
0.009393857792019844,
0.008287571370601654,
0.012831038795411587,
0.008026733994483948,
0.00905293133109808,
0.008120592683553696,
0.012243949808180332,
0.008479002863168716,
0.012258118949830532,
0.00907627958804369,
0.013431218452751637,
0.00841403380036354,
0.01483316533267498,
0.008773921057581902,
0.015388047322630882,
0.008821850642561913,
0.014634119346737862,
0.009164483286440372
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.00039664652254938686,
0.00033413090452743513,
0.0003161152806455274,
0.0003097753114302562,
0.00028318594507881776,
0.00028174654548060826,
0.00029366070958977545,
0.0002617199768643518,
0.00023850476284541366,
0.0002578321956415913,
0.00023580491915923377,
0.00024129375767696618,
0.00024677521358199467,
0.00023282043614108414,
0.00020802360655182162,
0.00020816264493602737,
0.00019546204845585216,
0.00020244473639840523,
0.00019508742079131332,
0.00018509966437476374,
0.00018490270602469487,
0.0001750081453781441,
0.00017332318168420213,
0.00016951527760705314,
0.0001849276373753642,
0.00014904354377519617,
0.00017214209100244258,
0.0001582098123018746,
0.0001606899226932654,
0.0001683991686835421,
0.00016911992547040899,
0.00016138660409180947
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.00030962264952076986,
"first_to_last_ratio": 1.8709465967014842,
"imbalance_abs_log_ratio": 0.6264445042344079,
"last_quartile_mean": 0.00016548983817423794,
"mean": 0.0002234675193377695,
"normalized": [
1.774962749507496,
1.4952101563466982,
1.4145916220053505,
1.3862207463001954,
1.2672353723620313,
1.2607941696204648,
1.314109139708619,
1.1711768119140573,
1.0672905107292818,
1.1537792892929548,
1.0552089174213117,
1.0797710485712801,
1.104300143095945,
1.0418535849461763,
0.9308896754583627,
0.9315118615577912,
0.8746776669606858,
0.9059246596477921,
0.8730012369110346,
0.828306793413619,
0.8274254199117672,
0.7831480203331949,
0.775607937107967,
0.7585678585836542,
0.827536985792765,
0.6669584206996899,
0.7703226469447271,
0.7079767689314242,
0.719075072607683,
0.7535733568018358,
0.7567986881117407,
0.7221926684024036
],
"population_cv": 0.268339881652204
},
"depth_weights": [
{
"entropy_mean": 0.0,
"mean_weights": [
1.0
],
"sources": 1
},
{
"entropy_mean": 0.6930912733078003,
"mean_weights": [
0.5023526549339294,
0.49764734506607056
],
"sources": 2
},
{
"entropy_mean": 0.692903995513916,
"mean_weights": [
0.5103387832641602,
0.48966121673583984
],
"sources": 2
},
{
"entropy_mean": 0.6928629875183105,
"mean_weights": [
0.5053091049194336,
0.4946908950805664
],
"sources": 2
},
{
"entropy_mean": 0.6927823424339294,
"mean_weights": [
0.5126834511756897,
0.4873165488243103
],
"sources": 2
},
{
"entropy_mean": 0.6928155422210693,
"mean_weights": [
0.502953290939331,
0.49704670906066895
],
"sources": 2
},
{
"entropy_mean": 0.692751407623291,
"mean_weights": [
0.5129156708717346,
0.48708435893058777
],
"sources": 2
},
{
"entropy_mean": 0.6925709247589111,
"mean_weights": [
0.5112693905830383,
0.4887305796146393
],
"sources": 2
},
{
"entropy_mean": 0.6929247379302979,
"mean_weights": [
0.5095131397247314,
0.49048691987991333
],
"sources": 2
},
{
"entropy_mean": 1.0983518362045288,
"mean_weights": [
0.3398353159427643,
0.32833361625671387,
0.33183106780052185
],
"sources": 3
},
{
"entropy_mean": 1.098447322845459,
"mean_weights": [
0.3408135175704956,
0.3303682208061218,
0.3288182020187378
],
"sources": 3
},
{
"entropy_mean": 1.0980432033538818,
"mean_weights": [
0.3472674489021301,
0.3313220739364624,
0.32141047716140747
],
"sources": 3
},
{
"entropy_mean": 1.0985240936279297,
"mean_weights": [
0.3373557925224304,
0.33420175313949585,
0.32844245433807373
],
"sources": 3
},
{
"entropy_mean": 1.098268985748291,
"mean_weights": [
0.3445510268211365,
0.33061009645462036,
0.32483887672424316
],
"sources": 3
},
{
"entropy_mean": 1.0985302925109863,
"mean_weights": [
0.3371829390525818,
0.3332410156726837,
0.3295760750770569
],
"sources": 3
},
{
"entropy_mean": 1.0983161926269531,
"mean_weights": [
0.3423102796077728,
0.3327076733112335,
0.32498204708099365
],
"sources": 3
},
{
"entropy_mean": 1.098489761352539,
"mean_weights": [
0.33793729543685913,
0.3344462811946869,
0.327616423368454
],
"sources": 3
},
{
"entropy_mean": 1.3861088752746582,
"mean_weights": [
0.2561652958393097,
0.2513129711151123,
0.2458309531211853,
0.2466908097267151
],
"sources": 4
},
{
"entropy_mean": 1.3861844539642334,
"mean_weights": [
0.2536832094192505,
0.25214672088623047,
0.24721932411193848,
0.24695077538490295
],
"sources": 4
},
{
"entropy_mean": 1.3860987424850464,
"mean_weights": [
0.25705933570861816,
0.2498370260000229,
0.2462591528892517,
0.24684447050094604
],
"sources": 4
},
{
"entropy_mean": 1.3862221240997314,
"mean_weights": [
0.2514326870441437,
0.2528519928455353,
0.24950753152370453,
0.24620777368545532
],
"sources": 4
},
{
"entropy_mean": 1.386016845703125,
"mean_weights": [
0.25482749938964844,
0.2536851465702057,
0.24858617782592773,
0.24290117621421814
],
"sources": 4
},
{
"entropy_mean": 1.3862277269363403,
"mean_weights": [
0.25171467661857605,
0.2530350983142853,
0.24868467450141907,
0.24656552076339722
],
"sources": 4
},
{
"entropy_mean": 1.3859888315200806,
"mean_weights": [
0.25458407402038574,
0.2546530067920685,
0.248786062002182,
0.24197682738304138
],
"sources": 4
},
{
"entropy_mean": 1.3862541913986206,
"mean_weights": [
0.2501963675022125,
0.2521768808364868,
0.24995486438274384,
0.24767187237739563
],
"sources": 4
},
{
"entropy_mean": 1.6090729236602783,
"mean_weights": [
0.20360878109931946,
0.20631682872772217,
0.19986985623836517,
0.19557830691337585,
0.19462622702121735
],
"sources": 5
},
{
"entropy_mean": 1.6093083620071411,
"mean_weights": [
0.2017374038696289,
0.20362424850463867,
0.20011676847934723,
0.1968517303466797,
0.1976698487997055
],
"sources": 5
},
{
"entropy_mean": 1.608644723892212,
"mean_weights": [
0.20431089401245117,
0.20831988751888275,
0.20422273874282837,
0.19545811414718628,
0.18768835067749023
],
"sources": 5
},
{
"entropy_mean": 1.609345555305481,
"mean_weights": [
0.2002599537372589,
0.20386311411857605,
0.2001732438802719,
0.19764204323291779,
0.19806164503097534
],
"sources": 5
},
{
"entropy_mean": 1.6086196899414062,
"mean_weights": [
0.20515945553779602,
0.21025606989860535,
0.20179016888141632,
0.1938556730747223,
0.18893863260746002
],
"sources": 5
},
{
"entropy_mean": 1.609370470046997,
"mean_weights": [
0.2002836912870407,
0.20340535044670105,
0.20010054111480713,
0.1980183869600296,
0.1981920301914215
],
"sources": 5
},
{
"entropy_mean": 1.6083989143371582,
"mean_weights": [
0.20363344252109528,
0.21095217764377594,
0.20392383635044098,
0.19413790106773376,
0.18735264241695404
],
"sources": 5
},
{
"entropy_mean": 1.6093392372131348,
"mean_weights": [
0.20084109902381897,
0.20331618189811707,
0.2011999487876892,
0.19900763034820557,
0.19563516974449158
],
"sources": 5
},
{
"entropy_mean": 1.7908520698547363,
"mean_weights": [
0.17160072922706604,
0.17730243504047394,
0.16903036832809448,
0.16099891066551208,
0.15854957699775696,
0.1625179797410965
],
"sources": 6
},
{
"entropy_mean": 1.7916775941848755,
"mean_weights": [
0.16706335544586182,
0.16981256008148193,
0.16858771443367004,
0.1652718484401703,
0.16462719440460205,
0.16463729739189148
],
"sources": 6
},
{
"entropy_mean": 1.7898297309875488,
"mean_weights": [
0.17284834384918213,
0.18101516366004944,
0.1735592484474182,
0.16231873631477356,
0.15689018368721008,
0.153368279337883
],
"sources": 6
},
{
"entropy_mean": 1.7917213439941406,
"mean_weights": [
0.16615921258926392,
0.16885727643966675,
0.16764026880264282,
0.16661542654037476,
0.16609220206737518,
0.16463561356067657
],
"sources": 6
},
{
"entropy_mean": 1.7890949249267578,
"mean_weights": [
0.17241932451725006,
0.18413974344730377,
0.17484498023986816,
0.1621202528476715,
0.15553176403045654,
0.15094394981861115
],
"sources": 6
},
{
"entropy_mean": 1.7917208671569824,
"mean_weights": [
0.16663745045661926,
0.168935626745224,
0.16597260534763336,
0.16514870524406433,
0.16661295294761658,
0.16669265925884247
],
"sources": 6
},
{
"entropy_mean": 1.789101004600525,
"mean_weights": [
0.17229264974594116,
0.18365857005119324,
0.1747075319290161,
0.1640234887599945,
0.155386283993721,
0.14993149042129517
],
"sources": 6
},
{
"entropy_mean": 1.7917447090148926,
"mean_weights": [
0.1658121943473816,
0.167683407664299,
0.16629494726657867,
0.1669958382844925,
0.16702724993228912,
0.1661863625049591
],
"sources": 6
},
{
"entropy_mean": 1.9436898231506348,
"mean_weights": [
0.14778709411621094,
0.15856821835041046,
0.1500256359577179,
0.1408001184463501,
0.13533659279346466,
0.130964457988739,
0.13651786744594574
],
"sources": 7
},
{
"entropy_mean": 1.9458942413330078,
"mean_weights": [
0.142832413315773,
0.1429906189441681,
0.14157448709011078,
0.14285019040107727,
0.14392998814582825,
0.14335544407367706,
0.14246684312820435
],
"sources": 7
},
{
"entropy_mean": 1.9425597190856934,
"mean_weights": [
0.1483290046453476,
0.1615946888923645,
0.15280696749687195,
0.14239417016506195,
0.13462956249713898,
0.1307675987482071,
0.12947802245616913
],
"sources": 7
},
{
"entropy_mean": 1.9458987712860107,
"mean_weights": [
0.1426449418067932,
0.14291533827781677,
0.1416938602924347,
0.14274153113365173,
0.14325542747974396,
0.1432831734418869,
0.14346569776535034
],
"sources": 7
},
{
"entropy_mean": 1.941068410873413,
"mean_weights": [
0.14912807941436768,
0.16456490755081177,
0.15460123121738434,
0.14255177974700928,
0.13539987802505493,
0.1283380389213562,
0.1254161298274994
],
"sources": 7
},
{
"entropy_mean": 1.9458861351013184,
"mean_weights": [
0.14264336228370667,
0.1430070698261261,
0.14314037561416626,
0.14221979677677155,
0.14132913947105408,
0.1431397795677185,
0.14452049136161804
],
"sources": 7
},
{
"entropy_mean": 1.9415628910064697,
"mean_weights": [
0.14895881712436676,
0.16342055797576904,
0.15406624972820282,
0.14293311536312103,
0.1363803744316101,
0.12938986718654633,
0.1248510330915451
],
"sources": 7
},
{
"entropy_mean": 1.9458723068237305,
"mean_weights": [
0.14191991090774536,
0.1418565809726715,
0.14164230227470398,
0.14289972186088562,
0.14397987723350525,
0.14332248270511627,
0.14437910914421082
],
"sources": 7
},
{
"entropy_mean": 2.073223114013672,
"mean_weights": [
0.13280923664569855,
0.14827430248260498,
0.13796664774417877,
0.12756125628948212,
0.12019773572683334,
0.11310221254825592,
0.10922321677207947,
0.11086538434028625
],
"sources": 8
},
{
"entropy_mean": 2.079413890838623,
"mean_weights": [
0.12475188821554184,
0.12460128962993622,
0.12424701452255249,
0.12420951575040817,
0.12484367191791534,
0.12469655275344849,
0.12630701065063477,
0.12634305655956268
],
"sources": 8
},
{
"entropy_mean": 2.072925567626953,
"mean_weights": [
0.13228744268417358,
0.14843900501728058,
0.13848665356636047,
0.12843599915504456,
0.12169650197029114,
0.11442162096500397,
0.11023421585559845,
0.10599856078624725
],
"sources": 8
},
{
"entropy_mean": 2.0793263912200928,
"mean_weights": [
0.12420415878295898,
0.12273682653903961,
0.12360996007919312,
0.1238352432847023,
0.1241140365600586,
0.12607696652412415,
0.1275939792394638,
0.12782883644104004
],
"sources": 8
},
{
"entropy_mean": 2.073427200317383,
"mean_weights": [
0.13135744631290436,
0.1465209424495697,
0.13772746920585632,
0.12966200709342957,
0.12215200066566467,
0.11654195189476013,
0.11090746521949768,
0.10513069480657578
],
"sources": 8
},
{
"entropy_mean": 2.079422950744629,
"mean_weights": [
0.12477636337280273,
0.12493626773357391,
0.1242394968867302,
0.12470392882823944,
0.12560302019119263,
0.12461308389902115,
0.12591606378555298,
0.12521176040172577
],
"sources": 8
},
{
"entropy_mean": 2.0725789070129395,
"mean_weights": [
0.13186156749725342,
0.14827941358089447,
0.13866659998893738,
0.12988534569740295,
0.12183371186256409,
0.11524046957492828,
0.1099783331155777,
0.10425455868244171
],
"sources": 8
},
{
"entropy_mean": 2.0793380737304688,
"mean_weights": [
0.12417487055063248,
0.1225123256444931,
0.12277349829673767,
0.1243998259305954,
0.12567630410194397,
0.1261572688817978,
0.12747105956077576,
0.12683483958244324
],
"sources": 8
},
{
"entropy_mean": 2.1906280517578125,
"mean_weights": [
0.11854428052902222,
0.13291707634925842,
0.12378416955471039,
0.11745279282331467,
0.11140251904726028,
0.10409101843833923,
0.09838292002677917,
0.09486069530248642,
0.09856453537940979
],
"sources": 9
},
{
"entropy_mean": 2.197185516357422,
"mean_weights": [
0.11084984987974167,
0.11021925508975983,
0.11018812656402588,
0.11046157777309418,
0.11131840199232101,
0.11167057603597641,
0.11249847710132599,
0.1118280440568924,
0.11096569895744324
],
"sources": 9
},
{
"entropy_mean": 2.1899101734161377,
"mean_weights": [
0.1190595030784607,
0.13352498412132263,
0.1239522397518158,
0.11779220402240753,
0.11234381794929504,
0.10513705015182495,
0.09944089502096176,
0.0946362093091011,
0.09411308169364929
],
"sources": 9
},
{
"entropy_mean": 2.197190284729004,
"mean_weights": [
0.11067462712526321,
0.11021950840950012,
0.11038416624069214,
0.11103623360395432,
0.11016053706407547,
0.11144962906837463,
0.11242866516113281,
0.11139006912708282,
0.11225657165050507
],
"sources": 9
},
{
"entropy_mean": 2.1896841526031494,
"mean_weights": [
0.11968795955181122,
0.13428261876106262,
0.12373492866754532,
0.11790731549263,
0.11133000254631042,
0.10504785925149918,
0.09994535893201828,
0.09480959177017212,
0.09325435757637024
],
"sources": 9
},
{
"entropy_mean": 2.1971938610076904,
"mean_weights": [
0.11116788536310196,
0.1117495447397232,
0.11143069714307785,
0.11117202788591385,
0.11217275261878967,
0.11135637760162354,
0.11160033196210861,
0.10969291627407074,
0.10965746641159058
],
"sources": 9
},
{
"entropy_mean": 2.188610792160034,
"mean_weights": [
0.11987313628196716,
0.1360916942358017,
0.12462760508060455,
0.11755925416946411,
0.11237476021051407,
0.10438287258148193,
0.09920360147953033,
0.09431086480617523,
0.09157621115446091
],
"sources": 9
}
],
"layer_input_rms_by_sublayer": [
0.028571506962180138,
0.014360599219799042,
0.01502692885696888,
0.014914286322891712,
0.015751885250210762,
0.015590262599289417,
0.016715344041585922,
0.016720104962587357,
0.01783851347863674,
0.01203431747853756,
0.012985615991055965,
0.01319553330540657,
0.014028912410140038,
0.014193638227880001,
0.015184907242655754,
0.015358141623437405,
0.016659293323755264,
0.012656479142606258,
0.013609340414404869,
0.013699201866984367,
0.014725575223565102,
0.01495310477912426,
0.016002535820007324,
0.01618308201432228,
0.017315881326794624,
0.01409733947366476,
0.014967667870223522,
0.01521329116076231,
0.016154561191797256,
0.016374915838241577,
0.01737469807267189,
0.017517324537038803,
0.018479224294424057,
0.015613679774105549,
0.01647813245654106,
0.016780734062194824,
0.017671043053269386,
0.017834074795246124,
0.01892261393368244,
0.01904122717678547,
0.020303834229707718,
0.017550688236951828,
0.01861351728439331,
0.018688000738620758,
0.01984807848930359,
0.019847052171826363,
0.021230308338999748,
0.021104788407683372,
0.022779423743486404,
0.019968591630458832,
0.02138628251850605,
0.021149026229977608,
0.02279111184179783,
0.02249406836926937,
0.024287279695272446,
0.02368815243244171,
0.02611272968351841,
0.02278727851808071,
0.024725768715143204,
0.024227207526564598,
0.026407435536384583,
0.02562388963997364,
0.028016721829771996,
0.0269329734146595
],
"loss_nats": 3.570180892944336,
"loss_scale": 1.0,
"output_weights": {
"entropy_mean": 2.1930694580078125,
"mean_weights": [
0.10071488469839096,
0.09411877393722534,
0.10181896388530731,
0.10989432036876678,
0.1145806610584259,
0.11861918121576309,
0.11984598636627197,
0.12094368040561676,
0.11946355551481247
],
"sources": 9
},
"step": 100,
"stream_state_rms_by_sublayer": [
0.002919268561527133,
0.005866481456905603,
0.0065401275642216206,
0.010142817161977291,
0.010618876665830612,
0.01482025533914566,
0.015120964497327805,
0.01956234686076641,
0.0030634324066340923,
0.0067119961604475975,
0.007598300464451313,
0.011908270418643951,
0.012669830583035946,
0.017094247043132782,
0.017725100740790367,
0.02295343205332756,
0.0037359180860221386,
0.007494525518268347,
0.009108707308769226,
0.013766046613454819,
0.015153576619923115,
0.02038806490600109,
0.021781234070658684,
0.02682213857769966,
0.005531937349587679,
0.009193498641252518,
0.0119093032553792,
0.016781754791736603,
0.01904802769422531,
0.024297412484884262,
0.02626260370016098,
0.03140362724661827,
0.005659298039972782,
0.009884056635200977,
0.013823030516505241,
0.018740201368927956,
0.02223494090139866,
0.02799132652580738,
0.0319429449737072,
0.03797563910484314,
0.007882392033934593,
0.012623735703527927,
0.01784195750951767,
0.02351362258195877,
0.028768405318260193,
0.034899674355983734,
0.040818072855472565,
0.047332651913166046,
0.012831038795411587,
0.01744839735329151,
0.023826664313673973,
0.029514454305171967,
0.03824474662542343,
0.044660378247499466,
0.053081437945365906,
0.0601838044822216,
0.013431218452751637,
0.018972838297486305,
0.03125866502523422,
0.0376126803457737,
0.04969283565878868,
0.0565216988325119,
0.06860727071762085,
0.07583927363157272
]
},
{
"activation_grad_rms_by_block": [
0.00014248774095904082,
0.00014670786913484335,
0.00014566494792234153,
0.00013774850231129676,
0.00013259105617180467,
0.00011885961430380121,
0.00010788240615511313,
9.963813499780372e-05,
9.571473492542282e-05,
8.723558858036995e-05,
8.013081242097542e-05,
7.453432044712827e-05,
7.300077413674444e-05,
6.691268208669499e-05,
6.17568803136237e-05,
5.684744246536866e-05,
5.7382596423849463e-05,
5.201507519814186e-05,
4.7512799937976524e-05,
4.3485648347996175e-05,
4.6350149204954505e-05,
4.233795698382892e-05,
3.877062044921331e-05,
3.5837489122059196e-05,
3.0636481824330986e-05,
2.926486195065081e-05,
2.8229198505869135e-05,
2.7580599635257386e-05,
2.3305021386477165e-05,
2.3107419110601768e-05,
2.298491745023057e-05,
2.3026072085485794e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 0.00012894753399450565,
"first_to_last_ratio": 4.956313899688395,
"imbalance_abs_log_ratio": 1.600662299067145,
"last_quartile_mean": 2.601682149361295e-05,
"mean": 6.873563796716553e-05,
"normalized": [
2.0729820101052394,
2.1343785185339303,
2.1192055857825096,
2.004033226215171,
1.9290001532413559,
1.7292283569198776,
1.5695265126752398,
1.449584785193964,
1.392505223726082,
1.2691464160417294,
1.1657826244262586,
1.0843620958713254,
1.0620513069452608,
0.9734787377496757,
0.8984695878304712,
0.827044661933948,
0.8348303459591176,
0.7567409969045322,
0.6912396733798123,
0.6326506836056272,
0.674324856446311,
0.615953502956551,
0.5640541296457265,
0.5213814868377082,
0.4457146646251511,
0.42575966145291916,
0.4106923182898804,
0.40125618166856064,
0.3390529581992065,
0.3361781427218294,
0.33439592808042723,
0.33499466603460004
],
"population_cv": 0.5868596491376813
},
"activation_output_rms_by_block": [
0.021530626341700554,
0.03857670724391937,
0.04954109713435173,
0.057694368064403534,
0.02140580676496029,
0.025963041931390762,
0.03339661285281181,
0.040483180433511734,
0.02151828072965145,
0.035047177225351334,
0.03725159168243408,
0.043239567428827286,
0.02124657854437828,
0.03028978779911995,
0.03979094699025154,
0.05222286656498909,
0.03916332125663757,
0.04088718071579933,
0.04756740853190422,
0.05542457476258278,
0.020541559904813766,
0.03680417314171791,
0.05126549303531647,
0.06891486048698425,
0.02905990369617939,
0.05358703061938286,
0.07880406081676483,
0.10752817988395691,
0.029148343950510025,
0.054091889411211014,
0.08628532290458679,
0.11884573101997375
],
"activation_output_statistics": {
"first_quartile_mean": 0.036073930095881224,
"first_to_last_ratio": 0.5177916953272103,
"imbalance_abs_log_ratio": 0.6581822501725932,
"last_quartile_mean": 0.0696688077878207,
"mean": 0.04647241474594921,
"normalized": [
0.46329906589537145,
0.8300990481085756,
1.06603234209322,
1.2414755802942754,
0.4606131805713009,
0.5586764120892566,
0.7186330436105184,
0.8711228081179162,
0.4630334112674251,
0.7541501214633191,
0.8015850238486076,
0.9304351337283648,
0.4571868851775181,
0.6517799422454194,
0.8562272308804426,
1.1237390363826774,
0.8427218914862018,
0.8798161433899512,
1.0235622313138022,
1.1926338466716728,
0.4420161942758587,
0.7919574083446129,
1.1031381372276345,
1.482919724824311,
0.6253151219931478,
1.1530933116414785,
1.6957169376190813,
2.3138065984257823,
0.6272181919070765,
1.163956934601208,
1.8566997944110037,
2.5573392660929675
],
"population_cv": 0.5056023373965206
},
"bits_per_byte": 3.0609520355704922,
"branch_output_rms_by_sublayer": [
0.010549778118729591,
0.017611408606171608,
0.013795096427202225,
0.014558257535099983,
0.012347182258963585,
0.01080919150263071,
0.012820021249353886,
0.010327119380235672,
0.01913420483469963,
0.008616750128567219,
0.018582463264465332,
0.007770510856062174,
0.013614931143820286,
0.007606723811477423,
0.013612493872642517,
0.008487755432724953,
0.019129201769828796,
0.00865182839334011,
0.02525762841105461,
0.009020541794598103,
0.027986859902739525,
0.008682658895850182,
0.014458909630775452,
0.008706990629434586,
0.018231205642223358,
0.009003226645290852,
0.016914349049329758,
0.009712434373795986,
0.022478405386209488,
0.00981286820024252,
0.03932488337159157,
0.011337134055793285,
0.03611915931105614,
0.010840619914233685,
0.0266753900796175,
0.010956712998449802,
0.018954744562506676,
0.012231985107064247,
0.014813725836575031,
0.014082341454923153,
0.013802599161863327,
0.014436324127018452,
0.016533223912119865,
0.016183899715542793,
0.016679473221302032,
0.017385432496666908,
0.01826235093176365,
0.02254617027938366,
0.015529165975749493,
0.022119678556919098,
0.011229476891458035,
0.022372635081410408,
0.008122270926833153,
0.024847008287906647,
0.008894859813153744,
0.026897024363279343,
0.009738311171531677,
0.026162855327129364,
0.007465044502168894,
0.024674534797668457,
0.008605006150901318,
0.030845578759908676,
0.009693782776594162,
0.029862264171242714
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.0006151943096300597,
0.0003715258289130363,
0.0003398511463891951,
0.0003030496529751275,
0.00028984271802693696,
0.0002579116693919001,
0.00022400787473702318,
0.00028477639847268295,
0.000278695861793727,
0.00022792247112618847,
0.00023417680672174036,
0.00019296669269337194,
0.00020274366236211453,
0.00017465616055496573,
0.00015881266908702134,
0.00017808900330691153,
0.00016226510475368574,
0.00014508607560593053,
0.00013101353173913878,
0.00011930204103094746,
0.00012786456583361604,
0.00012508100764064805,
0.00012354146515800217,
0.00012514897623388166,
0.00011397604830293932,
9.556324385288696e-05,
9.289013346881659e-05,
9.744025747256133e-05,
8.302691520063822e-05,
7.404706874242776e-05,
8.914255818565412e-05,
8.643783624226864e-05
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.0003357699498169952,
"first_to_last_ratio": 3.666991625028822,
"imbalance_abs_log_ratio": 1.2993716052111575,
"last_quartile_mean": 9.156550768352412e-05,
"mean": 0.00019143905486393895,
"normalized": [
3.2135256312631473,
1.940700288021637,
1.7752445896201103,
1.5830085098911153,
1.514020836724963,
1.3472259857070703,
1.1701263093689624,
1.487556437609251,
1.4557941794677367,
1.1905745736583337,
1.223244686870066,
1.0079797606111183,
1.0590506859021536,
0.9123329650739171,
0.8295729896906037,
0.9302647436986373,
0.8476071137574935,
0.7578708310540252,
0.6843615678747147,
0.6231854890619823,
0.6679126467924371,
0.6533724674390322,
0.645330521746498,
0.6537275078109245,
0.5953646625760102,
0.49918363795102527,
0.4852203931681348,
0.5089883960292994,
0.4336989401647838,
0.386791863316788,
0.46564457941463516,
0.45151620866339115
],
"population_cv": 0.578773657781326
},
"depth_weights": [
{
"entropy_mean": 0.0,
"mean_weights": [
1.0
],
"sources": 1
},
{
"entropy_mean": 0.658259391784668,
"mean_weights": [
0.3887256383895874,
0.6112743616104126
],
"sources": 2
},
{
"entropy_mean": 0.6694332361221313,
"mean_weights": [
0.42782124876976013,
0.5721787214279175
],
"sources": 2
},
{
"entropy_mean": 0.6890689730644226,
"mean_weights": [
0.504529595375061,
0.49547040462493896
],
"sources": 2
},
{
"entropy_mean": 0.6740086078643799,
"mean_weights": [
0.45225459337234497,
0.547745406627655
],
"sources": 2
},
{
"entropy_mean": 0.679557204246521,
"mean_weights": [
0.4461178183555603,
0.5538821816444397
],
"sources": 2
},
{
"entropy_mean": 0.6740646362304688,
"mean_weights": [
0.44189155101776123,
0.5581084489822388
],
"sources": 2
},
{
"entropy_mean": 0.6792672276496887,
"mean_weights": [
0.4455937147140503,
0.5544062852859497
],
"sources": 2
},
{
"entropy_mean": 0.674970269203186,
"mean_weights": [
0.44674086570739746,
0.5532591342926025
],
"sources": 2
},
{
"entropy_mean": 1.093062162399292,
"mean_weights": [
0.3082374036312103,
0.33985745906829834,
0.35190510749816895
],
"sources": 3
},
{
"entropy_mean": 1.0853323936462402,
"mean_weights": [
0.30180174112319946,
0.3268374502658844,
0.3713608384132385
],
"sources": 3
},
{
"entropy_mean": 1.0921990871429443,
"mean_weights": [
0.3034760355949402,
0.3359980285167694,
0.3605259656906128
],
"sources": 3
},
{
"entropy_mean": 1.076724648475647,
"mean_weights": [
0.29931002855300903,
0.30481812357902527,
0.3958718776702881
],
"sources": 3
},
{
"entropy_mean": 1.0856893062591553,
"mean_weights": [
0.2801036834716797,
0.3310527801513672,
0.3888435363769531
],
"sources": 3
},
{
"entropy_mean": 1.0752873420715332,
"mean_weights": [
0.29463040828704834,
0.31052330136299133,
0.3948462903499603
],
"sources": 3
},
{
"entropy_mean": 1.088913917541504,
"mean_weights": [
0.29400017857551575,
0.32166045904159546,
0.3843393623828888
],
"sources": 3
},
{
"entropy_mean": 1.054490327835083,
"mean_weights": [
0.2850497364997864,
0.2890807092189789,
0.42586958408355713
],
"sources": 3
},
{
"entropy_mean": 1.3700100183486938,
"mean_weights": [
0.21610887348651886,
0.2282472848892212,
0.2728016972541809,
0.28284209966659546
],
"sources": 4
},
{
"entropy_mean": 1.3620216846466064,
"mean_weights": [
0.22791308164596558,
0.2214205265045166,
0.2562018632888794,
0.2944645285606384
],
"sources": 4
},
{
"entropy_mean": 1.3784711360931396,
"mean_weights": [
0.23347607254981995,
0.23307976126670837,
0.2570483684539795,
0.2763957977294922
],
"sources": 4
},
{
"entropy_mean": 1.3619427680969238,
"mean_weights": [
0.24715131521224976,
0.22003202140331268,
0.23014183342456818,
0.3026748299598694
],
"sources": 4
},
{
"entropy_mean": 1.3804662227630615,
"mean_weights": [
0.24302782118320465,
0.23108476400375366,
0.24228686094284058,
0.2836005389690399
],
"sources": 4
},
{
"entropy_mean": 1.3502743244171143,
"mean_weights": [
0.2455797642469406,
0.21441739797592163,
0.2208172082901001,
0.31918564438819885
],
"sources": 4
},
{
"entropy_mean": 1.3820215463638306,
"mean_weights": [
0.24522465467453003,
0.22846508026123047,
0.25027886033058167,
0.27603137493133545
],
"sources": 4
},
{
"entropy_mean": 1.3491160869598389,
"mean_weights": [
0.24498313665390015,
0.21178750693798065,
0.2205154001712799,
0.3227139711380005
],
"sources": 4
},
{
"entropy_mean": 1.6032840013504028,
"mean_weights": [
0.20074918866157532,
0.17959122359752655,
0.18619459867477417,
0.2124267816543579,
0.22103822231292725
],
"sources": 5
},
{
"entropy_mean": 1.5699318647384644,
"mean_weights": [
0.1938496232032776,
0.17019830644130707,
0.16257275640964508,
0.20397180318832397,
0.2694075107574463
],
"sources": 5
},
{
"entropy_mean": 1.6068798303604126,
"mean_weights": [
0.19717518985271454,
0.1942940056324005,
0.19104912877082825,
0.20343774557113647,
0.21404394507408142
],
"sources": 5
},
{
"entropy_mean": 1.5761134624481201,
"mean_weights": [
0.1940101534128189,
0.18250694870948792,
0.16849923133850098,
0.18937936425209045,
0.26560431718826294
],
"sources": 5
},
{
"entropy_mean": 1.6027262210845947,
"mean_weights": [
0.20452985167503357,
0.18702910840511322,
0.17977532744407654,
0.20095866918563843,
0.22770702838897705
],
"sources": 5
},
{
"entropy_mean": 1.577937126159668,
"mean_weights": [
0.1962222307920456,
0.1818992793560028,
0.17167484760284424,
0.18149182200431824,
0.2687118649482727
],
"sources": 5
},
{
"entropy_mean": 1.5977532863616943,
"mean_weights": [
0.2153339385986328,
0.18402469158172607,
0.17019470036029816,
0.1919252574443817,
0.23852142691612244
],
"sources": 5
},
{
"entropy_mean": 1.578121542930603,
"mean_weights": [
0.20422281324863434,
0.1835813820362091,
0.16642534732818604,
0.17722904682159424,
0.26854145526885986
],
"sources": 5
},
{
"entropy_mean": 1.782798409461975,
"mean_weights": [
0.15961389243602753,
0.16475999355316162,
0.14714640378952026,
0.15768738090991974,
0.1760568618774414,
0.19473549723625183
],
"sources": 6
},
{
"entropy_mean": 1.7707297801971436,
"mean_weights": [
0.16306114196777344,
0.1578725278377533,
0.139655202627182,
0.14299748837947845,
0.18966570496559143,
0.20674791932106018
],
"sources": 6
},
{
"entropy_mean": 1.781312108039856,
"mean_weights": [
0.15919369459152222,
0.16844786703586578,
0.14742718636989594,
0.1539241373538971,
0.1669306755065918,
0.20407646894454956
],
"sources": 6
},
{
"entropy_mean": 1.7736209630966187,
"mean_weights": [
0.15695160627365112,
0.16567477583885193,
0.1478763222694397,
0.14320015907287598,
0.17375096678733826,
0.2125461995601654
],
"sources": 6
},
{
"entropy_mean": 1.781569480895996,
"mean_weights": [
0.15368077158927917,
0.1725173443555832,
0.14978237450122833,
0.15674246847629547,
0.16688688099384308,
0.20039016008377075
],
"sources": 6
},
{
"entropy_mean": 1.7793617248535156,
"mean_weights": [
0.1678246557712555,
0.1683025360107422,
0.14878368377685547,
0.14351718127727509,
0.16739681363105774,
0.20417512953281403
],
"sources": 6
},
{
"entropy_mean": 1.7772380113601685,
"mean_weights": [
0.1536770462989807,
0.18459689617156982,
0.14832255244255066,
0.15016385912895203,
0.1652815043926239,
0.19795814156532288
],
"sources": 6
},
{
"entropy_mean": 1.780207633972168,
"mean_weights": [
0.16951784491539001,
0.17200809717178345,
0.15038040280342102,
0.14669056236743927,
0.16062122583389282,
0.20078186690807343
],
"sources": 6
},
{
"entropy_mean": 1.937079668045044,
"mean_weights": [
0.13008031249046326,
0.15163566172122955,
0.1323973685503006,
0.13641127943992615,
0.13806134462356567,
0.1444631665945053,
0.16695083677768707
],
"sources": 7
},
{
"entropy_mean": 1.939245343208313,
"mean_weights": [
0.14692582190036774,
0.14671875536441803,
0.13336658477783203,
0.1307726800441742,
0.131200909614563,
0.147438645362854,
0.16357660293579102
],
"sources": 7
},
{
"entropy_mean": 1.9312366247177124,
"mean_weights": [
0.12334205210208893,
0.16950437426567078,
0.13870283961296082,
0.1300538182258606,
0.12885476648807526,
0.14288470149040222,
0.1666574478149414
],
"sources": 7
},
{
"entropy_mean": 1.9345850944519043,
"mean_weights": [
0.1465124636888504,
0.153631791472435,
0.13159149885177612,
0.13120220601558685,
0.1253916621208191,
0.14393125474452972,
0.1677391082048416
],
"sources": 7
},
{
"entropy_mean": 1.9284589290618896,
"mean_weights": [
0.12412485480308533,
0.1698783040046692,
0.13807369768619537,
0.13034150004386902,
0.13028821349143982,
0.1419651210308075,
0.16532832384109497
],
"sources": 7
},
{
"entropy_mean": 1.9329564571380615,
"mean_weights": [
0.1454196274280548,
0.1566689908504486,
0.1286500096321106,
0.12666445970535278,
0.12782412767410278,
0.14222374558448792,
0.17254900932312012
],
"sources": 7
},
{
"entropy_mean": 1.9259672164916992,
"mean_weights": [
0.12082186341285706,
0.17791953682899475,
0.1424643099308014,
0.13298648595809937,
0.12929362058639526,
0.1344166398048401,
0.16209754347801208
],
"sources": 7
},
{
"entropy_mean": 1.9201786518096924,
"mean_weights": [
0.13756656646728516,
0.17355048656463623,
0.14306491613388062,
0.13006147742271423,
0.11658851057291031,
0.139857679605484,
0.15931037068367004
],
"sources": 7
},
{
"entropy_mean": 2.043663501739502,
"mean_weights": [
0.10298450291156769,
0.1751018464565277,
0.1414714753627777,
0.12628036737442017,
0.11767096072435379,
0.12069195508956909,
0.12702946364879608,
0.08876943588256836
],
"sources": 8
},
{
"entropy_mean": 2.0607244968414307,
"mean_weights": [
0.1220984011888504,
0.1498527079820633,
0.12913095951080322,
0.1179434061050415,
0.10775719583034515,
0.12020602822303772,
0.13844498991966248,
0.11456632614135742
],
"sources": 8
},
{
"entropy_mean": 2.0388383865356445,
"mean_weights": [
0.1025313287973404,
0.18260905146598816,
0.14208021759986877,
0.12606360018253326,
0.11357533931732178,
0.11596501618623734,
0.12797299027442932,
0.08920245617628098
],
"sources": 8
},
{
"entropy_mean": 2.0634398460388184,
"mean_weights": [
0.12216806411743164,
0.15090107917785645,
0.12766355276107788,
0.1134132519364357,
0.10988184064626694,
0.11335282772779465,
0.13999530673027039,
0.12262405455112457
],
"sources": 8
},
{
"entropy_mean": 2.0386834144592285,
"mean_weights": [
0.10029368102550507,
0.17931804060935974,
0.14185777306556702,
0.13114282488822937,
0.11872631311416626,
0.11815309524536133,
0.12660562992095947,
0.08390261977910995
],
"sources": 8
},
{
"entropy_mean": 2.0626261234283447,
"mean_weights": [
0.11644256860017776,
0.15398627519607544,
0.1359611600637436,
0.12036904692649841,
0.10964122414588928,
0.11886263638734818,
0.13317911326885223,
0.1115579903125763
],
"sources": 8
},
{
"entropy_mean": 2.020051956176758,
"mean_weights": [
0.09590239822864532,
0.1913527548313141,
0.14906750619411469,
0.1341754049062729,
0.11847006529569626,
0.11603336781263351,
0.12013453245162964,
0.07486395537853241
],
"sources": 8
},
{
"entropy_mean": 2.0452136993408203,
"mean_weights": [
0.11420649290084839,
0.15673109889030457,
0.13583534955978394,
0.12758757174015045,
0.1112617775797844,
0.13417255878448486,
0.12519711256027222,
0.09500804543495178
],
"sources": 8
},
{
"entropy_mean": 2.1403071880340576,
"mean_weights": [
0.08488726615905762,
0.1673685610294342,
0.13726134598255157,
0.12220048159360886,
0.10616928339004517,
0.1037440225481987,
0.10401669889688492,
0.06658264249563217,
0.10776971280574799
],
"sources": 9
},
{
"entropy_mean": 2.18704891204834,
"mean_weights": [
0.10885682702064514,
0.12684625387191772,
0.11236055195331573,
0.10673345625400543,
0.10181289166212082,
0.11441421508789062,
0.11840946972370148,
0.10466625541448593,
0.1059001013636589
],
"sources": 9
},
{
"entropy_mean": 2.1283018589019775,
"mean_weights": [
0.08667369186878204,
0.1812124401330948,
0.14178472757339478,
0.12440058588981628,
0.11160320788621902,
0.10330478847026825,
0.11197598278522491,
0.06831350177526474,
0.07073107361793518
],
"sources": 9
},
{
"entropy_mean": 2.173035144805908,
"mean_weights": [
0.10752685368061066,
0.13639645278453827,
0.11839058995246887,
0.11235916614532471,
0.10138258337974548,
0.11953261494636536,
0.11256720125675201,
0.09452059864997864,
0.097323939204216
],
"sources": 9
},
{
"entropy_mean": 2.1028335094451904,
"mean_weights": [
0.08200174570083618,
0.19083696603775024,
0.15512439608573914,
0.1322997808456421,
0.11209174990653992,
0.10435720533132553,
0.1024213582277298,
0.06072331964969635,
0.06014348566532135
],
"sources": 9
},
{
"entropy_mean": 2.147671937942505,
"mean_weights": [
0.09926989674568176,
0.14984291791915894,
0.12505242228507996,
0.13004425168037415,
0.1122550517320633,
0.12226949632167816,
0.10689133405685425,
0.07532045990228653,
0.07905416935682297
],
"sources": 9
},
{
"entropy_mean": 2.109609603881836,
"mean_weights": [
0.08386656641960144,
0.18956288695335388,
0.1487923562526703,
0.12990039587020874,
0.11159682273864746,
0.10738210380077362,
0.10535432398319244,
0.0630122721195221,
0.06053228676319122
],
"sources": 9
}
],
"layer_input_rms_by_sublayer": [
0.04393457621335983,
0.018067941069602966,
0.024121688678860664,
0.027584951370954514,
0.030756350606679916,
0.03231721371412277,
0.03544071316719055,
0.036101218312978745,
0.03840360417962074,
0.025345133617520332,
0.026969486847519875,
0.027166159823536873,
0.027978016063570976,
0.028526438400149345,
0.029923241585493088,
0.02979987859725952,
0.03152087330818176,
0.02248864620923996,
0.024161184206604958,
0.023894935846328735,
0.025710709393024445,
0.02451697364449501,
0.02641460672020912,
0.025183402001857758,
0.02742084674537182,
0.020499059930443764,
0.021717891097068787,
0.02105838619172573,
0.0224718376994133,
0.0216353852301836,
0.023445574566721916,
0.022715488448739052,
0.024849263951182365,
0.01881815865635872,
0.01992032676935196,
0.0193756353110075,
0.02058842033147812,
0.02001863345503807,
0.020979218184947968,
0.021088071167469025,
0.02214338816702366,
0.018417155370116234,
0.01903437450528145,
0.019219009205698967,
0.02008410543203354,
0.020342160016298294,
0.021635910496115685,
0.02150510996580124,
0.023218298330903053,
0.02099662832915783,
0.022257626056671143,
0.02211105264723301,
0.024358727037906647,
0.02328667603433132,
0.025900650769472122,
0.024160878732800484,
0.027197737246751785,
0.02260657213628292,
0.02823900803923607,
0.024989914149045944,
0.02913021855056286,
0.02536177635192871,
0.029048049822449684,
0.027097374200820923
],
"loss_nats": 2.121690273284912,
"loss_scale": 1.0,
"output_weights": {
"entropy_mean": 2.1622862815856934,
"mean_weights": [
0.12356589734554291,
0.09235234558582306,
0.08736804127693176,
0.08735492080450058,
0.08889827132225037,
0.10504580289125443,
0.14004790782928467,
0.14956718683242798,
0.12579965591430664
],
"sources": 9
},
"step": 500,
"stream_state_rms_by_sublayer": [
0.010549778118729591,
0.021530626341700554,
0.02910016104578972,
0.03857670724391937,
0.04255657270550728,
0.04954109713435173,
0.05177471786737442,
0.057694368064403534,
0.01913420483469963,
0.02140580676496029,
0.02246284671127796,
0.025963041931390762,
0.029843268916010857,
0.03339661285281181,
0.036163609474897385,
0.040483180433511734,
0.019129201769828796,
0.02151828072965145,
0.0314205139875412,
0.035047177225351334,
0.03302976116538048,
0.03725159168243408,
0.03871368616819382,
0.043239567428827286,
0.018231205642223358,
0.02124657854437828,
0.02580643631517887,
0.03028978779911995,
0.0349617674946785,
0.03979094699025154,
0.04659247398376465,
0.05222286656498909,
0.03611915931105614,
0.03916332125663757,
0.03664228320121765,
0.04088718071579933,
0.04177439957857132,
0.04756740853190422,
0.04776441678404808,
0.05542457476258278,
0.013802599161863327,
0.020541559904813766,
0.02693362347781658,
0.03680417314171791,
0.03932841867208481,
0.05126549303531647,
0.0514267161488533,
0.06891486048698425,
0.015529165975749493,
0.02905990369617939,
0.03558400645852089,
0.05358703061938286,
0.05707564577460289,
0.07880406081676483,
0.08332383632659912,
0.10752817988395691,
0.009738311171531677,
0.029148343950510025,
0.03187238425016403,
0.054091889411211014,
0.05786185711622238,
0.08628532290458679,
0.09122034162282944,
0.11884573101997375
]
},
{
"activation_grad_rms_by_block": [
6.054548066458665e-05,
7.096362969605252e-05,
6.466427294071764e-05,
5.866752690053545e-05,
5.6290406064363196e-05,
5.253175913821906e-05,
4.870586417382583e-05,
4.417003583512269e-05,
3.822600410785526e-05,
3.59821024176199e-05,
3.430073047638871e-05,
3.306051803519949e-05,
3.607416874729097e-05,
3.430179276620038e-05,
3.275281051173806e-05,
3.160809865221381e-05,
3.903685137629509e-05,
3.708450458361767e-05,
3.9304726669797674e-05,
4.4666798203252256e-05,
0.00014839931100141257,
0.00012039772263960913,
9.753467020345852e-05,
7.931287109386176e-05,
7.865376392146572e-05,
6.111936090746894e-05,
4.65054290543776e-05,
4.1926850826712325e-05,
2.5328776246169582e-05,
2.2645597709924914e-05,
2.1881067368667573e-05,
2.167661841667723e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 5.706737192667788e-05,
"first_to_last_ratio": 1.4278557447018398,
"imbalance_abs_log_ratio": 0.3561738396982594,
"last_quartile_mean": 3.996718305643299e-05,
"mean": 5.182250379220932e-05,
"normalized": [
1.1683241108409874,
1.3693593420455452,
1.2478029467661282,
1.1320859203517522,
1.0862154844943164,
1.0136862423485675,
0.9398593392770029,
0.8523331101914637,
0.7376332926932397,
0.6943335382230071,
0.6618887156422048,
0.6379567874173152,
0.6961101086882286,
0.6619092142621851,
0.6320190672968202,
0.6099299790483219,
0.7532799174046017,
0.7156061917099562,
0.758449009476584,
0.8619189528616946,
2.863607509132394,
2.3232710469251585,
1.8820910428129736,
1.530471615417879,
1.517753064129064,
1.1793980690808934,
0.8973983434078874,
0.8090471852696426,
0.4887601793176453,
0.43698385938136236,
0.4222309955613731,
0.41828581852380436
],
"population_cv": 0.5384894543542103
},
"activation_output_rms_by_block": [
0.09451264888048172,
0.2031717449426651,
0.22856928408145905,
0.2430499792098999,
0.08243431895971298,
0.09709372371435165,
0.11282631754875183,
0.13645486533641815,
0.05442482605576515,
0.0885990634560585,
0.12215005606412888,
0.15440259873867035,
0.060873307287693024,
0.09471380710601807,
0.1260664463043213,
0.163865864276886,
0.08853588253259659,
0.12131853401660919,
0.15660560131072998,
0.2097783088684082,
0.04455159604549408,
0.07411240041255951,
0.10008715838193893,
0.13061410188674927,
0.04960336163640022,
0.07874798029661179,
0.11358420550823212,
0.1580832600593567,
0.06955241411924362,
0.13302887976169586,
0.22457914054393768,
0.32859006524086
],
"activation_output_statistics": {
"first_quartile_mean": 0.14976411033421755,
"first_to_last_ratio": 1.036636701844262,
"imbalance_abs_log_ratio": 0.035981532131895505,
"last_quartile_mean": 0.14447116339579225,
"mean": 0.12951817945577204,
"normalized": [
0.7297249643070838,
1.5686735699681784,
1.7647660354854744,
1.8765703797812938,
0.636469098825333,
0.749653246535227,
0.8711234054002422,
1.0535576234146717,
0.42020993720306404,
0.6840666215997374,
0.9431112804194469,
1.192130706187048,
0.4699981697046636,
0.7312780916470571,
0.9733494312076132,
1.2651958587237788,
0.6835788065012903,
0.9366911620158862,
1.2091399212742004,
1.6196823469097898,
0.34397948019882096,
0.5722161994862464,
0.7727653276358534,
1.008461533628578,
0.3829837776042769,
0.608007158744095,
0.8769750006177236,
1.2205488119398642,
0.5370088925855727,
1.0271058497013745,
1.7339584411053826,
2.5370188696411313
],
"population_cv": 0.4929365839545065
},
"bits_per_byte": 2.073383110809672,
"branch_output_rms_by_sublayer": [
0.04804428666830063,
0.07808538526296616,
0.12356939166784286,
0.03881331905722618,
0.05923242121934891,
0.01480295043438673,
0.04508936405181885,
0.02500341460108757,
0.0799286738038063,
0.015180142596364021,
0.03290615975856781,
0.013721407391130924,
0.03170505166053772,
0.016902483999729156,
0.03961387276649475,
0.03325798362493515,
0.04508088901638985,
0.026172807440161705,
0.0312613770365715,
0.027887089177966118,
0.029994890093803406,
0.03240044787526131,
0.03058313950896263,
0.03682815283536911,
0.0497574545443058,
0.029301028698682785,
0.03759258985519409,
0.0338764488697052,
0.030690157786011696,
0.0393521711230278,
0.03000541403889656,
0.046694882214069366,
0.079553984105587,
0.029503758996725082,
0.0414140485227108,
0.03615153953433037,
0.04122982919216156,
0.030687781050801277,
0.05830265209078789,
0.05279085040092468,
0.03440643846988678,
0.024021649733185768,
0.033897366374731064,
0.03336174041032791,
0.02957344427704811,
0.04255227372050285,
0.03406442329287529,
0.0520954355597496,
0.025225287303328514,
0.03984410688281059,
0.01915411651134491,
0.04249191656708717,
0.014526215381920338,
0.05309459939599037,
0.011566666886210442,
0.06231515854597092,
0.0242468249052763,
0.06388300657272339,
0.013597902841866016,
0.0749494656920433,
0.020146220922470093,
0.10156872868537903,
0.02710210345685482,
0.11044161766767502
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.0002879944983597963,
0.00022853159560210997,
0.00014071406888316784,
0.00014318671869519129,
0.00013886536811003923,
0.00010286844600842235,
9.677924556041764e-05,
0.0001337288949114584,
0.00010758657043361882,
9.13787469069405e-05,
9.373485859236601e-05,
9.381017142422257e-05,
0.00010148426099734976,
9.552468088000852e-05,
0.00010200246393672037,
0.00010753255944705701,
0.00013288008273706816,
0.00012966970691820486,
0.0001395588859415414,
0.00019785772756123382,
0.0004949688435464942,
0.0005919211037074493,
0.000516172452953512,
0.0004666614668327092,
0.0003441119880144902,
0.00027788329882105144,
0.00022798541204965527,
0.00021216050695355764,
0.00012431994828037904,
0.00011340006302052245,
0.00011536177905162644,
0.00011797314476595472
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.00015908360451632537,
"first_to_last_ratio": 0.8300756844691923,
"imbalance_abs_log_ratio": 0.18623839624123317,
"last_quartile_mean": 0.00019164951761965464,
"mean": 0.00019589404874701053,
"normalized": [
1.4701544033720488,
1.1666081591751138,
0.7183172219024098,
0.730939605419618,
0.7088799736299207,
0.5251228746681984,
0.4940387223575344,
0.682659303673712,
0.5492079576779928,
0.4664702551783621,
0.4784977348312449,
0.4788821917983569,
0.5180568866000248,
0.4876344201930039,
0.5207022091235275,
0.5489322423772612,
0.6783262870260932,
0.661937959563377,
0.7124202436684344,
1.01002418821186,
2.526717104028653,
3.0216390313719645,
2.6349572958192744,
2.3822135987162336,
1.7566229817369148,
1.4185387488719823,
1.1638200012093756,
1.0830370208313709,
0.6346285100316312,
0.5788846764149236,
0.5888988450109153,
0.6022293455086652
],
"population_cv": 0.7015245062301059
},
"depth_weights": [
{
"entropy_mean": 0.0,
"mean_weights": [
1.0
],
"sources": 1
},
{
"entropy_mean": 0.4738388657569885,
"mean_weights": [
0.8067666292190552,
0.19323334097862244
],
"sources": 2
},
{
"entropy_mean": 0.6344536542892456,
"mean_weights": [
0.4895974397659302,
0.5104025602340698
],
"sources": 2
},
{
"entropy_mean": 0.41773349046707153,
"mean_weights": [
0.8461704254150391,
0.15382957458496094
],
"sources": 2
},
{
"entropy_mean": 0.6868985891342163,
"mean_weights": [
0.5008196830749512,
0.49918028712272644
],
"sources": 2
},
{
"entropy_mean": 0.5841774344444275,
"mean_weights": [
0.684928297996521,
0.315071702003479
],
"sources": 2
},
{
"entropy_mean": 0.6896515488624573,
"mean_weights": [
0.5092520117759705,
0.49074798822402954
],
"sources": 2
},
{
"entropy_mean": 0.5611414909362793,
"mean_weights": [
0.7280503511428833,
0.2719496488571167
],
"sources": 2
},
{
"entropy_mean": 0.6817618608474731,
"mean_weights": [
0.5313050746917725,
0.46869492530822754
],
"sources": 2
},
{
"entropy_mean": 1.0218309164047241,
"mean_weights": [
0.4974328279495239,
0.24236315488815308,
0.2602040767669678
],
"sources": 3
},
{
"entropy_mean": 1.0932960510253906,
"mean_weights": [
0.3533782958984375,
0.29999595880508423,
0.34662577509880066
],
"sources": 3
},
{
"entropy_mean": 1.0456881523132324,
"mean_weights": [
0.4338536858558655,
0.23105540871620178,
0.3350909352302551
],
"sources": 3
},
{
"entropy_mean": 1.0891339778900146,
"mean_weights": [
0.33998236060142517,
0.2821658253669739,
0.37785181403160095
],
"sources": 3
},
{
"entropy_mean": 1.038641333580017,
"mean_weights": [
0.34165239334106445,
0.23904743790626526,
0.4193001389503479
],
"sources": 3
},
{
"entropy_mean": 1.0754427909851074,
"mean_weights": [
0.3781457543373108,
0.24021697044372559,
0.3816372752189636
],
"sources": 3
},
{
"entropy_mean": 1.0433809757232666,
"mean_weights": [
0.37802231311798096,
0.21331989765167236,
0.4086577892303467
],
"sources": 3
},
{
"entropy_mean": 1.0922105312347412,
"mean_weights": [
0.3460429906845093,
0.3074926435947418,
0.3464643955230713
],
"sources": 3
},
{
"entropy_mean": 1.3404325246810913,
"mean_weights": [
0.2880150079727173,
0.16260021924972534,
0.2385387420654297,
0.3108460605144501
],
"sources": 4
},
{
"entropy_mean": 1.3765960931777954,
"mean_weights": [
0.28703436255455017,
0.2552259564399719,
0.2101576328277588,
0.24758204817771912
],
"sources": 4
},
{
"entropy_mean": 1.3274900913238525,
"mean_weights": [
0.33539247512817383,
0.1987970769405365,
0.20543015003204346,
0.2603802978992462
],
"sources": 4
},
{
"entropy_mean": 1.3747133016586304,
"mean_weights": [
0.27988845109939575,
0.23608103394508362,
0.20066317915916443,
0.2833673357963562
],
"sources": 4
},
{
"entropy_mean": 1.3239198923110962,
"mean_weights": [
0.3361120820045471,
0.20296622812747955,
0.1885247528553009,
0.2723969519138336
],
"sources": 4
},
{
"entropy_mean": 1.3692121505737305,
"mean_weights": [
0.28167375922203064,
0.21946553885936737,
0.20140981674194336,
0.2974509000778198
],
"sources": 4
},
{
"entropy_mean": 1.3342361450195312,
"mean_weights": [
0.36468517780303955,
0.20593920350074768,
0.18943950533866882,
0.23993611335754395
],
"sources": 4
},
{
"entropy_mean": 1.3645737171173096,
"mean_weights": [
0.2597482204437256,
0.22607669234275818,
0.209133118391037,
0.30504196882247925
],
"sources": 4
},
{
"entropy_mean": 1.5910131931304932,
"mean_weights": [
0.2617476284503937,
0.18299485743045807,
0.17935843765735626,
0.17898105084896088,
0.19691801071166992
],
"sources": 5
},
{
"entropy_mean": 1.5884685516357422,
"mean_weights": [
0.2046908289194107,
0.17294223606586456,
0.15044087171554565,
0.20923088490962982,
0.26269516348838806
],
"sources": 5
},
{
"entropy_mean": 1.599259853363037,
"mean_weights": [
0.21657392382621765,
0.20580387115478516,
0.19366897642612457,
0.1731838881969452,
0.21076934039592743
],
"sources": 5
},
{
"entropy_mean": 1.5786192417144775,
"mean_weights": [
0.20599734783172607,
0.20593789219856262,
0.13473063707351685,
0.17552703619003296,
0.2778070569038391
],
"sources": 5
},
{
"entropy_mean": 1.580681324005127,
"mean_weights": [
0.21277281641960144,
0.18522506952285767,
0.18489867448806763,
0.17205308377742767,
0.2450503408908844
],
"sources": 5
},
{
"entropy_mean": 1.5705549716949463,
"mean_weights": [
0.2125053107738495,
0.20224159955978394,
0.12915469706058502,
0.1714920997619629,
0.28460633754730225
],
"sources": 5
},
{
"entropy_mean": 1.5599372386932373,
"mean_weights": [
0.2665032148361206,
0.181569442152977,
0.15844659507274628,
0.14596089720726013,
0.2475198209285736
],
"sources": 5
},
{
"entropy_mean": 1.562828540802002,
"mean_weights": [
0.21806953847408295,
0.22324788570404053,
0.11591021716594696,
0.17848318815231323,
0.26428914070129395
],
"sources": 5
},
{
"entropy_mean": 1.761409044265747,
"mean_weights": [
0.1612042486667633,
0.1651904582977295,
0.1752963364124298,
0.14996090531349182,
0.2072838991880417,
0.14106415212154388
],
"sources": 6
},
{
"entropy_mean": 1.7366840839385986,
"mean_weights": [
0.1851401925086975,
0.1663871705532074,
0.09063514322042465,
0.11971023678779602,
0.18503433465957642,
0.2530929446220398
],
"sources": 6
},
{
"entropy_mean": 1.777672290802002,
"mean_weights": [
0.15483248233795166,
0.16576308012008667,
0.16957050561904907,
0.14225167036056519,
0.16980668902397156,
0.19777557253837585
],
"sources": 6
},
{
"entropy_mean": 1.7414222955703735,
"mean_weights": [
0.18020270764827728,
0.1900298297405243,
0.09944288432598114,
0.11960913985967636,
0.15577411651611328,
0.25494131445884705
],
"sources": 6
},
{
"entropy_mean": 1.733978509902954,
"mean_weights": [
0.15204599499702454,
0.14166373014450073,
0.1616208702325821,
0.1256796270608902,
0.1571422517299652,
0.26184749603271484
],
"sources": 6
},
{
"entropy_mean": 1.7344707250595093,
"mean_weights": [
0.1829337775707245,
0.18416841328144073,
0.09014187753200531,
0.12335793673992157,
0.15779981017112732,
0.2615981996059418
],
"sources": 6
},
{
"entropy_mean": 1.735896348953247,
"mean_weights": [
0.17276549339294434,
0.15101772546768188,
0.13220663368701935,
0.11932596564292908,
0.15458261966705322,
0.27010154724121094
],
"sources": 6
},
{
"entropy_mean": 1.7644665241241455,
"mean_weights": [
0.19160008430480957,
0.1915181279182434,
0.11520625650882721,
0.1341419219970703,
0.1580837219953537,
0.2094498574733734
],
"sources": 6
},
{
"entropy_mean": 1.6835086345672607,
"mean_weights": [
0.10065847635269165,
0.09339895844459534,
0.09682720899581909,
0.08155890554189682,
0.09126688539981842,
0.10349369794130325,
0.4327958822250366
],
"sources": 7
},
{
"entropy_mean": 1.7937740087509155,
"mean_weights": [
0.13723835349082947,
0.12937892973423004,
0.07450413703918457,
0.08537043631076813,
0.09880411624908447,
0.12944120168685913,
0.3452628552913666
],
"sources": 7
},
{
"entropy_mean": 1.5790722370147705,
"mean_weights": [
0.09500400722026825,
0.08892714232206345,
0.09081116318702698,
0.07538282871246338,
0.07685283571481705,
0.08106325566768646,
0.49195876717567444
],
"sources": 7
},
{
"entropy_mean": 1.8034920692443848,
"mean_weights": [
0.142653226852417,
0.13419847190380096,
0.07437048852443695,
0.09070360660552979,
0.1013825386762619,
0.12073953449726105,
0.33595213294029236
],
"sources": 7
},
{
"entropy_mean": 1.5278494358062744,
"mean_weights": [
0.10045769065618515,
0.08844951540231705,
0.08561251312494278,
0.06598107516765594,
0.06767018139362335,
0.07361933588981628,
0.51820969581604
],
"sources": 7
},
{
"entropy_mean": 1.8435695171356201,
"mean_weights": [
0.14839640259742737,
0.1482572704553604,
0.08113027364015579,
0.09534022957086563,
0.10612697899341583,
0.12550383806228638,
0.2952450215816498
],
"sources": 7
},
{
"entropy_mean": 1.6421003341674805,
"mean_weights": [
0.11422763019800186,
0.10543526709079742,
0.09285669028759003,
0.07746599614620209,
0.07571054995059967,
0.08126918226480484,
0.4530346691608429
],
"sources": 7
},
{
"entropy_mean": 1.8733552694320679,
"mean_weights": [
0.14798465371131897,
0.16052685678005219,
0.09668852388858795,
0.10477185249328613,
0.10857058316469193,
0.11694246530532837,
0.26451510190963745
],
"sources": 7
},
{
"entropy_mean": 1.784506916999817,
"mean_weights": [
0.08589775860309601,
0.08786045759916306,
0.07714233547449112,
0.06314891576766968,
0.05684572085738182,
0.056072384119033813,
0.29875272512435913,
0.2742796838283539
],
"sources": 8
},
{
"entropy_mean": 1.9675533771514893,
"mean_weights": [
0.11549945175647736,
0.12592051923274994,
0.07530814409255981,
0.08277833461761475,
0.08296641707420349,
0.09016790241003036,
0.17876628041267395,
0.24859298765659332
],
"sources": 8
},
{
"entropy_mean": 1.7119876146316528,
"mean_weights": [
0.08804735541343689,
0.07527872174978256,
0.05666808784008026,
0.05041901767253876,
0.05142981559038162,
0.05313319340348244,
0.2591783106327057,
0.3658455014228821
],
"sources": 8
},
{
"entropy_mean": 1.9094492197036743,
"mean_weights": [
0.10337945818901062,
0.10583682358264923,
0.08642245084047318,
0.08043928444385529,
0.0717030018568039,
0.0645938515663147,
0.21096080541610718,
0.2766643166542053
],
"sources": 8
},
{
"entropy_mean": 1.7860023975372314,
"mean_weights": [
0.09247160702943802,
0.0784597396850586,
0.07236532866954803,
0.05986309051513672,
0.05389959365129471,
0.06059400737285614,
0.2579019069671631,
0.3244447112083435
],
"sources": 8
},
{
"entropy_mean": 1.9232951402664185,
"mean_weights": [
0.1160542219877243,
0.11346358060836792,
0.08816123008728027,
0.08138670027256012,
0.07161372900009155,
0.06013647839426994,
0.21822679042816162,
0.2509572505950928
],
"sources": 8
},
{
"entropy_mean": 1.8294003009796143,
"mean_weights": [
0.08831655979156494,
0.10072161257266998,
0.0818098857998848,
0.07514279335737228,
0.06651484966278076,
0.06491033732891083,
0.31431835889816284,
0.20826560258865356
],
"sources": 8
},
{
"entropy_mean": 1.9393219947814941,
"mean_weights": [
0.11041727662086487,
0.12080886960029602,
0.09995828568935394,
0.09934850037097931,
0.08031868189573288,
0.075287364423275,
0.2604207992553711,
0.15344023704528809
],
"sources": 8
},
{
"entropy_mean": 1.9925593137741089,
"mean_weights": [
0.08698172867298126,
0.08857265114784241,
0.07299254834651947,
0.060478877276182175,
0.054115522652864456,
0.056821923702955246,
0.2503892779350281,
0.14502599835395813,
0.18462148308753967
],
"sources": 9
},
{
"entropy_mean": 2.068819999694824,
"mean_weights": [
0.09429103136062622,
0.09028280526399612,
0.09188265353441238,
0.0735517293214798,
0.06746295094490051,
0.060166649520397186,
0.21415996551513672,
0.15325874090194702,
0.15494348108768463
],
"sources": 9
},
{
"entropy_mean": 2.0139355659484863,
"mean_weights": [
0.0956040769815445,
0.09831400215625763,
0.0714857280254364,
0.0677608773112297,
0.05654233694076538,
0.05388312786817551,
0.24272149801254272,
0.17641445994377136,
0.137273907661438
],
"sources": 9
},
{
"entropy_mean": 2.0349321365356445,
"mean_weights": [
0.10256819427013397,
0.0938093364238739,
0.0912778377532959,
0.08394120633602142,
0.06161238253116608,
0.053358763456344604,
0.2261163890361786,
0.15197426080703735,
0.13534162938594818
],
"sources": 9
},
{
"entropy_mean": 2.046657085418701,
"mean_weights": [
0.08380690217018127,
0.1217331811785698,
0.10242430865764618,
0.09225839376449585,
0.06788390129804611,
0.06336672604084015,
0.2572976052761078,
0.11757771670818329,
0.09365126490592957
],
"sources": 9
},
{
"entropy_mean": 2.094015121459961,
"mean_weights": [
0.10284869372844696,
0.13057856261730194,
0.10152992606163025,
0.10079468786716461,
0.0796234980225563,
0.07438632845878601,
0.2018478661775589,
0.1111527606844902,
0.09723767638206482
],
"sources": 9
},
{
"entropy_mean": 2.057201385498047,
"mean_weights": [
0.09265108406543732,
0.11481083929538727,
0.09007447212934494,
0.08496075868606567,
0.06770835071802139,
0.05925413966178894,
0.24551761150360107,
0.15022093057632446,
0.09480181336402893
],
"sources": 9
}
],
"layer_input_rms_by_sublayer": [
0.05595625191926956,
0.047033343464136124,
0.0632273480296135,
0.060984931886196136,
0.11160287261009216,
0.09079429507255554,
0.12218338996171951,
0.08531255275011063,
0.12365243583917618,
0.0786692351102829,
0.0926642045378685,
0.08076046407222748,
0.093480683863163,
0.08904207497835159,
0.08749273419380188,
0.08403915911912918,
0.09927791357040405,
0.06012389436364174,
0.07820390164852142,
0.06841155141592026,
0.07669360190629959,
0.07131828367710114,
0.07840896397829056,
0.07358616590499878,
0.08920575678348541,
0.06597626209259033,
0.06850729882717133,
0.07235056161880493,
0.07440610229969025,
0.07261744141578674,
0.07867594808340073,
0.07352335751056671,
0.09058429300785065,
0.07520031183958054,
0.07234266400337219,
0.07378771901130676,
0.07771008461713791,
0.07715436071157455,
0.08416424691677094,
0.08649503439664841,
0.08947562426328659,
0.04556623473763466,
0.056279782205820084,
0.04546353965997696,
0.057985786348581314,
0.05167657509446144,
0.06238175556063652,
0.058827146887779236,
0.06515856087207794,
0.04798740893602371,
0.05041429027915001,
0.046526361256837845,
0.05083387345075607,
0.051699329167604446,
0.05600840970873833,
0.059578344225883484,
0.06128782033920288,
0.05090610682964325,
0.056301049888134,
0.05821637064218521,
0.06424970924854279,
0.06188955530524254,
0.06844019144773483,
0.06796324998140335
],
"loss_nats": 1.4371596574783325,
"loss_scale": 1.0,
"output_weights": {
"entropy_mean": 1.8435449600219727,
"mean_weights": [
0.1112452894449234,
0.06898629665374756,
0.05835210159420967,
0.042265649884939194,
0.041034020483493805,
0.04826357960700989,
0.10493715852499008,
0.31774088740348816,
0.20717504620552063
],
"sources": 9
},
"step": 2000,
"stream_state_rms_by_sublayer": [
0.04804428666830063,
0.09451264888048172,
0.18500657379627228,
0.2031717449426651,
0.22299833595752716,
0.22856928408145905,
0.2331935465335846,
0.2430499792098999,
0.0799286738038063,
0.08243431895971298,
0.09402555972337723,
0.09709372371435165,
0.10843002796173096,
0.11282631754875183,
0.12457633018493652,
0.13645486533641815,
0.04508088901638985,
0.05442482605576515,
0.0760042667388916,
0.0885990634560585,
0.10498681664466858,
0.12215005606412888,
0.13159531354904175,
0.15440259873867035,
0.0497574545443058,
0.060873307287693024,
0.07735268771648407,
0.09471380710601807,
0.10224150121212006,
0.1260664463043213,
0.13362281024456024,
0.163865864276886,
0.079553984105587,
0.08853588253259659,
0.10536550730466843,
0.12131853401660919,
0.14260481297969818,
0.15660560131072998,
0.19080305099487305,
0.2097783088684082,
0.03440643846988678,
0.04455159604549408,
0.060867179185152054,
0.07411240041255951,
0.08364810794591904,
0.10008715838193893,
0.11119203269481659,
0.13061410188674927,
0.025225287303328514,
0.04960336163640022,
0.05530283972620964,
0.07874798029661179,
0.08037404716014862,
0.11358420550823212,
0.11502843350172043,
0.1580832600593567,
0.0242468249052763,
0.06955241411924362,
0.07164724916219711,
0.13302887976169586,
0.1371813863515854,
0.22457914054393768,
0.23144805431365967,
0.32859006524086
]
},
{
"activation_grad_rms_by_block": [
4.638188693206757e-05,
5.534173033083789e-05,
4.9559435865376145e-05,
4.482309304876253e-05,
4.6912617108318955e-05,
4.315303158364259e-05,
3.9931001083459705e-05,
3.624077362474054e-05,
2.8622678655665368e-05,
2.680278521438595e-05,
2.541001776990015e-05,
2.4583727281424217e-05,
3.3810731110861525e-05,
3.128740718238987e-05,
2.9118271413608454e-05,
2.7763266189140268e-05,
3.4525284718256444e-05,
3.251996167819016e-05,
3.314791320008226e-05,
3.811598799075e-05,
0.00014694883429910988,
0.00011575810640351847,
9.27479995880276e-05,
7.475013262592256e-05,
9.14012489374727e-05,
7.025576633168384e-05,
4.882529901806265e-05,
4.170324973529205e-05,
3.074307824135758e-05,
2.38938500842778e-05,
2.2532458388013765e-05,
2.1951122107566334e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 4.529294619715074e-05,
"first_to_last_ratio": 1.0314184626645753,
"imbalance_abs_log_ratio": 0.03093500306067748,
"last_quartile_mean": 4.391325910546584e-05,
"mean": 4.717383586694268e-05,
"normalized": [
0.9832121149293677,
1.1731445898725168,
1.0505704052806353,
0.9501685038967237,
0.9944626347672784,
0.9147662213723499,
0.8464650022543867,
0.7682388544141363,
0.6067490194436969,
0.5681705700165066,
0.53864641920515,
0.5211305553096096,
0.7167263481864653,
0.6632364446817158,
0.6172546895643319,
0.588531029519173,
0.7318735922946267,
0.6893643707481226,
0.7026758072754231,
0.8079900073900919,
3.115049509935762,
2.4538624912763685,
1.9660898437352061,
1.5845676157597377,
1.937541165727475,
1.4892951790023068,
1.0350080319051866,
0.8840334683174624,
0.6516976556256888,
0.50650640646803,
0.47764736477161285,
0.4653240870528551
],
"population_cv": 0.6027314599912318
},
"activation_output_rms_by_block": [
0.16101159155368805,
0.3334391713142395,
0.3570481836795807,
0.37708213925361633,
0.11129721254110336,
0.12943366169929504,
0.1512180119752884,
0.1945808082818985,
0.07549218833446503,
0.11765842139720917,
0.16134388744831085,
0.21493762731552124,
0.08329613506793976,
0.12312135100364685,
0.16809259355068207,
0.23101076483726501,
0.11538279056549072,
0.16158254444599152,
0.20080390572547913,
0.27125677466392517,
0.0543946772813797,
0.09335283935070038,
0.12897421419620514,
0.17114612460136414,
0.06173322722315788,
0.09693212062120438,
0.13771499693393707,
0.18387214839458466,
0.07983565330505371,
0.15526357293128967,
0.26127612590789795,
0.39550676941871643
],
"activation_output_statistics": {
"first_quartile_mean": 0.22688884753733873,
"first_to_last_ratio": 1.3228372499356764,
"imbalance_abs_log_ratio": 0.2797788616441042,
"last_quartile_mean": 0.17151682684198022,
"mean": 0.17372163233812898,
"normalized": [
0.9268367409782202,
1.919387740182172,
2.055289136269615,
2.170611306021289,
0.6406640960204442,
0.7450635822219737,
0.8704616111421293,
1.1200724150644035,
0.4345583639666029,
0.677281348405711,
0.9287495476341928,
1.2372530952113672,
0.4794804996179952,
0.7087278040538177,
0.9675973641757485,
1.32977546738468,
0.6641820538556276,
0.930123337383164,
1.1558946518222766,
1.5614450026347633,
0.3131140120506511,
0.5373702635317168,
0.742418848247821,
0.9851745133746388,
0.3553571676266622,
0.5579738073870804,
0.7927337262517246,
1.0584297759573136,
0.4595608055861626,
0.8937492173058054,
1.5039930398498347,
2.276669658784396
],
"population_cv": 0.5173518438224235
},
"bits_per_byte": 1.9433676664104056,
"branch_output_rms_by_sublayer": [
0.08528144657611847,
0.13266965746879578,
0.2063225358724594,
0.045500293374061584,
0.07479097694158554,
0.018729008734226227,
0.053203217685222626,
0.03815881907939911,
0.1070452407002449,
0.02209169790148735,
0.03943018242716789,
0.01640710048377514,
0.041509829461574554,
0.02340921014547348,
0.06257687509059906,
0.05481670796871185,
0.061002109199762344,
0.037687551230192184,
0.04113767296075821,
0.04149388521909714,
0.040399398654699326,
0.048693906515836716,
0.03907898813486099,
0.06190986558794975,
0.06714949756860733,
0.04065176844596863,
0.04437125846743584,
0.049332331866025925,
0.0412614606320858,
0.0589165985584259,
0.046277787536382675,
0.08173932880163193,
0.10073095560073853,
0.043543290346860886,
0.05776162073016167,
0.05228494107723236,
0.05185067281126976,
0.04403071105480194,
0.07827401161193848,
0.0687810555100441,
0.04170985147356987,
0.030229199677705765,
0.041845910251140594,
0.04699210822582245,
0.038240060210227966,
0.06082639843225479,
0.046402644366025925,
0.07667754590511322,
0.030182205140590668,
0.050203632563352585,
0.025695007294416428,
0.05539015680551529,
0.024306239560246468,
0.06946423649787903,
0.017157377675175667,
0.07630576938390732,
0.030125634744763374,
0.07239200174808502,
0.01829664409160614,
0.09628339111804962,
0.024464696645736694,
0.12468792498111725,
0.032671861350536346,
0.15056027472019196
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.0002458031044227025,
0.00017276993155813553,
0.00010939491852011586,
0.000125258738441463,
0.00011215544691093282,
7.31087164016768e-05,
9.049562402296399e-05,
0.00013630654451635742,
9.25970647966133e-05,
7.580293890923953e-05,
7.957502622146584e-05,
8.776523713070475e-05,
0.00010318467043462154,
0.00010632361913360551,
0.00011315585331317908,
0.00012408116098017353,
0.00012849590289080713,
0.00014012462333747852,
0.00013373629199672077,
0.00019202792118308513,
0.0005640243510171648,
0.0006705621295490249,
0.000579819507581129,
0.0005486985299520294,
0.0004559472160958734,
0.00038131537611027277,
0.00030455316817913544,
0.000254991251417659,
0.00017923215935365346,
0.0001463414879582755,
0.00013146832622599984,
0.0001387805380572583
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.0001331616280992935,
"first_to_last_ratio": 0.5346167023449758,
"imbalance_abs_log_ratio": 0.6262052330699333,
"last_quartile_mean": 0.000249078690424766,
"mean": 0.00021243429301935995,
"normalized": [
1.1570782707870122,
0.8132864477883067,
0.5149588466403882,
0.5896352074852985,
0.5279535865742339,
0.34414743195447317,
0.4259934812630124,
0.6416409637964399,
0.4358856730733892,
0.3568300476907056,
0.37458653727914765,
0.4131406275478619,
0.4857251104237578,
0.5005012026185237,
0.5326628375526298,
0.584091952464876,
0.6048736344046712,
0.6596139509580424,
0.6295419166835406,
0.9039403123373536,
2.655053207279312,
3.156562530551101,
2.7294063464992813,
2.5829093888434693,
2.146297613325176,
1.7949803239890372,
1.433634672869791,
1.200329983419488,
0.8437063376453977,
0.6888788334421074,
0.6188658354422025,
0.6532868873699723
],
"population_cv": 0.7868568919954686
},
"depth_weights": [
{
"entropy_mean": 0.0,
"mean_weights": [
1.0
],
"sources": 1
},
{
"entropy_mean": 0.24809812009334564,
"mean_weights": [
0.9288002252578735,
0.07119979709386826
],
"sources": 2
},
{
"entropy_mean": 0.6067080497741699,
"mean_weights": [
0.5604246854782104,
0.43957534432411194
],
"sources": 2
},
{
"entropy_mean": 0.3847596049308777,
"mean_weights": [
0.8648147583007812,
0.13518527150154114
],
"sources": 2
},
{
"entropy_mean": 0.6843481659889221,
"mean_weights": [
0.49438929557800293,
0.5056107044219971
],
"sources": 2
},
{
"entropy_mean": 0.5728601813316345,
"mean_weights": [
0.6954652070999146,
0.3045347332954407
],
"sources": 2
},
{
"entropy_mean": 0.686518132686615,
"mean_weights": [
0.5263072848320007,
0.4736926555633545
],
"sources": 2
},
{
"entropy_mean": 0.5044870376586914,
"mean_weights": [
0.777843713760376,
0.2221563160419464
],
"sources": 2
},
{
"entropy_mean": 0.6724032163619995,
"mean_weights": [
0.5694063901901245,
0.4305936098098755
],
"sources": 2
},
{
"entropy_mean": 0.9718273282051086,
"mean_weights": [
0.5544795989990234,
0.20597517490386963,
0.23954525589942932
],
"sources": 3
},
{
"entropy_mean": 1.0856361389160156,
"mean_weights": [
0.3703838884830475,
0.26754841208457947,
0.36206769943237305
],
"sources": 3
},
{
"entropy_mean": 1.022249460220337,
"mean_weights": [
0.46441033482551575,
0.21309645473957062,
0.32249322533607483
],
"sources": 3
},
{
"entropy_mean": 1.0742855072021484,
"mean_weights": [
0.38770636916160583,
0.24731171131134033,
0.36498188972473145
],
"sources": 3
},
{
"entropy_mean": 1.0207322835922241,
"mean_weights": [
0.3404029607772827,
0.22699853777885437,
0.4325985312461853
],
"sources": 3
},
{
"entropy_mean": 1.0628657341003418,
"mean_weights": [
0.4176652133464813,
0.22035521268844604,
0.361979603767395
],
"sources": 3
},
{
"entropy_mean": 1.029329538345337,
"mean_weights": [
0.42153143882751465,
0.19259268045425415,
0.3858758807182312
],
"sources": 3
},
{
"entropy_mean": 1.0849454402923584,
"mean_weights": [
0.37970077991485596,
0.2838303744792938,
0.3364688754081726
],
"sources": 3
},
{
"entropy_mean": 1.3144235610961914,
"mean_weights": [
0.3205832540988922,
0.13048580288887024,
0.2396528720855713,
0.30927807092666626
],
"sources": 4
},
{
"entropy_mean": 1.3647149801254272,
"mean_weights": [
0.3188267648220062,
0.2379944771528244,
0.19448311626911163,
0.24869562685489655
],
"sources": 4
},
{
"entropy_mean": 1.2778496742248535,
"mean_weights": [
0.37896889448165894,
0.16353651881217957,
0.1984466314315796,
0.2590479254722595
],
"sources": 4
},
{
"entropy_mean": 1.3567583560943604,
"mean_weights": [
0.29770219326019287,
0.21492166817188263,
0.17857596278190613,
0.3088001608848572
],
"sources": 4
},
{
"entropy_mean": 1.2773678302764893,
"mean_weights": [
0.3856483995914459,
0.18032220005989075,
0.1928499937057495,
0.24117937684059143
],
"sources": 4
},
{
"entropy_mean": 1.3655540943145752,
"mean_weights": [
0.29819539189338684,
0.20063453912734985,
0.20929867029190063,
0.29187142848968506
],
"sources": 4
},
{
"entropy_mean": 1.2754871845245361,
"mean_weights": [
0.42651012539863586,
0.1713070571422577,
0.1726072132587433,
0.22957557439804077
],
"sources": 4
},
{
"entropy_mean": 1.3596348762512207,
"mean_weights": [
0.2750997543334961,
0.22746169567108154,
0.18702520430088043,
0.31041333079338074
],
"sources": 4
},
{
"entropy_mean": 1.5752925872802734,
"mean_weights": [
0.28211507201194763,
0.1675817370414734,
0.1759152114391327,
0.16564446687698364,
0.20874354243278503
],
"sources": 5
},
{
"entropy_mean": 1.5704567432403564,
"mean_weights": [
0.21000899374485016,
0.18079039454460144,
0.13544103503227234,
0.1749160885810852,
0.29884350299835205
],
"sources": 5
},
{
"entropy_mean": 1.579707384109497,
"mean_weights": [
0.21335941553115845,
0.18788449466228485,
0.20250459015369415,
0.1511598825454712,
0.24509160220623016
],
"sources": 5
},
{
"entropy_mean": 1.5663042068481445,
"mean_weights": [
0.21676909923553467,
0.19467836618423462,
0.13713446259498596,
0.15313217043876648,
0.29828590154647827
],
"sources": 5
},
{
"entropy_mean": 1.5384478569030762,
"mean_weights": [
0.21904908120632172,
0.15864285826683044,
0.18178997933864594,
0.14981383085250854,
0.29070425033569336
],
"sources": 5
},
{
"entropy_mean": 1.552668571472168,
"mean_weights": [
0.22710487246513367,
0.1922147125005722,
0.13047492504119873,
0.14686013758182526,
0.30334532260894775
],
"sources": 5
},
{
"entropy_mean": 1.5210195779800415,
"mean_weights": [
0.2812899947166443,
0.14511609077453613,
0.15909329056739807,
0.13197475671768188,
0.282525897026062
],
"sources": 5
},
{
"entropy_mean": 1.5376527309417725,
"mean_weights": [
0.23905543982982635,
0.21673071384429932,
0.0950920507311821,
0.16199833154678345,
0.2871234714984894
],
"sources": 5
},
{
"entropy_mean": 1.7209875583648682,
"mean_weights": [
0.1703755110502243,
0.14151470363140106,
0.17087554931640625,
0.12742051482200623,
0.25328749418258667,
0.1365262269973755
],
"sources": 6
},
{
"entropy_mean": 1.7200093269348145,
"mean_weights": [
0.18992580473423004,
0.16036251187324524,
0.08439569175243378,
0.11250905692577362,
0.1775888204574585,
0.27521812915802
],
"sources": 6
},
{
"entropy_mean": 1.753357172012329,
"mean_weights": [
0.15539541840553284,
0.1388990879058838,
0.15745995938777924,
0.12427005916833878,
0.17903099954128265,
0.24494445323944092
],
"sources": 6
},
{
"entropy_mean": 1.7199265956878662,
"mean_weights": [
0.19207912683486938,
0.17904852330684662,
0.08410340547561646,
0.11219910532236099,
0.15951165556907654,
0.2730581760406494
],
"sources": 6
},
{
"entropy_mean": 1.6756346225738525,
"mean_weights": [
0.15863081812858582,
0.11242729425430298,
0.15970338881015778,
0.09774985909461975,
0.15905189514160156,
0.3124367296695709
],
"sources": 6
},
{
"entropy_mean": 1.7271867990493774,
"mean_weights": [
0.19560003280639648,
0.18333327770233154,
0.08475087583065033,
0.11598999053239822,
0.15889793634414673,
0.2614278793334961
],
"sources": 6
},
{
"entropy_mean": 1.6916961669921875,
"mean_weights": [
0.19065657258033752,
0.12983909249305725,
0.11857748031616211,
0.09883742034435272,
0.1529499739408493,
0.3091394901275635
],
"sources": 6
},
{
"entropy_mean": 1.7504222393035889,
"mean_weights": [
0.20809361338615417,
0.19103100895881653,
0.10743936896324158,
0.1194620355963707,
0.15873855352401733,
0.2152354121208191
],
"sources": 6
},
{
"entropy_mean": 1.5379457473754883,
"mean_weights": [
0.09571269154548645,
0.07078731805086136,
0.0824701339006424,
0.060012347996234894,
0.08145605027675629,
0.09452696144580841,
0.5150344967842102
],
"sources": 7
},
{
"entropy_mean": 1.7319592237472534,
"mean_weights": [
0.13678354024887085,
0.11740625649690628,
0.06668869405984879,
0.07360264658927917,
0.09287553280591965,
0.12232495844364166,
0.390318363904953
],
"sources": 7
},
{
"entropy_mean": 1.4663677215576172,
"mean_weights": [
0.09485072642564774,
0.07167200744152069,
0.0841417908668518,
0.05784624442458153,
0.07052354514598846,
0.0724712610244751,
0.5484944581985474
],
"sources": 7
},
{
"entropy_mean": 1.749481201171875,
"mean_weights": [
0.14530614018440247,
0.12230368703603745,
0.0660901889204979,
0.07630561292171478,
0.09453228116035461,
0.12125740200281143,
0.3742046654224396
],
"sources": 7
},
{
"entropy_mean": 1.472255825996399,
"mean_weights": [
0.10688216984272003,
0.07465942203998566,
0.08327433466911316,
0.054589346051216125,
0.0637175589799881,
0.07299347221851349,
0.5438836812973022
],
"sources": 7
},
{
"entropy_mean": 1.7924723625183105,
"mean_weights": [
0.15522946417331696,
0.13724805414676666,
0.06889298558235168,
0.08198928833007812,
0.09816981852054596,
0.123870849609375,
0.3345995247364044
],
"sources": 7
},
{
"entropy_mean": 1.5755163431167603,
"mean_weights": [
0.12274101376533508,
0.08900541067123413,
0.08551642298698425,
0.06285787373781204,
0.07429315149784088,
0.07904880493879318,
0.4865373373031616
],
"sources": 7
},
{
"entropy_mean": 1.8390922546386719,
"mean_weights": [
0.15884171426296234,
0.14726482331752777,
0.08053631335496902,
0.09097073972225189,
0.10738551616668701,
0.12150256335735321,
0.29349833726882935
],
"sources": 7
},
{
"entropy_mean": 1.6550344228744507,
"mean_weights": [
0.0844201147556305,
0.06695197522640228,
0.061310961842536926,
0.0447731651365757,
0.04748314991593361,
0.045562684535980225,
0.2905467748641968,
0.3589511513710022
],
"sources": 8
},
{
"entropy_mean": 1.8679088354110718,
"mean_weights": [
0.11674155294895172,
0.1088850349187851,
0.05779466778039932,
0.06273136287927628,
0.07066237181425095,
0.08129343390464783,
0.17753683030605316,
0.32435476779937744
],
"sources": 8
},
{
"entropy_mean": 1.5923268795013428,
"mean_weights": [
0.08656083047389984,
0.05770102143287659,
0.05105586349964142,
0.039375390857458115,
0.04410157352685928,
0.04500962793827057,
0.2357216477394104,
0.4404740333557129
],
"sources": 8
},
{
"entropy_mean": 1.8469831943511963,
"mean_weights": [
0.10824646055698395,
0.09116974472999573,
0.0701109915971756,
0.0629766583442688,
0.07049155235290527,
0.07297641038894653,
0.18795298039913177,
0.33607521653175354
],
"sources": 8
},
{
"entropy_mean": 1.6590619087219238,
"mean_weights": [
0.0944472923874855,
0.0654030293226242,
0.060137998312711716,
0.0449567511677742,
0.044405773282051086,
0.05244143307209015,
0.2202538251876831,
0.4179539084434509
],
"sources": 8
},
{
"entropy_mean": 1.884438157081604,
"mean_weights": [
0.11613816022872925,
0.09775452315807343,
0.08394932001829147,
0.07057420164346695,
0.06920696794986725,
0.06340071558952332,
0.21605131030082703,
0.2829247713088989
],
"sources": 8
},
{
"entropy_mean": 1.7424441576004028,
"mean_weights": [
0.08763163536787033,
0.07771545648574829,
0.07502467930316925,
0.06006643921136856,
0.06314834952354431,
0.054570816457271576,
0.3030073344707489,
0.278835266828537
],
"sources": 8
},
{
"entropy_mean": 1.8953055143356323,
"mean_weights": [
0.1135263741016388,
0.10274903476238251,
0.0858360230922699,
0.08475605398416519,
0.08078759908676147,
0.07204630225896835,
0.2779633104801178,
0.18233531713485718
],
"sources": 8
},
{
"entropy_mean": 1.801485300064087,
"mean_weights": [
0.07824120670557022,
0.06023171544075012,
0.054100219160318375,
0.039871927350759506,
0.04207881540060043,
0.045095670968294144,
0.2047339826822281,
0.15412676334381104,
0.321519672870636
],
"sources": 9
},
{
"entropy_mean": 2.008213520050049,
"mean_weights": [
0.09438058733940125,
0.07153058052062988,
0.07423336803913116,
0.06403130292892456,
0.0645662397146225,
0.05577655881643295,
0.21496367454528809,
0.1609123945236206,
0.1996053010225296
],
"sources": 9
},
{
"entropy_mean": 1.8966764211654663,
"mean_weights": [
0.09610478579998016,
0.07267358899116516,
0.060228414833545685,
0.046177469193935394,
0.04206448048353195,
0.04096413776278496,
0.21562573313713074,
0.21000833809375763,
0.21615305542945862
],
"sources": 9
},
{
"entropy_mean": 1.981720209121704,
"mean_weights": [
0.10501760244369507,
0.06946457922458649,
0.07816961407661438,
0.07385893911123276,
0.06364607810974121,
0.05328303575515747,
0.23084522783756256,
0.15472577512264252,
0.17098914086818695
],
"sources": 9
},
{
"entropy_mean": 2.0028607845306396,
"mean_weights": [
0.0842779204249382,
0.10298711061477661,
0.09207960218191147,
0.07771934568881989,
0.06265503168106079,
0.0552704893052578,
0.2620389461517334,
0.1382768750190735,
0.12469469010829926
],
"sources": 9
},
{
"entropy_mean": 2.0655057430267334,
"mean_weights": [
0.10472239553928375,
0.11792458593845367,
0.0928182601928711,
0.10183010995388031,
0.08287280797958374,
0.06570011377334595,
0.21871674060821533,
0.11616810411214828,
0.09924688935279846
],
"sources": 9
},
{
"entropy_mean": 2.0258567333221436,
"mean_weights": [
0.09674200415611267,
0.09482249617576599,
0.08996410667896271,
0.06836152076721191,
0.06293784081935883,
0.05016916245222092,
0.22868779301643372,
0.1848844438791275,
0.12343062460422516
],
"sources": 9
}
],
"layer_input_rms_by_sublayer": [
0.059412047266960144,
0.056020382791757584,
0.08935670554637909,
0.0742802768945694,
0.17747226357460022,
0.1285802572965622,
0.17946027219295502,
0.10598119348287582,
0.17260213196277618,
0.10133238881826401,
0.12674333155155182,
0.10985831171274185,
0.12412907183170319,
0.12611564993858337,
0.11767174303531647,
0.1125042662024498,
0.13582420349121094,
0.07848236709833145,
0.10855290293693542,
0.0899009183049202,
0.10545514523983002,
0.09790628403425217,
0.11005345731973648,
0.09713682532310486,
0.13049271702766418,
0.09147685021162033,
0.09666567295789719,
0.10080859810113907,
0.10358717292547226,
0.09896469116210938,
0.11041814833879471,
0.10007023066282272,
0.13209649920463562,
0.10974904894828796,
0.10239993035793304,
0.10291008651256561,
0.11153820157051086,
0.10674569010734558,
0.11912636458873749,
0.12148815393447876,
0.127323716878891,
0.055056143552064896,
0.07468672841787338,
0.05981758236885071,
0.07975661009550095,
0.07101839780807495,
0.08746611326932907,
0.08339047431945801,
0.09389390796422958,
0.06143829599022865,
0.06575146317481995,
0.06018613278865814,
0.0681111291050911,
0.06816941499710083,
0.07568862289190292,
0.08121132105588913,
0.08424623310565948,
0.05916989967226982,
0.0719028189778328,
0.07458337396383286,
0.08439797908067703,
0.08333029597997665,
0.08961156010627747,
0.09104464948177338
],
"loss_nats": 1.347039818763733,
"loss_scale": 1.0,
"output_weights": {
"entropy_mean": 1.7394249439239502,
"mean_weights": [
0.10602211207151413,
0.05902107432484627,
0.049884356558322906,
0.034865088760852814,
0.03282830864191055,
0.043302543461322784,
0.08693542331457138,
0.3314756751060486,
0.2556654214859009
],
"sources": 9
},
"step": 4000,
"stream_state_rms_by_sublayer": [
0.08528144657611847,
0.16101159155368805,
0.3133305609226227,
0.3334391713142395,
0.3510810136795044,
0.3570481836795807,
0.36267346143722534,
0.37708213925361633,
0.1070452407002449,
0.11129721254110336,
0.1260577291250229,
0.12943366169929504,
0.14502130448818207,
0.1512180119752884,
0.17329032719135284,
0.1945808082818985,
0.061002109199762344,
0.07549218833446503,
0.0987050011754036,
0.11765842139720917,
0.13416865468025208,
0.16134388744831085,
0.17569874227046967,
0.21493762731552124,
0.06714949756860733,
0.08329613506793976,
0.09735818952322006,
0.12312135100364685,
0.13312570750713348,
0.16809259355068207,
0.17941679060459137,
0.23101076483726501,
0.10073095560073853,
0.11538279056549072,
0.13780468702316284,
0.16158254444599152,
0.17965665459632874,
0.20080390572547913,
0.24453195929527283,
0.27125677466392517,
0.04170985147356987,
0.0543946772813797,
0.07352267950773239,
0.09335283935070038,
0.10462787002325058,
0.12897421419620514,
0.14256280660629272,
0.17114612460136414,
0.030182205140590668,
0.06173322722315788,
0.06915032118558884,
0.09693212062120438,
0.10093247890472412,
0.13771499693393707,
0.1392749696969986,
0.18387214839458466,
0.030125634744763374,
0.07983565330505371,
0.08267941325902939,
0.15526357293128967,
0.15948866307735443,
0.26127612590789795,
0.2670789062976837,
0.39550676941871643
]
},
{
"activation_grad_rms_by_block": [
5.980865898891352e-05,
6.95241178618744e-05,
6.203598604770377e-05,
5.727907409891486e-05,
7.378464943030849e-05,
6.77111092954874e-05,
6.276105705183e-05,
5.631716703646816e-05,
4.664637162932195e-05,
4.3178879423066974e-05,
4.080804137629457e-05,
3.917843787348829e-05,
5.551072536036372e-05,
5.1256221922812983e-05,
4.731794251711108e-05,
4.47981983597856e-05,
5.737639730796218e-05,
5.2802144637098536e-05,
5.3134874178795144e-05,
6.219849456101656e-05,
0.0002451144973747432,
0.00019094848539680243,
0.00015029680798761547,
0.00011999815615126863,
0.0001724466128507629,
0.00012853756197728217,
8.318446634802967e-05,
6.64951658109203e-05,
6.248620775295421e-05,
4.157395596848801e-05,
3.7306774174794555e-05,
3.4809250792022794e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 6.365272747643758e-05,
"first_to_last_ratio": 0.8123633197064085,
"imbalance_abs_log_ratio": 0.2078076008289402,
"last_quartile_mean": 7.835499945940683e-05,
"mean": 7.614457786075945e-05,
"normalized": [
0.7854618236676242,
0.9130540849409994,
0.8147131127464502,
0.7522410092502885,
0.9690072688462938,
0.8892440039434748,
0.8242354060534288,
0.7396083689563772,
0.6126026690255093,
0.5670644008562938,
0.5359283946772616,
0.5145269561429746,
0.7290174418180176,
0.6731434248219464,
0.6214223500409742,
0.5883307730946398,
0.7535191463387267,
0.6934458909688136,
0.6978155985835179,
0.8168473226649812,
3.219066994145898,
2.5077095541323673,
1.973834673592298,
1.5759251613516239,
2.2647260999477155,
1.6880724220748888,
1.092454232265147,
0.8732751258075867,
0.8206258345435784,
0.5459870832104624,
0.48994656248557816,
0.45714680900426247
],
"population_cv": 0.6420341358971389
},
"activation_output_rms_by_block": [
0.16915453970432281,
0.374446302652359,
0.39966297149658203,
0.4194087088108063,
0.09892481565475464,
0.11664740741252899,
0.13951337337493896,
0.18593810498714447,
0.06607726216316223,
0.10425819456577301,
0.14551182091236115,
0.19865906238555908,
0.06484340131282806,
0.10430748760700226,
0.14720691740512848,
0.20439592003822327,
0.10052042454481125,
0.14085541665554047,
0.1745898574590683,
0.2420409768819809,
0.049456026405096054,
0.08509506285190582,
0.11783231794834137,
0.15649692714214325,
0.05533855780959129,
0.08507220447063446,
0.11942370235919952,
0.1559232622385025,
0.06628894805908203,
0.12384890764951706,
0.19928614795207977,
0.30097496509552
],
"activation_output_statistics": {
"first_quartile_mean": 0.23796202801167965,
"first_to_last_ratio": 1.7210004980371292,
"imbalance_abs_log_ratio": 0.542905806617582,
"last_quartile_mean": 0.13826958695426583,
"mean": 0.15974999987520278,
"normalized": [
1.0588703589137207,
2.3439518181213,
2.5018026404306735,
2.625406629974646,
0.6192476728139908,
0.7301872144203714,
0.8733231516990753,
1.1639318005158055,
0.41362918444307983,
0.6526334563206251,
0.9108721190988135,
1.2435622068278698,
0.40590548584340497,
0.6529420199592338,
0.9214830517691831,
1.2794736788600816,
0.6292358348878756,
0.8817240486108118,
1.0928942572485663,
1.5151234871428112,
0.30958389010160414,
0.5326764501933169,
0.737604494775539,
0.9796364773984299,
0.3464072479049888,
0.5325333617345424,
0.7475662125351719,
0.9760454607844148,
0.41495429177381643,
0.7752670281456541,
1.247487625087717,
1.8840373416628649
],
"population_cv": 0.591409330949146
},
"bits_per_byte": 1.8740129107275643,
"branch_output_rms_by_sublayer": [
0.0933690071105957,
0.13583384454250336,
0.24537226557731628,
0.0415157787501812,
0.07698935270309448,
0.018140379339456558,
0.0523640513420105,
0.039518751204013824,
0.09380833059549332,
0.022877270355820656,
0.03647654876112938,
0.015532293356955051,
0.039120711386203766,
0.02314678393304348,
0.06371714919805527,
0.05432315170764923,
0.05156813934445381,
0.03743875026702881,
0.03728010877966881,
0.04006272554397583,
0.03983496502041817,
0.045583780854940414,
0.040284499526023865,
0.05995568633079529,
0.046871740370988846,
0.038651175796985626,
0.04314620420336723,
0.04559725895524025,
0.04033593460917473,
0.05632574483752251,
0.04471700266003609,
0.07952121645212173,
0.08397205173969269,
0.042337141931056976,
0.0528092160820961,
0.05044656619429588,
0.04443390294909477,
0.04418695345520973,
0.07560808956623077,
0.06310147047042847,
0.03708760067820549,
0.029010163620114326,
0.03599191829562187,
0.04586305841803551,
0.033304885029792786,
0.06033914536237717,
0.04061548039317131,
0.07656749337911606,
0.025487789884209633,
0.045884523540735245,
0.021565141156315804,
0.051604680716991425,
0.02342291921377182,
0.06502768397331238,
0.017244853079319,
0.06792232394218445,
0.024166326969861984,
0.060306549072265625,
0.016819356009364128,
0.08025261014699936,
0.021590016782283783,
0.09663844108581543,
0.026201345026493073,
0.12230555713176727
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.0003065916678101079,
0.00023586292031870838,
0.0001322871105193671,
0.00016391009158296018,
0.0001640597280926294,
0.00011452252186811768,
0.00012670462239247783,
0.00021513822087305528,
0.00015124888511926465,
0.00012797922905715274,
0.0001317390201682035,
0.00014536342393434786,
0.00017054401801644127,
0.00018163687748006386,
0.00019727429589294548,
0.0002118946661868627,
0.0002209032617907253,
0.0002444386191807451,
0.00023590577825104205,
0.00032383840230939837,
0.0010316902394974224,
0.0010821684520459678,
0.0009765210521856827,
0.0008761640307174725,
0.0008326776627851861,
0.0006767392019523396,
0.00052552098450176,
0.0004066875433884613,
0.0003364842674135882,
0.00025378360531775436,
0.000205309661176443,
0.00020659565641729775
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.00018238461043217796,
"first_to_last_ratio": 0.42368241008054586,
"imbalance_abs_log_ratio": 0.8587711372443304,
"last_quartile_mean": 0.0004304748228691038,
"mean": 0.00035038080369512474,
"normalized": [
0.875024158220954,
0.6731616510701846,
0.37755239192405493,
0.46780556998089007,
0.46823263821091626,
0.3268515873596963,
0.3616197607181892,
0.6140125788975943,
0.43167001024082974,
0.3652575361078021,
0.37598812143496585,
0.4148726825252455,
0.486739045683667,
0.5183984840622454,
0.5630282647122382,
0.6047553517550511,
0.630466222638552,
0.69763701836085,
0.6732839692219831,
0.9242469875466649,
2.944482770223686,
3.0885494885375917,
2.7870278333951726,
2.500605117282182,
2.3764933866346176,
1.9314391508194257,
1.499856667259172,
1.160701554136312,
0.9603387624693381,
0.7243079604856947,
0.5859614996348036,
0.5896317784494322
],
"population_cv": 0.8248093894360681
},
"depth_weights": [
{
"entropy_mean": 0.0,
"mean_weights": [
1.0
],
"sources": 1
},
{
"entropy_mean": 0.22886337339878082,
"mean_weights": [
0.9364937543869019,
0.06350618600845337
],
"sources": 2
},
{
"entropy_mean": 0.603729784488678,
"mean_weights": [
0.5866844654083252,
0.4133155643939972
],
"sources": 2
},
{
"entropy_mean": 0.3967982828617096,
"mean_weights": [
0.858893632888794,
0.14110638201236725
],
"sources": 2
},
{
"entropy_mean": 0.6852496862411499,
"mean_weights": [
0.48102498054504395,
0.518975019454956
],
"sources": 2
},
{
"entropy_mean": 0.5890734195709229,
"mean_weights": [
0.6876576542854309,
0.3123423457145691
],
"sources": 2
},
{
"entropy_mean": 0.6873323321342468,
"mean_weights": [
0.5168514847755432,
0.4831485152244568
],
"sources": 2
},
{
"entropy_mean": 0.487301766872406,
"mean_weights": [
0.7939615249633789,
0.2060384750366211
],
"sources": 2
},
{
"entropy_mean": 0.674697756767273,
"mean_weights": [
0.5704737305641174,
0.42952626943588257
],
"sources": 2
},
{
"entropy_mean": 0.9691112041473389,
"mean_weights": [
0.5506306886672974,
0.18125343322753906,
0.2681158185005188
],
"sources": 3
},
{
"entropy_mean": 1.0745112895965576,
"mean_weights": [
0.342629998922348,
0.24457570910453796,
0.412794291973114
],
"sources": 3
},
{
"entropy_mean": 1.0074045658111572,
"mean_weights": [
0.4399106502532959,
0.18251104652881622,
0.3775783181190491
],
"sources": 3
},
{
"entropy_mean": 1.060497522354126,
"mean_weights": [
0.3786153495311737,
0.2188822329044342,
0.4025024175643921
],
"sources": 3
},
{
"entropy_mean": 0.9941019415855408,
"mean_weights": [
0.32296958565711975,
0.19686520099639893,
0.4801652133464813
],
"sources": 3
},
{
"entropy_mean": 1.0503522157669067,
"mean_weights": [
0.40130966901779175,
0.19853851199150085,
0.4001518487930298
],
"sources": 3
},
{
"entropy_mean": 1.0129402875900269,
"mean_weights": [
0.4093267321586609,
0.16906049847602844,
0.42161279916763306
],
"sources": 3
},
{
"entropy_mean": 1.0774019956588745,
"mean_weights": [
0.38510820269584656,
0.25884830951690674,
0.3560434877872467
],
"sources": 3
},
{
"entropy_mean": 1.2740377187728882,
"mean_weights": [
0.29550763964653015,
0.09541245549917221,
0.24569378793239594,
0.3633860945701599
],
"sources": 4
},
{
"entropy_mean": 1.360302448272705,
"mean_weights": [
0.3109527826309204,
0.20984822511672974,
0.1935039907693863,
0.28569501638412476
],
"sources": 4
},
{
"entropy_mean": 1.2722704410552979,
"mean_weights": [
0.350364625453949,
0.12631991505622864,
0.22418734431266785,
0.29912811517715454
],
"sources": 4
},
{
"entropy_mean": 1.3558036088943481,
"mean_weights": [
0.2989768385887146,
0.1941717267036438,
0.1926535815000534,
0.314197838306427
],
"sources": 4
},
{
"entropy_mean": 1.2696614265441895,
"mean_weights": [
0.3667730689048767,
0.15310388803482056,
0.19384080171585083,
0.2862822115421295
],
"sources": 4
},
{
"entropy_mean": 1.3606936931610107,
"mean_weights": [
0.2982274293899536,
0.18611131608486176,
0.21284276247024536,
0.3028184771537781
],
"sources": 4
},
{
"entropy_mean": 1.269316554069519,
"mean_weights": [
0.4081852436065674,
0.14420126378536224,
0.18228203058242798,
0.2653314471244812
],
"sources": 4
},
{
"entropy_mean": 1.3636136054992676,
"mean_weights": [
0.27427661418914795,
0.21341991424560547,
0.19867971539497375,
0.3136237859725952
],
"sources": 4
},
{
"entropy_mean": 1.570773959159851,
"mean_weights": [
0.26587405800819397,
0.13419361412525177,
0.1907704770565033,
0.17973750829696655,
0.2294243425130844
],
"sources": 5
},
{
"entropy_mean": 1.5560038089752197,
"mean_weights": [
0.20477107167243958,
0.15721666812896729,
0.13811248540878296,
0.1752161681652069,
0.32468360662460327
],
"sources": 5
},
{
"entropy_mean": 1.5659973621368408,
"mean_weights": [
0.20531430840492249,
0.15986156463623047,
0.21557101607322693,
0.15222224593162537,
0.26703086495399475
],
"sources": 5
},
{
"entropy_mean": 1.5552496910095215,
"mean_weights": [
0.2162160575389862,
0.17982497811317444,
0.1307673156261444,
0.1554446816444397,
0.31774696707725525
],
"sources": 5
},
{
"entropy_mean": 1.5068919658660889,
"mean_weights": [
0.20646695792675018,
0.1197461187839508,
0.19215428829193115,
0.15179987251758575,
0.3298327624797821
],
"sources": 5
},
{
"entropy_mean": 1.5435223579406738,
"mean_weights": [
0.22410975396633148,
0.16986176371574402,
0.13080401718616486,
0.15279728174209595,
0.3224271535873413
],
"sources": 5
},
{
"entropy_mean": 1.4949920177459717,
"mean_weights": [
0.27174079418182373,
0.12178683280944824,
0.1529826819896698,
0.1350194811820984,
0.31847020983695984
],
"sources": 5
},
{
"entropy_mean": 1.544264793395996,
"mean_weights": [
0.24016115069389343,
0.20730282366275787,
0.09692574292421341,
0.1695065051317215,
0.2861037850379944
],
"sources": 5
},
{
"entropy_mean": 1.686682939529419,
"mean_weights": [
0.16755425930023193,
0.10962115228176117,
0.1783454418182373,
0.13310004770755768,
0.2885603904724121,
0.12281867861747742
],
"sources": 6
},
{
"entropy_mean": 1.7070834636688232,
"mean_weights": [
0.181177020072937,
0.14557164907455444,
0.08318156003952026,
0.11424209922552109,
0.17698991298675537,
0.2988377511501312
],
"sources": 6
},
{
"entropy_mean": 1.7419235706329346,
"mean_weights": [
0.1532188206911087,
0.11420352011919022,
0.1659742295742035,
0.1250365972518921,
0.18140509724617004,
0.26016172766685486
],
"sources": 6
},
{
"entropy_mean": 1.7127439975738525,
"mean_weights": [
0.18731769919395447,
0.16715484857559204,
0.08496320992708206,
0.11362075805664062,
0.1561521738767624,
0.2907913327217102
],
"sources": 6
},
{
"entropy_mean": 1.6076834201812744,
"mean_weights": [
0.15032559633255005,
0.08624927699565887,
0.15736976265907288,
0.08584291487932205,
0.14632999897003174,
0.3738824725151062
],
"sources": 6
},
{
"entropy_mean": 1.709290862083435,
"mean_weights": [
0.18991519510746002,
0.17091524600982666,
0.07822999358177185,
0.116765096783638,
0.1540035903453827,
0.2901708781719208
],
"sources": 6
},
{
"entropy_mean": 1.6690887212753296,
"mean_weights": [
0.19029225409030914,
0.11002412438392639,
0.11471512168645859,
0.0999990925192833,
0.1507222205400467,
0.3342471718788147
],
"sources": 6
},
{
"entropy_mean": 1.7458693981170654,
"mean_weights": [
0.20528709888458252,
0.1722964346408844,
0.10435444861650467,
0.12075909972190857,
0.16116347908973694,
0.2361394464969635
],
"sources": 6
},
{
"entropy_mean": 1.4612125158309937,
"mean_weights": [
0.08934466540813446,
0.054379623383283615,
0.07567165046930313,
0.05758609622716904,
0.07884950935840607,
0.08982288837432861,
0.5543456077575684
],
"sources": 7
},
{
"entropy_mean": 1.7241814136505127,
"mean_weights": [
0.13360008597373962,
0.10609637945890427,
0.06363911926746368,
0.07426095008850098,
0.09383077174425125,
0.13373199105262756,
0.39484071731567383
],
"sources": 7
},
{
"entropy_mean": 1.3979300260543823,
"mean_weights": [
0.08959861099720001,
0.05414731800556183,
0.07668881118297577,
0.05603286623954773,
0.06979228556156158,
0.07073447853326797,
0.5830056071281433
],
"sources": 7
},
{
"entropy_mean": 1.7459560632705688,
"mean_weights": [
0.14318783581256866,
0.1094653457403183,
0.06349228322505951,
0.07859592139720917,
0.09567482024431229,
0.13266830146312714,
0.37691545486450195
],
"sources": 7
},
{
"entropy_mean": 1.4055393934249878,
"mean_weights": [
0.10154954344034195,
0.05619275942444801,
0.07791627943515778,
0.053915880620479584,
0.06398355215787888,
0.06915920972824097,
0.5772827863693237
],
"sources": 7
},
{
"entropy_mean": 1.7853620052337646,
"mean_weights": [
0.15284112095832825,
0.12206703424453735,
0.06508444249629974,
0.08347312361001968,
0.09811970591545105,
0.13767032325267792,
0.3407442569732666
],
"sources": 7
},
{
"entropy_mean": 1.509842872619629,
"mean_weights": [
0.11788781732320786,
0.06822571903467178,
0.08556114137172699,
0.06067090481519699,
0.07228855788707733,
0.07258446514606476,
0.5227814316749573
],
"sources": 7
},
{
"entropy_mean": 1.839367389678955,
"mean_weights": [
0.15906387567520142,
0.13139380514621735,
0.07717745006084442,
0.09607303887605667,
0.11078508198261261,
0.12967455387115479,
0.29583218693733215
],
"sources": 7
},
{
"entropy_mean": 1.5841834545135498,
"mean_weights": [
0.07896879315376282,
0.048752427101135254,
0.055707160383462906,
0.04139826446771622,
0.04449019208550453,
0.04285799711942673,
0.2801477909088135,
0.40767738223075867
],
"sources": 8
},
{
"entropy_mean": 1.8125314712524414,
"mean_weights": [
0.10889512300491333,
0.08656036853790283,
0.048123255372047424,
0.06561170518398285,
0.06917259097099304,
0.08286689221858978,
0.16869932413101196,
0.3700707256793976
],
"sources": 8
},
{
"entropy_mean": 1.50089693069458,
"mean_weights": [
0.07974998652935028,
0.04166398197412491,
0.04607272148132324,
0.03586764261126518,
0.04223340004682541,
0.04095568507909775,
0.22249138355255127,
0.49096518754959106
],
"sources": 8
},
{
"entropy_mean": 1.8059929609298706,
"mean_weights": [
0.10711923241615295,
0.07651708275079727,
0.058284737169742584,
0.06609588116407394,
0.07139427959918976,
0.07493151724338531,
0.17425751686096191,
0.37139976024627686
],
"sources": 8
},
{
"entropy_mean": 1.5622732639312744,
"mean_weights": [
0.08775827288627625,
0.047765426337718964,
0.053928621113300323,
0.04293380677700043,
0.04174989461898804,
0.04734005779027939,
0.2016349583864212,
0.4768889844417572
],
"sources": 8
},
{
"entropy_mean": 1.8757455348968506,
"mean_weights": [
0.11563669890165329,
0.08108697831630707,
0.07280858606100082,
0.07553227990865707,
0.07606013864278793,
0.06935927271842957,
0.19856664538383484,
0.3109493851661682
],
"sources": 8
},
{
"entropy_mean": 1.6924852132797241,
"mean_weights": [
0.08638577163219452,
0.059912849217653275,
0.06609142571687698,
0.056745607405900955,
0.06277773529291153,
0.05352145433425903,
0.2830069661140442,
0.3315581977367401
],
"sources": 8
},
{
"entropy_mean": 1.8980522155761719,
"mean_weights": [
0.11603894829750061,
0.08804173022508621,
0.07975024729967117,
0.09026999771595001,
0.08430293202400208,
0.07433462888002396,
0.26602888107299805,
0.2012326419353485
],
"sources": 8
},
{
"entropy_mean": 1.7512003183364868,
"mean_weights": [
0.07744461297988892,
0.046086352318525314,
0.05025433003902435,
0.037484876811504364,
0.04005061089992523,
0.041623204946517944,
0.198423832654953,
0.17380405962467194,
0.33482813835144043
],
"sources": 9
},
{
"entropy_mean": 2.0160694122314453,
"mean_weights": [
0.0938527062535286,
0.06208505481481552,
0.06560768187046051,
0.07029052078723907,
0.06935097277164459,
0.056832704693078995,
0.1942581683397293,
0.1706070452928543,
0.21711516380310059
],
"sources": 9
},
{
"entropy_mean": 1.7837426662445068,
"mean_weights": [
0.08836328238248825,
0.04953558370471001,
0.04813014343380928,
0.0373263955116272,
0.037015400826931,
0.03390160948038101,
0.1840541511774063,
0.23140482604503632,
0.29026859998703003
],
"sources": 9
},
{
"entropy_mean": 1.9973840713500977,
"mean_weights": [
0.10261696577072144,
0.06039813905954361,
0.06671924889087677,
0.08377155661582947,
0.07002593576908112,
0.05768897384405136,
0.20831166207790375,
0.1594475358724594,
0.19101998209953308
],
"sources": 9
},
{
"entropy_mean": 1.969987154006958,
"mean_weights": [
0.08120635896921158,
0.07894632965326309,
0.08615730702877045,
0.07609778642654419,
0.058728985488414764,
0.05076971277594566,
0.24503237009048462,
0.1544169783592224,
0.16864418983459473
],
"sources": 9
},
{
"entropy_mean": 2.071648359298706,
"mean_weights": [
0.10569609701633453,
0.09922641515731812,
0.08180510997772217,
0.10646798461675644,
0.0826798751950264,
0.06615597754716873,
0.20955613255500793,
0.13320739567279816,
0.11520501971244812
],
"sources": 9
},
{
"entropy_mean": 1.9781701564788818,
"mean_weights": [
0.09446848928928375,
0.07056106626987457,
0.08033449202775955,
0.0658978745341301,
0.058234505355358124,
0.04316519945859909,
0.2036968618631363,
0.22251945734024048,
0.16112206876277924
],
"sources": 9
}
],
"layer_input_rms_by_sublayer": [
0.0626930519938469,
0.059514161199331284,
0.09030644595623016,
0.08157797157764435,
0.20229491591453552,
0.14292451739311218,
0.20300470292568207,
0.10997853428125381,
0.19049881398677826,
0.0991024449467659,
0.12695962190628052,
0.1065012738108635,
0.12132706493139267,
0.12155740708112717,
0.11585728824138641,
0.1095810979604721,
0.13427142798900604,
0.06964563578367233,
0.10485932976007462,
0.08311934769153595,
0.10419609397649765,
0.09363401681184769,
0.10852954536676407,
0.0941167026758194,
0.1286265105009079,
0.08564268052577972,
0.09096142649650574,
0.09493856877088547,
0.10080574452877045,
0.09063401818275452,
0.10566592961549759,
0.09473779052495956,
0.1287592649459839,
0.10505300015211105,
0.09650235623121262,
0.09355910867452621,
0.1065765842795372,
0.09618847072124481,
0.11446042358875275,
0.11416846513748169,
0.12395420670509338,
0.047205954790115356,
0.07221997529268265,
0.053352247923612595,
0.07677769660949707,
0.06512638926506042,
0.08416415005922318,
0.07829347252845764,
0.09094838052988052,
0.05368548259139061,
0.059439923614263535,
0.05310077592730522,
0.062182098627090454,
0.061235878616571426,
0.06896103918552399,
0.07320034503936768,
0.0776243805885315,
0.05304349213838577,
0.06285123527050018,
0.0639304369688034,
0.07231374084949493,
0.07404901832342148,
0.08016443997621536,
0.08157160878181458
],
"loss_nats": 1.2989667654037476,
"loss_scale": 1.0,
"output_weights": {
"entropy_mean": 1.6036605834960938,
"mean_weights": [
0.09759747982025146,
0.04215940088033676,
0.041145484894514084,
0.029554029926657677,
0.02899969555437565,
0.03647579252719879,
0.06642461568117142,
0.341248482465744,
0.31639501452445984
],
"sources": 9
},
"step": 8000,
"stream_state_rms_by_sublayer": [
0.0933690071105957,
0.16915453970432281,
0.35791492462158203,
0.374446302652359,
0.3940892517566681,
0.39966297149658203,
0.4057151973247528,
0.4194087088108063,
0.09380833059549332,
0.09892481565475464,
0.11286329478025436,
0.11664740741252899,
0.13231655955314636,
0.13951337337493896,
0.1621972620487213,
0.18593810498714447,
0.05156813934445381,
0.06607726216316223,
0.08533531427383423,
0.10425819456577301,
0.12003090232610703,
0.14551182091236115,
0.16063299775123596,
0.19865906238555908,
0.046871740370988846,
0.06484340131282806,
0.079384446144104,
0.10430748760700226,
0.11459959298372269,
0.14720691740512848,
0.15625107288360596,
0.20439592003822327,
0.08397205173969269,
0.10052042454481125,
0.11654024571180344,
0.14085541665554047,
0.15294702351093292,
0.1745898574590683,
0.21572710573673248,
0.2420409768819809,
0.03708760067820549,
0.049456026405096054,
0.06553087383508682,
0.08509506285190582,
0.09390194714069366,
0.11783231794834137,
0.12852425873279572,
0.15649692714214325,
0.025487789884209633,
0.05533855780959129,
0.06059965491294861,
0.08507220447063446,
0.08860157430171967,
0.11942370235919952,
0.12100546807050705,
0.1559232622385025,
0.024166326969861984,
0.06628894805908203,
0.06877554953098297,
0.12384890764951706,
0.12682415544986725,
0.19928614795207977,
0.2025238573551178,
0.30097496509552
]
}
],
"environment": {
"autocast": "cuda-bfloat16-forward-fp32-cross-entropy",
"compile": false,
"compute_capability": [
12,
0
],
"cublas_workspace_config": ":4096:8",
"cuda": "12.8",
"deterministic_algorithms": true,
"gpu": "NVIDIA GeForce RTX 5090",
"python": "3.10.14",
"torch": "2.11.0+cu128"
},
"evaluations": [
{
"bits_per_byte": 7.9557801867581395,
"cross_entropy_nats": 5.514526605606079,
"step": 0
},
{
"bits_per_byte": 5.165135820292823,
"cross_entropy_nats": 3.5801993310451508,
"step": 100
},
{
"bits_per_byte": 3.109974443324717,
"cross_entropy_nats": 2.155670017004013,
"step": 500
},
{
"bits_per_byte": 1.9592225965086283,
"cross_entropy_nats": 1.358029618859291,
"step": 2000
},
{
"bits_per_byte": 1.8027438806514628,
"cross_entropy_nats": 1.249566838145256,
"step": 4000
},
{
"bits_per_byte": 1.7123525940760473,
"cross_entropy_nats": 1.1869123727083206,
"step": 8000
}
],
"gradient_gate": null,
"hashes": {
"final_mixer_parameters": "f16f089e57699cb35addeee1608f1119edec41037e2b24b645f889b4f23c5f71",
"final_model_state": "3f0b97ece3a15571ba3d656f589f512ca0bb9e20083c9f58a42ccaee14892f59",
"final_optimizer_state": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637",
"final_public_parameters": "1fbfd02c38aae6a81ba07a28d1135ce7d290651ca00bf5fa93595a997937551e",
"initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d",
"initial_public_parameter_elements": 18985152,
"initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670",
"initial_public_parameter_tensors": 227,
"initial_public_parameters": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58"
},
"manifest": {
"diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716",
"file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371",
"formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
"input_gate_tensor_hashes": {
"0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434",
"1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94",
"7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc"
},
"path": "experiments/k3/attnres_gradient/manifest.json",
"validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338"
},
"model": {
"attnres_aggregation_groups": 8,
"context": 256,
"d_ff": 768,
"d_head": 32,
"d_model": 192,
"heads": 6,
"layers": 32,
"parameters": {
"core": 18985152,
"embedding": 98304,
"mixer": 24960,
"total": 19010112
},
"sublayers": 64,
"sublayers_per_attnres_group": 8,
"transformer_blocks_per_attnres_group": 4,
"vocabulary": 256
},
"optimizer": {
"betas": [
0.9,
0.95
],
"epsilon": 1e-08,
"grad_clip": 1.0,
"min_lr": 3e-05,
"name": "AdamW",
"peak_lr": 0.0003,
"warmup_steps": 400,
"weight_decay_ndim_ge_2": 0.1
},
"protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
"run_kind": "formal",
"schema_version": 1,
"seed": 2026073001,
"steps": 8000,
"target_bytes_seen": 65536000,
"timing": {
"mean_ms": 109.14386622172245,
"measured_steps": 7980,
"median_ms": 109.04977201425936,
"p95_ms": 109.64786536205793,
"peak_allocated_bytes": 12815843328,
"peak_reserved_bytes": 13337886720,
"warmup_steps_excluded": 20
},
"training_history": [
{
"bits_per_byte": 7.958777844362678,
"learning_rate": 7.499999999999999e-07,
"loss_nats": 5.516604423522949,
"step": 1,
"unclipped_grad_norm": 21.323171615600586
},
{
"bits_per_byte": 7.247870358756172,
"learning_rate": 7.499999999999999e-06,
"loss_nats": 5.02384090423584,
"step": 10,
"unclipped_grad_norm": 11.153014183044434
},
{
"bits_per_byte": 6.7336346654397365,
"learning_rate": 1.4999999999999999e-05,
"loss_nats": 4.667399883270264,
"step": 20,
"unclipped_grad_norm": 3.4888553619384766
},
{
"bits_per_byte": 6.611855126224988,
"learning_rate": 2.2499999999999998e-05,
"loss_nats": 4.582988739013672,
"step": 30,
"unclipped_grad_norm": 3.075686454772949
},
{
"bits_per_byte": 6.3915955694037825,
"learning_rate": 2.9999999999999997e-05,
"loss_nats": 4.43031644821167,
"step": 40,
"unclipped_grad_norm": 3.2318925857543945
},
{
"bits_per_byte": 6.093941755902502,
"learning_rate": 3.75e-05,
"loss_nats": 4.223998546600342,
"step": 50,
"unclipped_grad_norm": 2.8704335689544678
},
{
"bits_per_byte": 5.8858000966610895,
"learning_rate": 4.4999999999999996e-05,
"loss_nats": 4.079725742340088,
"step": 60,
"unclipped_grad_norm": 2.669306755065918
},
{
"bits_per_byte": 5.715495996470702,
"learning_rate": 5.2499999999999995e-05,
"loss_nats": 3.9616799354553223,
"step": 70,
"unclipped_grad_norm": 3.1737496852874756
},
{
"bits_per_byte": 5.565897919513593,
"learning_rate": 5.9999999999999995e-05,
"loss_nats": 3.8579864501953125,
"step": 80,
"unclipped_grad_norm": 2.5544967651367188
},
{
"bits_per_byte": 5.354669171689194,
"learning_rate": 6.749999999999999e-05,
"loss_nats": 3.711573839187622,
"step": 90,
"unclipped_grad_norm": 2.425950050354004
},
{
"bits_per_byte": 5.180876231443403,
"learning_rate": 7.5e-05,
"loss_nats": 3.5911097526550293,
"step": 100,
"unclipped_grad_norm": 2.2783501148223877
},
{
"bits_per_byte": 4.992272489270609,
"learning_rate": 8.249999999999999e-05,
"loss_nats": 3.4603796005249023,
"step": 110,
"unclipped_grad_norm": 2.1105899810791016
},
{
"bits_per_byte": 4.775783480086919,
"learning_rate": 8.999999999999999e-05,
"loss_nats": 3.3103208541870117,
"step": 120,
"unclipped_grad_norm": 1.8969556093215942
},
{
"bits_per_byte": 4.582375918510225,
"learning_rate": 9.75e-05,
"loss_nats": 3.1762609481811523,
"step": 130,
"unclipped_grad_norm": 1.7015109062194824
},
{
"bits_per_byte": 4.427903165297061,
"learning_rate": 0.00010499999999999999,
"loss_nats": 3.0691885948181152,
"step": 140,
"unclipped_grad_norm": 1.446854829788208
},
{
"bits_per_byte": 4.271120341117507,
"learning_rate": 0.0001125,
"loss_nats": 2.960515022277832,
"step": 150,
"unclipped_grad_norm": 1.2307006120681763
},
{
"bits_per_byte": 4.166711049639903,
"learning_rate": 0.00011999999999999999,
"loss_nats": 2.888144016265869,
"step": 160,
"unclipped_grad_norm": 1.0598198175430298
},
{
"bits_per_byte": 4.026318516052877,
"learning_rate": 0.00012749999999999998,
"loss_nats": 2.7908313274383545,
"step": 170,
"unclipped_grad_norm": 0.9654788374900818
},
{
"bits_per_byte": 3.94755837315965,
"learning_rate": 0.00013499999999999997,
"loss_nats": 2.736238956451416,
"step": 180,
"unclipped_grad_norm": 0.8419675230979919
},
{
"bits_per_byte": 3.838975750676209,
"learning_rate": 0.0001425,
"loss_nats": 2.660975217819214,
"step": 190,
"unclipped_grad_norm": 0.817121684551239
},
{
"bits_per_byte": 3.774983758012572,
"learning_rate": 0.00015,
"loss_nats": 2.616619348526001,
"step": 200,
"unclipped_grad_norm": 0.955824613571167
},
{
"bits_per_byte": 3.7344016998190783,
"learning_rate": 0.0001575,
"loss_nats": 2.5884900093078613,
"step": 210,
"unclipped_grad_norm": 0.8557147979736328
},
{
"bits_per_byte": 3.63166201591605,
"learning_rate": 0.00016499999999999997,
"loss_nats": 2.5172762870788574,
"step": 220,
"unclipped_grad_norm": 0.6891153454780579
},
{
"bits_per_byte": 3.6208855830129867,
"learning_rate": 0.0001725,
"loss_nats": 2.5098066329956055,
"step": 230,
"unclipped_grad_norm": 1.036860466003418
},
{
"bits_per_byte": 3.4987607026353063,
"learning_rate": 0.00017999999999999998,
"loss_nats": 2.4251561164855957,
"step": 240,
"unclipped_grad_norm": 0.6406477689743042
},
{
"bits_per_byte": 3.5387748741156506,
"learning_rate": 0.0001875,
"loss_nats": 2.4528918266296387,
"step": 250,
"unclipped_grad_norm": 1.106186866760254
},
{
"bits_per_byte": 3.5260464381252943,
"learning_rate": 0.000195,
"loss_nats": 2.4440691471099854,
"step": 260,
"unclipped_grad_norm": 0.8556521534919739
},
{
"bits_per_byte": 3.5219934949750615,
"learning_rate": 0.00020249999999999996,
"loss_nats": 2.4412598609924316,
"step": 270,
"unclipped_grad_norm": 0.9917709827423096
},
{
"bits_per_byte": 3.474740573719901,
"learning_rate": 0.00020999999999999998,
"loss_nats": 2.4085066318511963,
"step": 280,
"unclipped_grad_norm": 1.2901222705841064
},
{
"bits_per_byte": 3.363479149466621,
"learning_rate": 0.00021749999999999997,
"loss_nats": 2.331386089324951,
"step": 290,
"unclipped_grad_norm": 0.7895627617835999
},
{
"bits_per_byte": 3.4037392561376993,
"learning_rate": 0.000225,
"loss_nats": 2.3592922687530518,
"step": 300,
"unclipped_grad_norm": 1.5281883478164673
},
{
"bits_per_byte": 3.3541442751430894,
"learning_rate": 0.00023249999999999996,
"loss_nats": 2.324915647506714,
"step": 310,
"unclipped_grad_norm": 0.8648361563682556
},
{
"bits_per_byte": 3.3434108379027414,
"learning_rate": 0.00023999999999999998,
"loss_nats": 2.3174757957458496,
"step": 320,
"unclipped_grad_norm": 1.4613573551177979
},
{
"bits_per_byte": 3.3325662998699364,
"learning_rate": 0.0002475,
"loss_nats": 2.3099589347839355,
"step": 330,
"unclipped_grad_norm": 1.1215400695800781
},
{
"bits_per_byte": 3.4180712583647006,
"learning_rate": 0.00025499999999999996,
"loss_nats": 2.3692264556884766,
"step": 340,
"unclipped_grad_norm": 1.1166281700134277
},
{
"bits_per_byte": 3.3577222022116393,
"learning_rate": 0.0002625,
"loss_nats": 2.3273956775665283,
"step": 350,
"unclipped_grad_norm": 1.0810235738754272
},
{
"bits_per_byte": 3.2923828974611427,
"learning_rate": 0.00026999999999999995,
"loss_nats": 2.2821059226989746,
"step": 360,
"unclipped_grad_norm": 1.128420114517212
},
{
"bits_per_byte": 3.291989401155969,
"learning_rate": 0.00027749999999999997,
"loss_nats": 2.2818331718444824,
"step": 370,
"unclipped_grad_norm": 1.0846341848373413
},
{
"bits_per_byte": 3.3389695579268603,
"learning_rate": 0.000285,
"loss_nats": 2.3143973350524902,
"step": 380,
"unclipped_grad_norm": 1.0868512392044067
},
{
"bits_per_byte": 3.256548592327368,
"learning_rate": 0.0002925,
"loss_nats": 2.257267475128174,
"step": 390,
"unclipped_grad_norm": 1.1476690769195557
},
{
"bits_per_byte": 3.2058776878689605,
"learning_rate": 0.0003,
"loss_nats": 2.2221450805664062,
"step": 400,
"unclipped_grad_norm": 1.1263198852539062
},
{
"bits_per_byte": 3.195324144481415,
"learning_rate": 0.0002999988466112388,
"loss_nats": 2.214829921722412,
"step": 410,
"unclipped_grad_norm": 1.1377208232879639
},
{
"bits_per_byte": 3.227808915506986,
"learning_rate": 0.00029999538646466355,
"loss_nats": 2.237346649169922,
"step": 420,
"unclipped_grad_norm": 1.2929860353469849
},
{
"bits_per_byte": 3.1876103786249166,
"learning_rate": 0.0002999896196193985,
"loss_nats": 2.2094831466674805,
"step": 430,
"unclipped_grad_norm": 1.071694016456604
},
{
"bits_per_byte": 3.2374495749837515,
"learning_rate": 0.00029998154617398326,
"loss_nats": 2.2440290451049805,
"step": 440,
"unclipped_grad_norm": 0.9089367985725403
},
{
"bits_per_byte": 3.155426921203657,
"learning_rate": 0.0002999711662663707,
"loss_nats": 2.1871752738952637,
"step": 450,
"unclipped_grad_norm": 1.1129212379455566
},
{
"bits_per_byte": 3.188086770567806,
"learning_rate": 0.0002999584800739247,
"loss_nats": 2.209813356399536,
"step": 460,
"unclipped_grad_norm": 1.3905333280563354
},
{
"bits_per_byte": 3.128408790718412,
"learning_rate": 0.0002999434878134173,
"loss_nats": 2.168447732925415,
"step": 470,
"unclipped_grad_norm": 1.059533953666687
},
{
"bits_per_byte": 3.136691131218663,
"learning_rate": 0.000299926189741025,
"loss_nats": 2.1741886138916016,
"step": 480,
"unclipped_grad_norm": 1.0161854028701782
},
{
"bits_per_byte": 3.0887341129908705,
"learning_rate": 0.000299906586152324,
"loss_nats": 2.1409473419189453,
"step": 490,
"unclipped_grad_norm": 1.385905385017395
},
{
"bits_per_byte": 3.033835874974255,
"learning_rate": 0.0002998846773822856,
"loss_nats": 2.1028947830200195,
"step": 500,
"unclipped_grad_norm": 1.1716443300247192
},
{
"bits_per_byte": 3.0607071322756636,
"learning_rate": 0.0002998604638052702,
"loss_nats": 2.121520519256592,
"step": 510,
"unclipped_grad_norm": 0.947066605091095
},
{
"bits_per_byte": 2.997100036784657,
"learning_rate": 0.0002998339458350211,
"loss_nats": 2.0774314403533936,
"step": 520,
"unclipped_grad_norm": 1.352820634841919
},
{
"bits_per_byte": 3.0643050093317137,
"learning_rate": 0.0002998051239246572,
"loss_nats": 2.124014377593994,
"step": 530,
"unclipped_grad_norm": 1.2646857500076294
},
{
"bits_per_byte": 3.008989197438803,
"learning_rate": 0.00029977399856666537,
"loss_nats": 2.085672378540039,
"step": 540,
"unclipped_grad_norm": 1.0451104640960693
},
{
"bits_per_byte": 2.978105584892765,
"learning_rate": 0.0002997405702928922,
"loss_nats": 2.064265489578247,
"step": 550,
"unclipped_grad_norm": 1.2619587182998657
},
{
"bits_per_byte": 2.968300166036473,
"learning_rate": 0.0002997048396745345,
"loss_nats": 2.057468891143799,
"step": 560,
"unclipped_grad_norm": 0.9861598610877991
},
{
"bits_per_byte": 2.9221661639078262,
"learning_rate": 0.0002996668073221302,
"loss_nats": 2.025491237640381,
"step": 570,
"unclipped_grad_norm": 1.1261875629425049
},
{
"bits_per_byte": 2.96999178739036,
"learning_rate": 0.000299626473885547,
"loss_nats": 2.0586414337158203,
"step": 580,
"unclipped_grad_norm": 0.8640331625938416
},
{
"bits_per_byte": 2.883698804389342,
"learning_rate": 0.00029958384005397226,
"loss_nats": 1.9988276958465576,
"step": 590,
"unclipped_grad_norm": 1.0034456253051758
},
{
"bits_per_byte": 2.9404706214044567,
"learning_rate": 0.0002995389065559004,
"loss_nats": 2.0381789207458496,
"step": 600,
"unclipped_grad_norm": 0.8714175224304199
},
{
"bits_per_byte": 2.8380133330141457,
"learning_rate": 0.000299491674159121,
"loss_nats": 1.967160940170288,
"step": 610,
"unclipped_grad_norm": 0.9427422881126404
},
{
"bits_per_byte": 2.894644124255553,
"learning_rate": 0.00029944214367070535,
"loss_nats": 2.0064144134521484,
"step": 620,
"unclipped_grad_norm": 1.0238139629364014
},
{
"bits_per_byte": 2.81883675149499,
"learning_rate": 0.00029939031593699284,
"loss_nats": 1.9538687467575073,
"step": 630,
"unclipped_grad_norm": 0.9875481128692627
},
{
"bits_per_byte": 2.7787291934352267,
"learning_rate": 0.00029933619184357654,
"loss_nats": 1.9260683059692383,
"step": 640,
"unclipped_grad_norm": 1.0125378370285034
},
{
"bits_per_byte": 2.728423580127252,
"learning_rate": 0.00029927977231528774,
"loss_nats": 1.8911991119384766,
"step": 650,
"unclipped_grad_norm": 1.5507988929748535
},
{
"bits_per_byte": 2.649404087396524,
"learning_rate": 0.00029922105831618057,
"loss_nats": 1.8364269733428955,
"step": 660,
"unclipped_grad_norm": 0.9040125012397766
},
{
"bits_per_byte": 2.7024973674923345,
"learning_rate": 0.0002991600508495153,
"loss_nats": 1.8732284307479858,
"step": 670,
"unclipped_grad_norm": 1.205861210823059
},
{
"bits_per_byte": 2.641403626461037,
"learning_rate": 0.00029909675095774114,
"loss_nats": 1.8308814764022827,
"step": 680,
"unclipped_grad_norm": 1.3370649814605713
},
{
"bits_per_byte": 2.7325929555321187,
"learning_rate": 0.0002990311597224787,
"loss_nats": 1.8940891027450562,
"step": 690,
"unclipped_grad_norm": 1.6124995946884155
},
{
"bits_per_byte": 2.6130706886099455,
"learning_rate": 0.00029896327826450103,
"loss_nats": 1.8112425804138184,
"step": 700,
"unclipped_grad_norm": 1.08383309841156
},
{
"bits_per_byte": 2.628106443844277,
"learning_rate": 0.0002988931077437152,
"loss_nats": 1.821664571762085,
"step": 710,
"unclipped_grad_norm": 1.0767779350280762
},
{
"bits_per_byte": 2.6725746220166666,
"learning_rate": 0.00029882064935914164,
"loss_nats": 1.852487564086914,
"step": 720,
"unclipped_grad_norm": 1.296352505683899
},
{
"bits_per_byte": 2.608218370098502,
"learning_rate": 0.0002987459043488943,
"loss_nats": 1.8078792095184326,
"step": 730,
"unclipped_grad_norm": 1.0864766836166382
},
{
"bits_per_byte": 2.555699340073923,
"learning_rate": 0.0002986688739901591,
"loss_nats": 1.7714757919311523,
"step": 740,
"unclipped_grad_norm": 1.0339913368225098
},
{
"bits_per_byte": 2.515936435246489,
"learning_rate": 0.0002985895595991723,
"loss_nats": 1.743914246559143,
"step": 750,
"unclipped_grad_norm": 1.1017143726348877
},
{
"bits_per_byte": 2.4657125136976736,
"learning_rate": 0.00029850796253119824,
"loss_nats": 1.709101676940918,
"step": 760,
"unclipped_grad_norm": 1.7173939943313599
},
{
"bits_per_byte": 2.601046349592221,
"learning_rate": 0.00029842408418050544,
"loss_nats": 1.802907943725586,
"step": 770,
"unclipped_grad_norm": 0.9487742781639099
},
{
"bits_per_byte": 2.4794518637098015,
"learning_rate": 0.0002983379259803436,
"loss_nats": 1.7186250686645508,
"step": 780,
"unclipped_grad_norm": 0.9247332811355591
},
{
"bits_per_byte": 2.466098098800908,
"learning_rate": 0.0002982494894029186,
"loss_nats": 1.7093689441680908,
"step": 790,
"unclipped_grad_norm": 1.9121034145355225
},
{
"bits_per_byte": 2.4025570446478333,
"learning_rate": 0.0002981587759593675,
"loss_nats": 1.66532564163208,
"step": 800,
"unclipped_grad_norm": 1.1767630577087402
},
{
"bits_per_byte": 2.455517898102479,
"learning_rate": 0.00029806578719973283,
"loss_nats": 1.7020353078842163,
"step": 810,
"unclipped_grad_norm": 1.1299240589141846
},
{
"bits_per_byte": 2.400107839716896,
"learning_rate": 0.00029797052471293596,
"loss_nats": 1.6636279821395874,
"step": 820,
"unclipped_grad_norm": 1.0006046295166016
},
{
"bits_per_byte": 2.4500607166079513,
"learning_rate": 0.0002978729901267499,
"loss_nats": 1.6982526779174805,
"step": 830,
"unclipped_grad_norm": 1.0554603338241577
},
{
"bits_per_byte": 2.5153210813217015,
"learning_rate": 0.0002977731851077716,
"loss_nats": 1.7434877157211304,
"step": 840,
"unclipped_grad_norm": 1.0736713409423828
},
{
"bits_per_byte": 2.399556119372928,
"learning_rate": 0.0002976711113613935,
"loss_nats": 1.6632455587387085,
"step": 850,
"unclipped_grad_norm": 0.8279580473899841
},
{
"bits_per_byte": 2.414819407654351,
"learning_rate": 0.0002975667706317743,
"loss_nats": 1.6738252639770508,
"step": 860,
"unclipped_grad_norm": 1.2342168092727661
},
{
"bits_per_byte": 2.3222158652452247,
"learning_rate": 0.00029746016470180915,
"loss_nats": 1.6096373796463013,
"step": 870,
"unclipped_grad_norm": 1.0648512840270996
},
{
"bits_per_byte": 2.3629336177502487,
"learning_rate": 0.00029735129539309926,
"loss_nats": 1.6378607749938965,
"step": 880,
"unclipped_grad_norm": 0.9258757829666138
},
{
"bits_per_byte": 2.2961121665146167,
"learning_rate": 0.0002972401645659207,
"loss_nats": 1.5915436744689941,
"step": 890,
"unclipped_grad_norm": 1.0717874765396118
},
{
"bits_per_byte": 2.3158979105831285,
"learning_rate": 0.00029712677411919283,
"loss_nats": 1.6052581071853638,
"step": 900,
"unclipped_grad_norm": 0.9598665833473206
},
{
"bits_per_byte": 2.3301258633063577,
"learning_rate": 0.0002970111259904454,
"loss_nats": 1.6151201725006104,
"step": 910,
"unclipped_grad_norm": 1.0995051860809326
},
{
"bits_per_byte": 2.3123376977276173,
"learning_rate": 0.0002968932221557859,
"loss_nats": 1.602790355682373,
"step": 920,
"unclipped_grad_norm": 1.1837356090545654
},
{
"bits_per_byte": 2.3340250539667196,
"learning_rate": 0.0002967730646298655,
"loss_nats": 1.6178228855133057,
"step": 930,
"unclipped_grad_norm": 1.1502224206924438
},
{
"bits_per_byte": 2.330418921743428,
"learning_rate": 0.0002966506554658449,
"loss_nats": 1.6153233051300049,
"step": 940,
"unclipped_grad_norm": 1.0088714361190796
},
{
"bits_per_byte": 2.2995397807463123,
"learning_rate": 0.00029652599675535876,
"loss_nats": 1.5939195156097412,
"step": 950,
"unclipped_grad_norm": 0.9483612179756165
},
{
"bits_per_byte": 2.3536873144919115,
"learning_rate": 0.0002963990906284808,
"loss_nats": 1.6314517259597778,
"step": 960,
"unclipped_grad_norm": 0.9582105278968811
},
{
"bits_per_byte": 2.2526236015215417,
"learning_rate": 0.0002962699392536863,
"loss_nats": 1.5613996982574463,
"step": 970,
"unclipped_grad_norm": 1.2002545595169067
},
{
"bits_per_byte": 2.3230659754884395,
"learning_rate": 0.0002961385448378161,
"loss_nats": 1.6102266311645508,
"step": 980,
"unclipped_grad_norm": 0.9191399812698364
},
{
"bits_per_byte": 2.285470911975131,
"learning_rate": 0.00029600490962603817,
"loss_nats": 1.584167718887329,
"step": 990,
"unclipped_grad_norm": 1.053741216659546
},
{
"bits_per_byte": 2.248061417742104,
"learning_rate": 0.00029586903590180956,
"loss_nats": 1.5582374334335327,
"step": 1000,
"unclipped_grad_norm": 0.9024814963340759
},
{
"bits_per_byte": 2.2402749032242903,
"learning_rate": 0.00029573092598683735,
"loss_nats": 1.552840232849121,
"step": 1010,
"unclipped_grad_norm": 1.048907995223999
},
{
"bits_per_byte": 2.2041726491204714,
"learning_rate": 0.00029559058224103895,
"loss_nats": 1.5278160572052002,
"step": 1020,
"unclipped_grad_norm": 1.0486118793487549
},
{
"bits_per_byte": 2.216881995036582,
"learning_rate": 0.0002954480070625017,
"loss_nats": 1.5366255044937134,
"step": 1030,
"unclipped_grad_norm": 0.9243784546852112
},
{
"bits_per_byte": 2.2610579746851553,
"learning_rate": 0.00029530320288744205,
"loss_nats": 1.5672459602355957,
"step": 1040,
"unclipped_grad_norm": 1.102870225906372
},
{
"bits_per_byte": 2.2528270569975124,
"learning_rate": 0.0002951561721901638,
"loss_nats": 1.5615407228469849,
"step": 1050,
"unclipped_grad_norm": 1.0764544010162354
},
{
"bits_per_byte": 2.14889054582713,
"learning_rate": 0.00029500691748301594,
"loss_nats": 1.489497423171997,
"step": 1060,
"unclipped_grad_norm": 0.971049427986145
},
{
"bits_per_byte": 2.2350196293618754,
"learning_rate": 0.00029485544131634963,
"loss_nats": 1.5491975545883179,
"step": 1070,
"unclipped_grad_norm": 1.0203380584716797
},
{
"bits_per_byte": 2.217042282867186,
"learning_rate": 0.0002947017462784745,
"loss_nats": 1.5367366075515747,
"step": 1080,
"unclipped_grad_norm": 1.023221492767334
},
{
"bits_per_byte": 2.186512266651322,
"learning_rate": 0.00029454583499561476,
"loss_nats": 1.5155748128890991,
"step": 1090,
"unclipped_grad_norm": 1.111006498336792
},
{
"bits_per_byte": 2.2323391077655277,
"learning_rate": 0.00029438771013186413,
"loss_nats": 1.5473395586013794,
"step": 1100,
"unclipped_grad_norm": 0.9547863602638245
},
{
"bits_per_byte": 2.116467344590568,
"learning_rate": 0.0002942273743891401,
"loss_nats": 1.4670233726501465,
"step": 1110,
"unclipped_grad_norm": 1.1085364818572998
},
{
"bits_per_byte": 2.143238164023873,
"learning_rate": 0.00029406483050713824,
"loss_nats": 1.485579490661621,
"step": 1120,
"unclipped_grad_norm": 0.901618242263794
},
{
"bits_per_byte": 2.077939275196329,
"learning_rate": 0.00029390008126328504,
"loss_nats": 1.4403177499771118,
"step": 1130,
"unclipped_grad_norm": 0.8683149814605713
},
{
"bits_per_byte": 2.224320932617002,
"learning_rate": 0.00029373312947269045,
"loss_nats": 1.5417817831039429,
"step": 1140,
"unclipped_grad_norm": 1.030465006828308
},
{
"bits_per_byte": 2.0642131678483175,
"learning_rate": 0.00029356397798809995,
"loss_nats": 1.4308035373687744,
"step": 1150,
"unclipped_grad_norm": 0.827274739742279
},
{
"bits_per_byte": 2.103388407957753,
"learning_rate": 0.0002933926296998457,
"loss_nats": 1.4579577445983887,
"step": 1160,
"unclipped_grad_norm": 0.9158841967582703
},
{
"bits_per_byte": 2.1566975262803965,
"learning_rate": 0.0002932190875357972,
"loss_nats": 1.4949088096618652,
"step": 1170,
"unclipped_grad_norm": 0.9397322535514832
},
{
"bits_per_byte": 2.1322636071070153,
"learning_rate": 0.00029304335446131126,
"loss_nats": 1.4779725074768066,
"step": 1180,
"unclipped_grad_norm": 0.8510220050811768
},
{
"bits_per_byte": 2.068760561119781,
"learning_rate": 0.00029286543347918116,
"loss_nats": 1.4339555501937866,
"step": 1190,
"unclipped_grad_norm": 0.9270830154418945
},
{
"bits_per_byte": 2.0310528489701576,
"learning_rate": 0.0002926853276295856,
"loss_nats": 1.4078185558319092,
"step": 1200,
"unclipped_grad_norm": 0.8815417885780334
},
{
"bits_per_byte": 2.1008361854189492,
"learning_rate": 0.0002925030399900368,
"loss_nats": 1.456188678741455,
"step": 1210,
"unclipped_grad_norm": 1.2037686109542847
},
{
"bits_per_byte": 2.1123913557651197,
"learning_rate": 0.00029231857367532726,
"loss_nats": 1.464198112487793,
"step": 1220,
"unclipped_grad_norm": 0.8964646458625793
},
{
"bits_per_byte": 2.0711254945517954,
"learning_rate": 0.0002921319318374776,
"loss_nats": 1.4355947971343994,
"step": 1230,
"unclipped_grad_norm": 0.8661414980888367
},
{
"bits_per_byte": 2.0159510523978943,
"learning_rate": 0.00029194311766568174,
"loss_nats": 1.397350788116455,
"step": 1240,
"unclipped_grad_norm": 0.9392534494400024
},
{
"bits_per_byte": 2.104896867788471,
"learning_rate": 0.000291752134386253,
"loss_nats": 1.4590033292770386,
"step": 1250,
"unclipped_grad_norm": 0.8430818915367126
},
{
"bits_per_byte": 2.08180441329182,
"learning_rate": 0.00029155898526256856,
"loss_nats": 1.442996859550476,
"step": 1260,
"unclipped_grad_norm": 0.8917548656463623
},
{
"bits_per_byte": 2.087352401626003,
"learning_rate": 0.0002913636735950141,
"loss_nats": 1.4468424320220947,
"step": 1270,
"unclipped_grad_norm": 0.9866762161254883
},
{
"bits_per_byte": 1.9607164163161914,
"learning_rate": 0.0002911662027209272,
"loss_nats": 1.359065055847168,
"step": 1280,
"unclipped_grad_norm": 0.8596939444541931
},
{
"bits_per_byte": 2.088993460080536,
"learning_rate": 0.00029096657601454013,
"loss_nats": 1.4479799270629883,
"step": 1290,
"unclipped_grad_norm": 1.007126808166504
},
{
"bits_per_byte": 2.034349412421896,
"learning_rate": 0.00029076479688692266,
"loss_nats": 1.4101035594940186,
"step": 1300,
"unclipped_grad_norm": 0.8474438786506653
},
{
"bits_per_byte": 2.0247273270714237,
"learning_rate": 0.00029056086878592323,
"loss_nats": 1.4034340381622314,
"step": 1310,
"unclipped_grad_norm": 0.8226730823516846
},
{
"bits_per_byte": 2.020511000402871,
"learning_rate": 0.00029035479519611045,
"loss_nats": 1.4005115032196045,
"step": 1320,
"unclipped_grad_norm": 0.9414357542991638
},
{
"bits_per_byte": 2.085571349293668,
"learning_rate": 0.0002901465796387134,
"loss_nats": 1.4456079006195068,
"step": 1330,
"unclipped_grad_norm": 0.9917200207710266
},
{
"bits_per_byte": 2.0422444479924042,
"learning_rate": 0.00028993622567156154,
"loss_nats": 1.4155759811401367,
"step": 1340,
"unclipped_grad_norm": 0.8823419213294983
},
{
"bits_per_byte": 2.0447034559344406,
"learning_rate": 0.0002897237368890237,
"loss_nats": 1.4172804355621338,
"step": 1350,
"unclipped_grad_norm": 0.8861486315727234
},
{
"bits_per_byte": 2.0988803987133373,
"learning_rate": 0.0002895091169219469,
"loss_nats": 1.4548330307006836,
"step": 1360,
"unclipped_grad_norm": 0.9090827107429504
},
{
"bits_per_byte": 2.082760980795919,
"learning_rate": 0.00028929236943759425,
"loss_nats": 1.4436599016189575,
"step": 1370,
"unclipped_grad_norm": 0.8366561532020569
},
{
"bits_per_byte": 2.0317923743688677,
"learning_rate": 0.00028907349813958225,
"loss_nats": 1.4083311557769775,
"step": 1380,
"unclipped_grad_norm": 0.8379060626029968
},
{
"bits_per_byte": 2.0432778917414387,
"learning_rate": 0.0002888525067678175,
"loss_nats": 1.4162923097610474,
"step": 1390,
"unclipped_grad_norm": 0.8665340542793274
},
{
"bits_per_byte": 1.9746810636009497,
"learning_rate": 0.00028862939909843273,
"loss_nats": 1.3687446117401123,
"step": 1400,
"unclipped_grad_norm": 0.8112426996231079
},
{
"bits_per_byte": 2.0034977886489185,
"learning_rate": 0.0002884041789437225,
"loss_nats": 1.388718843460083,
"step": 1410,
"unclipped_grad_norm": 0.8745980262756348
},
{
"bits_per_byte": 2.019574210903621,
"learning_rate": 0.0002881768501520778,
"loss_nats": 1.3998621702194214,
"step": 1420,
"unclipped_grad_norm": 0.8596643209457397
},
{
"bits_per_byte": 2.0775151659793014,
"learning_rate": 0.0002879474166079205,
"loss_nats": 1.4400237798690796,
"step": 1430,
"unclipped_grad_norm": 0.913244903087616
},
{
"bits_per_byte": 2.027370700415185,
"learning_rate": 0.00028771588223163694,
"loss_nats": 1.405266284942627,
"step": 1440,
"unclipped_grad_norm": 1.0067864656448364
},
{
"bits_per_byte": 1.9631205617926026,
"learning_rate": 0.00028748225097951054,
"loss_nats": 1.3607314825057983,
"step": 1450,
"unclipped_grad_norm": 0.8125852942466736
},
{
"bits_per_byte": 1.9959419028659118,
"learning_rate": 0.000287246526843655,
"loss_nats": 1.383481502532959,
"step": 1460,
"unclipped_grad_norm": 0.9857028126716614
},
{
"bits_per_byte": 1.9364156117145783,
"learning_rate": 0.00028700871385194533,
"loss_nats": 1.3422210216522217,
"step": 1470,
"unclipped_grad_norm": 0.8698804974555969
},
{
"bits_per_byte": 1.996789089404062,
"learning_rate": 0.00028676881606794945,
"loss_nats": 1.3840687274932861,
"step": 1480,
"unclipped_grad_norm": 0.8397032022476196
},
{
"bits_per_byte": 2.017818268038312,
"learning_rate": 0.00028652683759085854,
"loss_nats": 1.398645043373108,
"step": 1490,
"unclipped_grad_norm": 0.8181726932525635
},
{
"bits_per_byte": 1.996446843928845,
"learning_rate": 0.0002862827825554171,
"loss_nats": 1.38383150100708,
"step": 1500,
"unclipped_grad_norm": 0.8242432475090027
},
{
"bits_per_byte": 2.0162594172908914,
"learning_rate": 0.00028603665513185245,
"loss_nats": 1.3975645303726196,
"step": 1510,
"unclipped_grad_norm": 0.9299814701080322
},
{
"bits_per_byte": 1.980743108078562,
"learning_rate": 0.000285788459525803,
"loss_nats": 1.3729465007781982,
"step": 1520,
"unclipped_grad_norm": 0.8446369171142578
},
{
"bits_per_byte": 2.047233664693934,
"learning_rate": 0.00028553819997824707,
"loss_nats": 1.4190342426300049,
"step": 1530,
"unclipped_grad_norm": 0.9717140197753906
},
{
"bits_per_byte": 1.993557363411699,
"learning_rate": 0.00028528588076542964,
"loss_nats": 1.3818286657333374,
"step": 1540,
"unclipped_grad_norm": 0.8417568802833557
},
{
"bits_per_byte": 2.049996737962696,
"learning_rate": 0.00028503150619878996,
"loss_nats": 1.4209494590759277,
"step": 1550,
"unclipped_grad_norm": 0.8546327352523804
},
{
"bits_per_byte": 1.966114779744124,
"learning_rate": 0.0002847750806248876,
"loss_nats": 1.3628069162368774,
"step": 1560,
"unclipped_grad_norm": 0.9258830547332764
},
{
"bits_per_byte": 2.0427641795633114,
"learning_rate": 0.000284516608425328,
"loss_nats": 1.4159362316131592,
"step": 1570,
"unclipped_grad_norm": 0.8567822575569153
},
{
"bits_per_byte": 1.9830621221427962,
"learning_rate": 0.000284256094016688,
"loss_nats": 1.374553918838501,
"step": 1580,
"unclipped_grad_norm": 0.8522606492042542
},
{
"bits_per_byte": 1.9876789964052082,
"learning_rate": 0.0002839935418504399,
"loss_nats": 1.3777540922164917,
"step": 1590,
"unclipped_grad_norm": 0.8731296062469482
},
{
"bits_per_byte": 2.0122188568915207,
"learning_rate": 0.000283728956412876,
"loss_nats": 1.3947638273239136,
"step": 1600,
"unclipped_grad_norm": 0.8627029657363892
},
{
"bits_per_byte": 1.9498011934138815,
"learning_rate": 0.00028346234222503136,
"loss_nats": 1.3514991998672485,
"step": 1610,
"unclipped_grad_norm": 0.7740172147750854
},
{
"bits_per_byte": 1.9095543294069197,
"learning_rate": 0.00028319370384260675,
"loss_nats": 1.3236021995544434,
"step": 1620,
"unclipped_grad_norm": 0.7910023331642151
},
{
"bits_per_byte": 1.9407452749500496,
"learning_rate": 0.00028292304585589105,
"loss_nats": 1.3452221155166626,
"step": 1630,
"unclipped_grad_norm": 0.8484107851982117
},
{
"bits_per_byte": 1.9911346438089947,
"learning_rate": 0.0002826503728896824,
"loss_nats": 1.3801493644714355,
"step": 1640,
"unclipped_grad_norm": 0.8722788691520691
},
{
"bits_per_byte": 2.0368601871418424,
"learning_rate": 0.0002823756896032095,
"loss_nats": 1.4118438959121704,
"step": 1650,
"unclipped_grad_norm": 0.8482462763786316
},
{
"bits_per_byte": 1.9665676100638458,
"learning_rate": 0.0002820990006900518,
"loss_nats": 1.3631207942962646,
"step": 1660,
"unclipped_grad_norm": 0.8444085121154785
},
{
"bits_per_byte": 1.9945896032821775,
"learning_rate": 0.00028182031087805947,
"loss_nats": 1.3825441598892212,
"step": 1670,
"unclipped_grad_norm": 0.8560684323310852
},
{
"bits_per_byte": 2.0061820938635853,
"learning_rate": 0.0002815396249292725,
"loss_nats": 1.3905794620513916,
"step": 1680,
"unclipped_grad_norm": 0.8526912927627563
},
{
"bits_per_byte": 2.0514603103215387,
"learning_rate": 0.0002812569476398391,
"loss_nats": 1.4219639301300049,
"step": 1690,
"unclipped_grad_norm": 0.8917502760887146
},
{
"bits_per_byte": 1.9422809080396035,
"learning_rate": 0.00028097228383993446,
"loss_nats": 1.3462865352630615,
"step": 1700,
"unclipped_grad_norm": 0.7474871873855591
},
{
"bits_per_byte": 1.878015978909047,
"learning_rate": 0.0002806856383936774,
"loss_nats": 1.3017414808273315,
"step": 1710,
"unclipped_grad_norm": 0.8072315454483032
},
{
"bits_per_byte": 1.8905051870320533,
"learning_rate": 0.00028039701619904774,
"loss_nats": 1.3103983402252197,
"step": 1720,
"unclipped_grad_norm": 0.845556914806366
},
{
"bits_per_byte": 1.8762399140735866,
"learning_rate": 0.0002801064221878024,
"loss_nats": 1.3005104064941406,
"step": 1730,
"unclipped_grad_norm": 0.9344940185546875
},
{
"bits_per_byte": 1.9632232354351677,
"learning_rate": 0.00027981386132539134,
"loss_nats": 1.3608026504516602,
"step": 1740,
"unclipped_grad_norm": 0.7860697507858276
},
{
"bits_per_byte": 1.989613629244764,
"learning_rate": 0.0002795193386108724,
"loss_nats": 1.3790950775146484,
"step": 1750,
"unclipped_grad_norm": 0.8348115086555481
},
{
"bits_per_byte": 1.9869124697303127,
"learning_rate": 0.00027922285907682625,
"loss_nats": 1.3772227764129639,
"step": 1760,
"unclipped_grad_norm": 0.8862095475196838
},
{
"bits_per_byte": 1.9928130224987648,
"learning_rate": 0.00027892442778927005,
"loss_nats": 1.3813127279281616,
"step": 1770,
"unclipped_grad_norm": 0.816246509552002
},
{
"bits_per_byte": 1.8514405317671223,
"learning_rate": 0.00027862404984757116,
"loss_nats": 1.2833207845687866,
"step": 1780,
"unclipped_grad_norm": 0.8118871450424194
},
{
"bits_per_byte": 1.9588221993995885,
"learning_rate": 0.0002783217303843597,
"loss_nats": 1.3577520847320557,
"step": 1790,
"unclipped_grad_norm": 0.839783251285553
},
{
"bits_per_byte": 1.8827697513615458,
"learning_rate": 0.00027801747456544134,
"loss_nats": 1.3050365447998047,
"step": 1800,
"unclipped_grad_norm": 0.8949951529502869
},
{
"bits_per_byte": 1.92937567388416,
"learning_rate": 0.00027771128758970843,
"loss_nats": 1.33734130859375,
"step": 1810,
"unclipped_grad_norm": 0.7295374274253845
},
{
"bits_per_byte": 1.8974312723476003,
"learning_rate": 0.00027740317468905165,
"loss_nats": 1.3151991367340088,
"step": 1820,
"unclipped_grad_norm": 0.7755441069602966
},
{
"bits_per_byte": 1.8886371114783813,
"learning_rate": 0.0002770931411282704,
"loss_nats": 1.3091034889221191,
"step": 1830,
"unclipped_grad_norm": 0.7666956186294556
},
{
"bits_per_byte": 1.8876932706904463,
"learning_rate": 0.0002767811922049827,
"loss_nats": 1.3084492683410645,
"step": 1840,
"unclipped_grad_norm": 0.8784642219543457
},
{
"bits_per_byte": 1.8964055678178544,
"learning_rate": 0.00027646733324953503,
"loss_nats": 1.314488172531128,
"step": 1850,
"unclipped_grad_norm": 0.8603965044021606
},
{
"bits_per_byte": 1.8927124123732264,
"learning_rate": 0.00027615156962491097,
"loss_nats": 1.3119282722473145,
"step": 1860,
"unclipped_grad_norm": 0.8056252002716064
},
{
"bits_per_byte": 1.8818489562488263,
"learning_rate": 0.0002758339067266396,
"loss_nats": 1.3043982982635498,
"step": 1870,
"unclipped_grad_norm": 0.8151036500930786
},
{
"bits_per_byte": 1.8811881989042114,
"learning_rate": 0.0002755143499827035,
"loss_nats": 1.3039402961730957,
"step": 1880,
"unclipped_grad_norm": 0.79389888048172
},
{
"bits_per_byte": 1.8949089747900463,
"learning_rate": 0.0002751929048534456,
"loss_nats": 1.313450813293457,
"step": 1890,
"unclipped_grad_norm": 0.8192400932312012
},
{
"bits_per_byte": 1.9233574849825177,
"learning_rate": 0.0002748695768314763,
"loss_nats": 1.3331698179244995,
"step": 1900,
"unclipped_grad_norm": 0.7950363159179688
},
{
"bits_per_byte": 1.8611783614416253,
"learning_rate": 0.00027454437144157947,
"loss_nats": 1.2900705337524414,
"step": 1910,
"unclipped_grad_norm": 0.8037525415420532
},
{
"bits_per_byte": 1.855479888287937,
"learning_rate": 0.00027421729424061787,
"loss_nats": 1.2861206531524658,
"step": 1920,
"unclipped_grad_norm": 0.7507341504096985
},
{
"bits_per_byte": 1.9602173226633874,
"learning_rate": 0.0002738883508174385,
"loss_nats": 1.3587191104888916,
"step": 1930,
"unclipped_grad_norm": 0.8329893350601196
},
{
"bits_per_byte": 1.8066889261861108,
"learning_rate": 0.0002735575467927768,
"loss_nats": 1.2523013353347778,
"step": 1940,
"unclipped_grad_norm": 0.7595474123954773
},
{
"bits_per_byte": 1.9033319970987033,
"learning_rate": 0.0002732248878191609,
"loss_nats": 1.319289207458496,
"step": 1950,
"unclipped_grad_norm": 0.7690061330795288
},
{
"bits_per_byte": 1.816397862775269,
"learning_rate": 0.0002728903795808148,
"loss_nats": 1.259031057357788,
"step": 1960,
"unclipped_grad_norm": 0.7758741974830627
},
{
"bits_per_byte": 1.909067618504978,
"learning_rate": 0.0002725540277935614,
"loss_nats": 1.3232648372650146,
"step": 1970,
"unclipped_grad_norm": 0.7503845691680908
},
{
"bits_per_byte": 1.9405849871194454,
"learning_rate": 0.00027221583820472456,
"loss_nats": 1.3451110124588013,
"step": 1980,
"unclipped_grad_norm": 0.8655099868774414
},
{
"bits_per_byte": 1.9436471382180576,
"learning_rate": 0.0002718758165930314,
"loss_nats": 1.347233533859253,
"step": 1990,
"unclipped_grad_norm": 0.8436980247497559
},
{
"bits_per_byte": 1.9015801818170146,
"learning_rate": 0.0002715339687685131,
"loss_nats": 1.3180749416351318,
"step": 2000,
"unclipped_grad_norm": 0.8021923303604126
},
{
"bits_per_byte": 1.8746163978494417,
"learning_rate": 0.00027119030057240576,
"loss_nats": 1.2993850708007812,
"step": 2010,
"unclipped_grad_norm": 0.7083144187927246
},
{
"bits_per_byte": 1.9072580170525995,
"learning_rate": 0.0002708448178770506,
"loss_nats": 1.3220105171203613,
"step": 2020,
"unclipped_grad_norm": 0.7678324580192566
},
{
"bits_per_byte": 1.983610230851096,
"learning_rate": 0.00027049752658579376,
"loss_nats": 1.3749338388442993,
"step": 2030,
"unclipped_grad_norm": 0.8317387104034424
},
{
"bits_per_byte": 1.909348466173837,
"learning_rate": 0.00027014843263288514,
"loss_nats": 1.323459506034851,
"step": 2040,
"unclipped_grad_norm": 0.8302199840545654
},
{
"bits_per_byte": 1.961244747019642,
"learning_rate": 0.00026979754198337724,
"loss_nats": 1.359431266784668,
"step": 2050,
"unclipped_grad_norm": 0.7990598082542419
},
{
"bits_per_byte": 1.9220292629699043,
"learning_rate": 0.00026944486063302305,
"loss_nats": 1.3322491645812988,
"step": 2060,
"unclipped_grad_norm": 0.8404503464698792
},
{
"bits_per_byte": 1.8663830724046866,
"learning_rate": 0.0002690903946081738,
"loss_nats": 1.2936781644821167,
"step": 2070,
"unclipped_grad_norm": 0.7720745801925659
},
{
"bits_per_byte": 1.8672120287818452,
"learning_rate": 0.00026873414996567596,
"loss_nats": 1.2942527532577515,
"step": 2080,
"unclipped_grad_norm": 0.8028935790061951
},
{
"bits_per_byte": 1.857879218250124,
"learning_rate": 0.00026837613279276736,
"loss_nats": 1.2877837419509888,
"step": 2090,
"unclipped_grad_norm": 0.7713416814804077
},
{
"bits_per_byte": 1.8603779541845098,
"learning_rate": 0.0002680163492069739,
"loss_nats": 1.289515733718872,
"step": 2100,
"unclipped_grad_norm": 0.7443708181381226
},
{
"bits_per_byte": 1.897931741861611,
"learning_rate": 0.00026765480535600414,
"loss_nats": 1.3155460357666016,
"step": 2110,
"unclipped_grad_norm": 0.7534405589103699
},
{
"bits_per_byte": 1.8710701155886507,
"learning_rate": 0.0002672915074176452,
"loss_nats": 1.2969269752502441,
"step": 2120,
"unclipped_grad_norm": 0.7505779266357422
},
{
"bits_per_byte": 1.8349618420920515,
"learning_rate": 0.00026692646159965624,
"loss_nats": 1.271898627281189,
"step": 2130,
"unclipped_grad_norm": 0.7835680246353149
},
{
"bits_per_byte": 1.9616896661374241,
"learning_rate": 0.0002665596741396633,
"loss_nats": 1.3597396612167358,
"step": 2140,
"unclipped_grad_norm": 0.7786996960639954
},
{
"bits_per_byte": 1.8168395142226847,
"learning_rate": 0.000266191151305052,
"loss_nats": 1.2593371868133545,
"step": 2150,
"unclipped_grad_norm": 0.7194197773933411
},
{
"bits_per_byte": 1.8739360344826286,
"learning_rate": 0.00026582089939286075,
"loss_nats": 1.2989134788513184,
"step": 2160,
"unclipped_grad_norm": 0.8012002110481262
},
{
"bits_per_byte": 1.9634492206384013,
"learning_rate": 0.0002654489247296732,
"loss_nats": 1.3609592914581299,
"step": 2170,
"unclipped_grad_norm": 0.8270084261894226
},
{
"bits_per_byte": 1.8175344961148212,
"learning_rate": 0.00026507523367151007,
"loss_nats": 1.2598189115524292,
"step": 2180,
"unclipped_grad_norm": 0.7824540734291077
},
{
"bits_per_byte": 1.8458584908680682,
"learning_rate": 0.00026469983260372037,
"loss_nats": 1.279451608657837,
"step": 2190,
"unclipped_grad_norm": 0.7973959445953369
},
{
"bits_per_byte": 1.8557901449900935,
"learning_rate": 0.0002643227279408727,
"loss_nats": 1.2863357067108154,
"step": 2200,
"unclipped_grad_norm": 0.7451788783073425
},
{
"bits_per_byte": 1.8380872828061812,
"learning_rate": 0.0002639439261266453,
"loss_nats": 1.2740650177001953,
"step": 2210,
"unclipped_grad_norm": 0.7804975509643555
},
{
"bits_per_byte": 1.8298283319464475,
"learning_rate": 0.000263563433633716,
"loss_nats": 1.2683403491973877,
"step": 2220,
"unclipped_grad_norm": 0.7694668173789978
},
{
"bits_per_byte": 1.9031531351418058,
"learning_rate": 0.00026318125696365156,
"loss_nats": 1.3191652297973633,
"step": 2230,
"unclipped_grad_norm": 0.8418469429016113
},
{
"bits_per_byte": 1.8400618156207371,
"learning_rate": 0.000262797402646797,
"loss_nats": 1.2754336595535278,
"step": 2240,
"unclipped_grad_norm": 0.7987614870071411
},
{
"bits_per_byte": 1.852736936989326,
"learning_rate": 0.00026241187724216334,
"loss_nats": 1.2842193841934204,
"step": 2250,
"unclipped_grad_norm": 0.7795421481132507
},
{
"bits_per_byte": 1.8339105121473271,
"learning_rate": 0.0002620246873373162,
"loss_nats": 1.271169900894165,
"step": 2260,
"unclipped_grad_norm": 0.7681040167808533
},
{
"bits_per_byte": 1.825916414569922,
"learning_rate": 0.00026163583954826264,
"loss_nats": 1.2656288146972656,
"step": 2270,
"unclipped_grad_norm": 0.7933830618858337
},
{
"bits_per_byte": 1.8451832869807807,
"learning_rate": 0.0002612453405193386,
"loss_nats": 1.2789835929870605,
"step": 2280,
"unclipped_grad_norm": 0.7935850024223328
},
{
"bits_per_byte": 1.8711039961708706,
"learning_rate": 0.000260853196923095,
"loss_nats": 1.2969504594802856,
"step": 2290,
"unclipped_grad_norm": 0.7818295955657959
},
{
"bits_per_byte": 1.7944778140095496,
"learning_rate": 0.0002604594154601839,
"loss_nats": 1.2438372373580933,
"step": 2300,
"unclipped_grad_norm": 0.7245032787322998
},
{
"bits_per_byte": 1.8221434591752936,
"learning_rate": 0.0002600640028592441,
"loss_nats": 1.2630136013031006,
"step": 2310,
"unclipped_grad_norm": 0.7396021485328674
},
{
"bits_per_byte": 1.7928696042413317,
"learning_rate": 0.0002596669658767859,
"loss_nats": 1.242722511291504,
"step": 2320,
"unclipped_grad_norm": 0.7618010640144348
},
{
"bits_per_byte": 1.8702191454321815,
"learning_rate": 0.000259268311297076,
"loss_nats": 1.2963371276855469,
"step": 2330,
"unclipped_grad_norm": 0.7263991236686707
},
{
"bits_per_byte": 1.7994940319872652,
"learning_rate": 0.00025886804593202114,
"loss_nats": 1.247314214706421,
"step": 2340,
"unclipped_grad_norm": 0.7788637280464172
},
{
"bits_per_byte": 1.849551646312696,
"learning_rate": 0.00025846617662105193,
"loss_nats": 1.2820115089416504,
"step": 2350,
"unclipped_grad_norm": 0.7732409238815308
},
{
"bits_per_byte": 1.8613248906601605,
"learning_rate": 0.00025806271023100637,
"loss_nats": 1.2901721000671387,
"step": 2360,
"unclipped_grad_norm": 0.8177725672721863
},
{
"bits_per_byte": 1.8822589628884834,
"learning_rate": 0.00025765765365601165,
"loss_nats": 1.3046824932098389,
"step": 2370,
"unclipped_grad_norm": 0.7931574583053589
},
{
"bits_per_byte": 1.8382422391746087,
"learning_rate": 0.00025725101381736707,
"loss_nats": 1.2741724252700806,
"step": 2380,
"unclipped_grad_norm": 0.8604201674461365
},
{
"bits_per_byte": 1.8797817247854554,
"learning_rate": 0.00025684279766342567,
"loss_nats": 1.3029654026031494,
"step": 2390,
"unclipped_grad_norm": 0.7636454105377197
},
{
"bits_per_byte": 1.817860919186159,
"learning_rate": 0.00025643301216947504,
"loss_nats": 1.2600451707839966,
"step": 2400,
"unclipped_grad_norm": 0.779283881187439
},
{
"bits_per_byte": 1.8354538844561699,
"learning_rate": 0.00025602166433761866,
"loss_nats": 1.2722396850585938,
"step": 2410,
"unclipped_grad_norm": 0.7781111001968384
},
{
"bits_per_byte": 1.8069649583407457,
"learning_rate": 0.0002556087611966561,
"loss_nats": 1.2524926662445068,
"step": 2420,
"unclipped_grad_norm": 0.7548633813858032
},
{
"bits_per_byte": 1.8169932667125561,
"learning_rate": 0.00025519430980196277,
"loss_nats": 1.259443759918213,
"step": 2430,
"unclipped_grad_norm": 0.7562755942344666
},
{
"bits_per_byte": 1.8760644917697067,
"learning_rate": 0.0002547783172353695,
"loss_nats": 1.3003888130187988,
"step": 2440,
"unclipped_grad_norm": 0.7478408813476562
},
{
"bits_per_byte": 1.838183937055966,
"learning_rate": 0.0002543607906050415,
"loss_nats": 1.2741320133209229,
"step": 2450,
"unclipped_grad_norm": 0.760445237159729
},
{
"bits_per_byte": 1.8195946762895068,
"learning_rate": 0.0002539417370453568,
"loss_nats": 1.261246919631958,
"step": 2460,
"unclipped_grad_norm": 0.7331485748291016
},
{
"bits_per_byte": 1.8163858239897086,
"learning_rate": 0.00025352116371678464,
"loss_nats": 1.2590227127075195,
"step": 2470,
"unclipped_grad_norm": 0.7682369351387024
},
{
"bits_per_byte": 1.8266683227194942,
"learning_rate": 0.0002530990778057627,
"loss_nats": 1.2661499977111816,
"step": 2480,
"unclipped_grad_norm": 0.7132375240325928
},
{
"bits_per_byte": 1.8042156436840528,
"learning_rate": 0.00025267548652457475,
"loss_nats": 1.250586986541748,
"step": 2490,
"unclipped_grad_norm": 0.7770085334777832
},
{
"bits_per_byte": 1.8313916542427904,
"learning_rate": 0.00025225039711122696,
"loss_nats": 1.2694239616394043,
"step": 2500,
"unclipped_grad_norm": 0.7582318782806396
},
{
"bits_per_byte": 1.7788244690759694,
"learning_rate": 0.00025182381682932464,
"loss_nats": 1.2329871654510498,
"step": 2510,
"unclipped_grad_norm": 0.8006963133811951
},
{
"bits_per_byte": 1.8448090527325032,
"learning_rate": 0.0002513957529679477,
"loss_nats": 1.278724193572998,
"step": 2520,
"unclipped_grad_norm": 0.7802237272262573
},
{
"bits_per_byte": 1.855150369528884,
"learning_rate": 0.00025096621284152656,
"loss_nats": 1.2858922481536865,
"step": 2530,
"unclipped_grad_norm": 0.7603861689567566
},
{
"bits_per_byte": 1.7974587892969547,
"learning_rate": 0.0002505352037897168,
"loss_nats": 1.245903491973877,
"step": 2540,
"unclipped_grad_norm": 0.7829954624176025
},
{
"bits_per_byte": 1.8048358131230642,
"learning_rate": 0.0002501027331772739,
"loss_nats": 1.2510168552398682,
"step": 2550,
"unclipped_grad_norm": 0.8207260966300964
},
{
"bits_per_byte": 1.8410618947355044,
"learning_rate": 0.00024966880839392746,
"loss_nats": 1.2761268615722656,
"step": 2560,
"unclipped_grad_norm": 0.8064310550689697
},
{
"bits_per_byte": 1.8072459779922554,
"learning_rate": 0.0002492334368542548,
"loss_nats": 1.2526874542236328,
"step": 2570,
"unclipped_grad_norm": 0.7300699353218079
},
{
"bits_per_byte": 1.8352736466380657,
"learning_rate": 0.00024879662599755416,
"loss_nats": 1.2721147537231445,
"step": 2580,
"unclipped_grad_norm": 0.7476873993873596
},
{
"bits_per_byte": 1.793617556789934,
"learning_rate": 0.00024835838328771803,
"loss_nats": 1.2432409524917603,
"step": 2590,
"unclipped_grad_norm": 0.7210070490837097
},
{
"bits_per_byte": 1.797604974550188,
"learning_rate": 0.0002479187162131051,
"loss_nats": 1.2460048198699951,
"step": 2600,
"unclipped_grad_norm": 0.7853913903236389
},
{
"bits_per_byte": 1.7385629865436842,
"learning_rate": 0.00024747763228641264,
"loss_nats": 1.2050800323486328,
"step": 2610,
"unclipped_grad_norm": 0.7092626094818115
},
{
"bits_per_byte": 1.8079149905041119,
"learning_rate": 0.00024703513904454795,
"loss_nats": 1.2531511783599854,
"step": 2620,
"unclipped_grad_norm": 0.7522015571594238
},
{
"bits_per_byte": 1.8198622812942493,
"learning_rate": 0.00024659124404849956,
"loss_nats": 1.261432409286499,
"step": 2630,
"unclipped_grad_norm": 0.7675716876983643
},
{
"bits_per_byte": 1.680096624469618,
"learning_rate": 0.00024614595488320833,
"loss_nats": 1.164554238319397,
"step": 2640,
"unclipped_grad_norm": 0.6825466156005859
},
{
"bits_per_byte": 1.7638260340768748,
"learning_rate": 0.00024569927915743745,
"loss_nats": 1.2225910425186157,
"step": 2650,
"unclipped_grad_norm": 0.7630164623260498
},
{
"bits_per_byte": 1.8201589513669878,
"learning_rate": 0.0002452512245036427,
"loss_nats": 1.2616380453109741,
"step": 2660,
"unclipped_grad_norm": 0.7772946953773499
},
{
"bits_per_byte": 1.811244230659514,
"learning_rate": 0.00024480179857784177,
"loss_nats": 1.2554588317871094,
"step": 2670,
"unclipped_grad_norm": 0.7582975625991821
},
{
"bits_per_byte": 1.74257860945868,
"learning_rate": 0.00024435100905948387,
"loss_nats": 1.207863450050354,
"step": 2680,
"unclipped_grad_norm": 0.7818799614906311
},
{
"bits_per_byte": 1.9030688636428832,
"learning_rate": 0.00024389886365131793,
"loss_nats": 1.3191068172454834,
"step": 2690,
"unclipped_grad_norm": 0.7075229287147522
},
{
"bits_per_byte": 1.8200497423836899,
"learning_rate": 0.00024344537007926163,
"loss_nats": 1.2615623474121094,
"step": 2700,
"unclipped_grad_norm": 0.8272333741188049
},
{
"bits_per_byte": 1.7354167359288,
"learning_rate": 0.00024299053609226878,
"loss_nats": 1.2028992176055908,
"step": 2710,
"unclipped_grad_norm": 0.7310962677001953
},
{
"bits_per_byte": 1.728331394678554,
"learning_rate": 0.00024253436946219733,
"loss_nats": 1.1979880332946777,
"step": 2720,
"unclipped_grad_norm": 0.7193791270256042
},
{
"bits_per_byte": 1.76859700479446,
"learning_rate": 0.0002420768779836764,
"loss_nats": 1.225898027420044,
"step": 2730,
"unclipped_grad_norm": 0.773657500743866
},
{
"bits_per_byte": 1.7790022991369616,
"learning_rate": 0.00024161806947397305,
"loss_nats": 1.2331104278564453,
"step": 2740,
"unclipped_grad_norm": 0.7759174704551697
},
{
"bits_per_byte": 1.7909771071512373,
"learning_rate": 0.00024115795177285886,
"loss_nats": 1.241410732269287,
"step": 2750,
"unclipped_grad_norm": 0.770866870880127
},
{
"bits_per_byte": 1.81886375002334,
"learning_rate": 0.0002406965327424758,
"loss_nats": 1.2607402801513672,
"step": 2760,
"unclipped_grad_norm": 0.7793042659759521
},
{
"bits_per_byte": 1.704546881994831,
"learning_rate": 0.00024023382026720197,
"loss_nats": 1.181501865386963,
"step": 2770,
"unclipped_grad_norm": 0.7179723978042603
},
{
"bits_per_byte": 1.7222445846992174,
"learning_rate": 0.00023976982225351698,
"loss_nats": 1.1937689781188965,
"step": 2780,
"unclipped_grad_norm": 0.711125910282135
},
{
"bits_per_byte": 1.8073385046584196,
"learning_rate": 0.0002393045466298666,
"loss_nats": 1.2527515888214111,
"step": 2790,
"unclipped_grad_norm": 0.7564381957054138
},
{
"bits_per_byte": 1.8564130661515186,
"learning_rate": 0.00023883800134652763,
"loss_nats": 1.2867674827575684,
"step": 2800,
"unclipped_grad_norm": 0.7982874512672424
},
{
"bits_per_byte": 1.7924368958917607,
"learning_rate": 0.0002383701943754715,
"loss_nats": 1.2424225807189941,
"step": 2810,
"unclipped_grad_norm": 0.827074408531189
},
{
"bits_per_byte": 1.781587886310033,
"learning_rate": 0.00023790113371022884,
"loss_nats": 1.2349026203155518,
"step": 2820,
"unclipped_grad_norm": 0.6945767998695374
},
{
"bits_per_byte": 1.862601517877515,
"learning_rate": 0.0002374308273657521,
"loss_nats": 1.2910569906234741,
"step": 2830,
"unclipped_grad_norm": 0.7473636269569397
},
{
"bits_per_byte": 1.7520161574247766,
"learning_rate": 0.00023695928337827911,
"loss_nats": 1.2144050598144531,
"step": 2840,
"unclipped_grad_norm": 0.7269325852394104
},
{
"bits_per_byte": 1.7726036846116109,
"learning_rate": 0.00023648650980519564,
"loss_nats": 1.2286752462387085,
"step": 2850,
"unclipped_grad_norm": 0.7537488341331482
},
{
"bits_per_byte": 1.7375121725469125,
"learning_rate": 0.00023601251472489757,
"loss_nats": 1.2043516635894775,
"step": 2860,
"unclipped_grad_norm": 0.7839256525039673
},
{
"bits_per_byte": 1.836773851301542,
"learning_rate": 0.00023553730623665305,
"loss_nats": 1.273154616355896,
"step": 2870,
"unclipped_grad_norm": 0.7615643739700317
},
{
"bits_per_byte": 1.8235200083127996,
"learning_rate": 0.00023506089246046403,
"loss_nats": 1.263967752456665,
"step": 2880,
"unclipped_grad_norm": 0.7500733137130737
},
{
"bits_per_byte": 1.7346435019305209,
"learning_rate": 0.00023458328153692736,
"loss_nats": 1.2023632526397705,
"step": 2890,
"unclipped_grad_norm": 0.7335168123245239
},
{
"bits_per_byte": 1.710140445731493,
"learning_rate": 0.0002341044816270961,
"loss_nats": 1.1853790283203125,
"step": 2900,
"unclipped_grad_norm": 0.7237004041671753
},
{
"bits_per_byte": 1.7640040361205178,
"learning_rate": 0.00023362450091233953,
"loss_nats": 1.2227144241333008,
"step": 2910,
"unclipped_grad_norm": 0.7653093338012695
},
{
"bits_per_byte": 1.7372325287566095,
"learning_rate": 0.00023314334759420388,
"loss_nats": 1.204157829284668,
"step": 2920,
"unclipped_grad_norm": 0.7592934966087341
},
{
"bits_per_byte": 1.7605243111456106,
"learning_rate": 0.0002326610298942717,
"loss_nats": 1.2203024625778198,
"step": 2930,
"unclipped_grad_norm": 0.7871323227882385
},
{
"bits_per_byte": 1.7773966691085064,
"learning_rate": 0.00023217755605402175,
"loss_nats": 1.2319974899291992,
"step": 2940,
"unclipped_grad_norm": 0.7994735240936279
},
{
"bits_per_byte": 1.761262116717812,
"learning_rate": 0.00023169293433468802,
"loss_nats": 1.2208138704299927,
"step": 2950,
"unclipped_grad_norm": 0.7672474384307861
},
{
"bits_per_byte": 1.8009139207527887,
"learning_rate": 0.00023120717301711857,
"loss_nats": 1.2482984066009521,
"step": 2960,
"unclipped_grad_norm": 0.7721540927886963
},
{
"bits_per_byte": 1.7577087831683356,
"learning_rate": 0.00023072028040163408,
"loss_nats": 1.218350887298584,
"step": 2970,
"unclipped_grad_norm": 0.7659915685653687
},
{
"bits_per_byte": 1.7511909846659368,
"learning_rate": 0.00023023226480788593,
"loss_nats": 1.2138330936431885,
"step": 2980,
"unclipped_grad_norm": 0.7457980513572693
},
{
"bits_per_byte": 1.7677185174138528,
"learning_rate": 0.00022974313457471416,
"loss_nats": 1.2252891063690186,
"step": 2990,
"unclipped_grad_norm": 0.7551211714744568
},
{
"bits_per_byte": 1.7804380103063644,
"learning_rate": 0.0002292528980600049,
"loss_nats": 1.2341055870056152,
"step": 3000,
"unclipped_grad_norm": 0.7369211316108704
},
{
"bits_per_byte": 1.77081558099059,
"learning_rate": 0.00022876156364054758,
"loss_nats": 1.227435827255249,
"step": 3010,
"unclipped_grad_norm": 0.7526711225509644
},
{
"bits_per_byte": 1.726040929734363,
"learning_rate": 0.0002282691397118918,
"loss_nats": 1.1964004039764404,
"step": 3020,
"unclipped_grad_norm": 0.8113403916358948
},
{
"bits_per_byte": 1.8045116258261877,
"learning_rate": 0.00022777563468820376,
"loss_nats": 1.250792145729065,
"step": 3030,
"unclipped_grad_norm": 0.7493385672569275
},
{
"bits_per_byte": 1.7229166928987893,
"learning_rate": 0.00022728105700212282,
"loss_nats": 1.194234848022461,
"step": 3040,
"unclipped_grad_norm": 0.7606903314590454
},
{
"bits_per_byte": 1.7403560776615798,
"learning_rate": 0.00022678541510461698,
"loss_nats": 1.2063229084014893,
"step": 3050,
"unclipped_grad_norm": 0.7690193057060242
},
{
"bits_per_byte": 1.759874388708,
"learning_rate": 0.00022628871746483876,
"loss_nats": 1.2198519706726074,
"step": 3060,
"unclipped_grad_norm": 0.7480750679969788
},
{
"bits_per_byte": 1.6873952242069339,
"learning_rate": 0.0002257909725699804,
"loss_nats": 1.169613242149353,
"step": 3070,
"unclipped_grad_norm": 0.7001752853393555
},
{
"bits_per_byte": 1.737272084766308,
"learning_rate": 0.00022529218892512886,
"loss_nats": 1.2041852474212646,
"step": 3080,
"unclipped_grad_norm": 0.8322204351425171
},
{
"bits_per_byte": 1.8015143121869506,
"learning_rate": 0.00022479237505312034,
"loss_nats": 1.248714566230774,
"step": 3090,
"unclipped_grad_norm": 0.8183287382125854
},
{
"bits_per_byte": 1.7001176408045102,
"learning_rate": 0.00022429153949439496,
"loss_nats": 1.178431749343872,
"step": 3100,
"unclipped_grad_norm": 0.7750749588012695
},
{
"bits_per_byte": 1.7770231227908324,
"learning_rate": 0.00022378969080685053,
"loss_nats": 1.231738567352295,
"step": 3110,
"unclipped_grad_norm": 0.7082787752151489
},
{
"bits_per_byte": 1.749435729731231,
"learning_rate": 0.0002232868375656965,
"loss_nats": 1.2126164436340332,
"step": 3120,
"unclipped_grad_norm": 0.736611545085907
},
{
"bits_per_byte": 1.7966217497352055,
"learning_rate": 0.00022278298836330733,
"loss_nats": 1.2453233003616333,
"step": 3130,
"unclipped_grad_norm": 0.7366204857826233
},
{
"bits_per_byte": 1.7636881039908827,
"learning_rate": 0.00022227815180907572,
"loss_nats": 1.222495436668396,
"step": 3140,
"unclipped_grad_norm": 0.7583194375038147
},
{
"bits_per_byte": 1.7359698321339747,
"learning_rate": 0.00022177233652926546,
"loss_nats": 1.2032825946807861,
"step": 3150,
"unclipped_grad_norm": 0.7600930333137512
},
{
"bits_per_byte": 1.7415977924008093,
"learning_rate": 0.00022126555116686416,
"loss_nats": 1.207183599472046,
"step": 3160,
"unclipped_grad_norm": 0.7639689445495605
},
{
"bits_per_byte": 1.654884311818432,
"learning_rate": 0.00022075780438143532,
"loss_nats": 1.1470783948898315,
"step": 3170,
"unclipped_grad_norm": 0.7011478543281555
},
{
"bits_per_byte": 1.7584833930278216,
"learning_rate": 0.00022024910484897065,
"loss_nats": 1.2188878059387207,
"step": 3180,
"unclipped_grad_norm": 0.7645173072814941
},
{
"bits_per_byte": 1.7305350083840592,
"learning_rate": 0.00021973946126174158,
"loss_nats": 1.199515461921692,
"step": 3190,
"unclipped_grad_norm": 0.7665988206863403
},
{
"bits_per_byte": 1.7003714871971838,
"learning_rate": 0.00021922888232815086,
"loss_nats": 1.178607702255249,
"step": 3200,
"unclipped_grad_norm": 0.7379894852638245
},
{
"bits_per_byte": 1.6959066455981344,
"learning_rate": 0.0002187173767725837,
"loss_nats": 1.1755129098892212,
"step": 3210,
"unclipped_grad_norm": 0.8013631701469421
},
{
"bits_per_byte": 1.7098007799960389,
"learning_rate": 0.0002182049533352588,
"loss_nats": 1.1851435899734497,
"step": 3220,
"unclipped_grad_norm": 0.7296364903450012
},
{
"bits_per_byte": 1.7987765203678656,
"learning_rate": 0.00021769162077207884,
"loss_nats": 1.246816873550415,
"step": 3230,
"unclipped_grad_norm": 0.7610434889793396
},
{
"bits_per_byte": 1.6862874839527267,
"learning_rate": 0.00021717738785448102,
"loss_nats": 1.1688454151153564,
"step": 3240,
"unclipped_grad_norm": 0.723595917224884
},
{
"bits_per_byte": 1.8345631863073515,
"learning_rate": 0.00021666226336928708,
"loss_nats": 1.2716223001480103,
"step": 3250,
"unclipped_grad_norm": 0.8353017568588257
},
{
"bits_per_byte": 1.6872093109613513,
"learning_rate": 0.0002161462561185532,
"loss_nats": 1.1694843769073486,
"step": 3260,
"unclipped_grad_norm": 0.7602720260620117
},
{
"bits_per_byte": 1.7198954736710823,
"learning_rate": 0.00021562937491941957,
"loss_nats": 1.1921406984329224,
"step": 3270,
"unclipped_grad_norm": 0.7516167163848877
},
{
"bits_per_byte": 1.7692164863028679,
"learning_rate": 0.00021511162860395975,
"loss_nats": 1.2263274192810059,
"step": 3280,
"unclipped_grad_norm": 0.7487898468971252
},
{
"bits_per_byte": 1.7624494849393688,
"learning_rate": 0.0002145930260190298,
"loss_nats": 1.2216368913650513,
"step": 3290,
"unclipped_grad_norm": 0.7708044052124023
},
{
"bits_per_byte": 1.7002387165907176,
"learning_rate": 0.00021407357602611698,
"loss_nats": 1.1785156726837158,
"step": 3300,
"unclipped_grad_norm": 0.7703046202659607
},
{
"bits_per_byte": 1.7332599015043295,
"learning_rate": 0.0002135532875011885,
"loss_nats": 1.2014042139053345,
"step": 3310,
"unclipped_grad_norm": 0.8132672905921936
},
{
"bits_per_byte": 1.7262779218272521,
"learning_rate": 0.0002130321693345397,
"loss_nats": 1.1965646743774414,
"step": 3320,
"unclipped_grad_norm": 0.7545798420906067
},
{
"bits_per_byte": 1.7272033604715449,
"learning_rate": 0.00021251023043064224,
"loss_nats": 1.1972061395645142,
"step": 3330,
"unclipped_grad_norm": 0.7775130867958069
},
{
"bits_per_byte": 1.6975052243379043,
"learning_rate": 0.00021198747970799188,
"loss_nats": 1.1766209602355957,
"step": 3340,
"unclipped_grad_norm": 0.7105540633201599
},
{
"bits_per_byte": 1.7083408192728646,
"learning_rate": 0.00021146392609895618,
"loss_nats": 1.1841316223144531,
"step": 3350,
"unclipped_grad_norm": 0.7163253426551819
},
{
"bits_per_byte": 1.732425613664994,
"learning_rate": 0.00021093957854962178,
"loss_nats": 1.2008259296417236,
"step": 3360,
"unclipped_grad_norm": 0.7626740336418152
},
{
"bits_per_byte": 1.6756070173688444,
"learning_rate": 0.00021041444601964157,
"loss_nats": 1.1614422798156738,
"step": 3370,
"unclipped_grad_norm": 0.8061453700065613
},
{
"bits_per_byte": 1.7092863798873084,
"learning_rate": 0.00020988853748208164,
"loss_nats": 1.1847870349884033,
"step": 3380,
"unclipped_grad_norm": 0.7406735420227051
},
{
"bits_per_byte": 1.7539413312185348,
"learning_rate": 0.00020936186192326782,
"loss_nats": 1.2157394886016846,
"step": 3390,
"unclipped_grad_norm": 0.7695116400718689
},
{
"bits_per_byte": 1.8092177590843976,
"learning_rate": 0.00020883442834263225,
"loss_nats": 1.2540541887283325,
"step": 3400,
"unclipped_grad_norm": 0.7858016490936279
},
{
"bits_per_byte": 1.8255297975707825,
"learning_rate": 0.0002083062457525596,
"loss_nats": 1.2653608322143555,
"step": 3410,
"unclipped_grad_norm": 0.7603964805603027
},
{
"bits_per_byte": 1.6857877023693193,
"learning_rate": 0.00020777732317823306,
"loss_nats": 1.1684989929199219,
"step": 3420,
"unclipped_grad_norm": 0.7168062329292297
},
{
"bits_per_byte": 1.6176447364444573,
"learning_rate": 0.00020724766965748005,
"loss_nats": 1.1212658882141113,
"step": 3430,
"unclipped_grad_norm": 0.7431134581565857
},
{
"bits_per_byte": 1.711189711884407,
"learning_rate": 0.00020671729424061788,
"loss_nats": 1.1861063241958618,
"step": 3440,
"unclipped_grad_norm": 0.7753933668136597
},
{
"bits_per_byte": 1.77535626693867,
"learning_rate": 0.00020618620599029906,
"loss_nats": 1.2305831909179688,
"step": 3450,
"unclipped_grad_norm": 0.8042635321617126
},
{
"bits_per_byte": 1.7083845028661837,
"learning_rate": 0.00020565441398135649,
"loss_nats": 1.184161901473999,
"step": 3460,
"unclipped_grad_norm": 0.7639697194099426
},
{
"bits_per_byte": 1.6937178224006033,
"learning_rate": 0.00020512192730064827,
"loss_nats": 1.1739957332611084,
"step": 3470,
"unclipped_grad_norm": 0.748127281665802
},
{
"bits_per_byte": 1.6501064617948125,
"learning_rate": 0.00020458875504690273,
"loss_nats": 1.1437666416168213,
"step": 3480,
"unclipped_grad_norm": 0.7506517767906189
},
{
"bits_per_byte": 1.8358463488654388,
"learning_rate": 0.00020405490633056245,
"loss_nats": 1.2725117206573486,
"step": 3490,
"unclipped_grad_norm": 0.836082398891449
},
{
"bits_per_byte": 1.714104301868579,
"learning_rate": 0.00020352039027362917,
"loss_nats": 1.188126564025879,
"step": 3500,
"unclipped_grad_norm": 0.7544166445732117
},
{
"bits_per_byte": 1.744907598516664,
"learning_rate": 0.0002029852160095075,
"loss_nats": 1.2094777822494507,
"step": 3510,
"unclipped_grad_norm": 0.7650617361068726
},
{
"bits_per_byte": 1.738600994709525,
"learning_rate": 0.000202449392682849,
"loss_nats": 1.2051063776016235,
"step": 3520,
"unclipped_grad_norm": 0.7794367671012878
},
{
"bits_per_byte": 1.7051804660806096,
"learning_rate": 0.0002019129294493959,
"loss_nats": 1.181941032409668,
"step": 3530,
"unclipped_grad_norm": 0.7445787787437439
},
{
"bits_per_byte": 1.7158172490711725,
"learning_rate": 0.00020137583547582478,
"loss_nats": 1.1893138885498047,
"step": 3540,
"unclipped_grad_norm": 0.7629107236862183
},
{
"bits_per_byte": 1.7173026632266746,
"learning_rate": 0.00020083811993958973,
"loss_nats": 1.1903434991836548,
"step": 3550,
"unclipped_grad_norm": 0.7763495445251465
},
{
"bits_per_byte": 1.6560372835098163,
"learning_rate": 0.00020029979202876563,
"loss_nats": 1.14787757396698,
"step": 3560,
"unclipped_grad_norm": 0.7450886964797974
},
{
"bits_per_byte": 1.6838996768281473,
"learning_rate": 0.00019976086094189118,
"loss_nats": 1.1671903133392334,
"step": 3570,
"unclipped_grad_norm": 0.7487124800682068
},
{
"bits_per_byte": 1.6946102403759304,
"learning_rate": 0.00019922133588781175,
"loss_nats": 1.1746143102645874,
"step": 3580,
"unclipped_grad_norm": 0.757170557975769
},
{
"bits_per_byte": 1.7307594457434352,
"learning_rate": 0.0001986812260855219,
"loss_nats": 1.1996710300445557,
"step": 3590,
"unclipped_grad_norm": 0.7700981497764587
},
{
"bits_per_byte": 1.7491381997452384,
"learning_rate": 0.00019814054076400787,
"loss_nats": 1.2124102115631104,
"step": 3600,
"unclipped_grad_norm": 0.7877120971679688
},
{
"bits_per_byte": 1.688025540622346,
"learning_rate": 0.00019759928916209004,
"loss_nats": 1.1700501441955566,
"step": 3610,
"unclipped_grad_norm": 0.780867874622345
},
{
"bits_per_byte": 1.6436277033541595,
"learning_rate": 0.00019705748052826493,
"loss_nats": 1.1392759084701538,
"step": 3620,
"unclipped_grad_norm": 0.7423334717750549
},
{
"bits_per_byte": 1.6950637586262558,
"learning_rate": 0.00019651512412054723,
"loss_nats": 1.1749286651611328,
"step": 3630,
"unclipped_grad_norm": 0.7513464689254761
},
{
"bits_per_byte": 1.6758522646289749,
"learning_rate": 0.00019597222920631144,
"loss_nats": 1.1616122722625732,
"step": 3640,
"unclipped_grad_norm": 0.7453336715698242
},
{
"bits_per_byte": 1.736063734661346,
"learning_rate": 0.0001954288050621338,
"loss_nats": 1.2033476829528809,
"step": 3650,
"unclipped_grad_norm": 0.7682862877845764
},
{
"bits_per_byte": 1.6360068081291268,
"learning_rate": 0.00019488486097363363,
"loss_nats": 1.1339935064315796,
"step": 3660,
"unclipped_grad_norm": 0.7531437277793884
},
{
"bits_per_byte": 1.7183529612754938,
"learning_rate": 0.00019434040623531452,
"loss_nats": 1.1910715103149414,
"step": 3670,
"unclipped_grad_norm": 0.7578480243682861
},
{
"bits_per_byte": 1.6583936178092877,
"learning_rate": 0.0001937954501504058,
"loss_nats": 1.1495108604431152,
"step": 3680,
"unclipped_grad_norm": 0.7224898934364319
},
{
"bits_per_byte": 1.6372521345040243,
"learning_rate": 0.00019325000203070333,
"loss_nats": 1.1348567008972168,
"step": 3690,
"unclipped_grad_norm": 0.7246464490890503
},
{
"bits_per_byte": 1.6659130432703113,
"learning_rate": 0.0001927040711964106,
"loss_nats": 1.1547229290008545,
"step": 3700,
"unclipped_grad_norm": 0.7878638505935669
},
{
"bits_per_byte": 1.694639821391879,
"learning_rate": 0.00019215766697597923,
"loss_nats": 1.1746348142623901,
"step": 3710,
"unclipped_grad_norm": 0.7455682754516602
},
{
"bits_per_byte": 1.6891824679147003,
"learning_rate": 0.00019161079870594978,
"loss_nats": 1.1708520650863647,
"step": 3720,
"unclipped_grad_norm": 0.7515487670898438
},
{
"bits_per_byte": 1.6091924770851538,
"learning_rate": 0.0001910634757307922,
"loss_nats": 1.1154072284698486,
"step": 3730,
"unclipped_grad_norm": 0.7353127598762512
},
{
"bits_per_byte": 1.6633680440028435,
"learning_rate": 0.0001905157074027459,
"loss_nats": 1.152958869934082,
"step": 3740,
"unclipped_grad_norm": 0.7253232002258301
},
{
"bits_per_byte": 1.6922951819126664,
"learning_rate": 0.00018996750308166037,
"loss_nats": 1.1730096340179443,
"step": 3750,
"unclipped_grad_norm": 0.7686370611190796
},
{
"bits_per_byte": 1.7713081393026613,
"learning_rate": 0.00018941887213483482,
"loss_nats": 1.2277772426605225,
"step": 3760,
"unclipped_grad_norm": 0.7601476907730103
},
{
"bits_per_byte": 1.7305630415561497,
"learning_rate": 0.00018886982393685845,
"loss_nats": 1.1995348930358887,
"step": 3770,
"unclipped_grad_norm": 0.7877833247184753
},
{
"bits_per_byte": 1.5644979697442287,
"learning_rate": 0.00018832036786945008,
"loss_nats": 1.0844273567199707,
"step": 3780,
"unclipped_grad_norm": 0.7386898994445801
},
{
"bits_per_byte": 1.6644766441703052,
"learning_rate": 0.0001877705133212979,
"loss_nats": 1.1537272930145264,
"step": 3790,
"unclipped_grad_norm": 0.7420179843902588
},
{
"bits_per_byte": 1.6653939996300076,
"learning_rate": 0.00018722026968789906,
"loss_nats": 1.1543631553649902,
"step": 3800,
"unclipped_grad_norm": 0.7838358283042908
},
{
"bits_per_byte": 1.6952983429620325,
"learning_rate": 0.00018666964637139907,
"loss_nats": 1.17509126663208,
"step": 3810,
"unclipped_grad_norm": 0.8048396110534668
},
{
"bits_per_byte": 1.6744323758634516,
"learning_rate": 0.00018611865278043115,
"loss_nats": 1.160628080368042,
"step": 3820,
"unclipped_grad_norm": 0.7709245681762695
},
{
"bits_per_byte": 1.720191799778519,
"learning_rate": 0.00018556729832995555,
"loss_nats": 1.1923460960388184,
"step": 3830,
"unclipped_grad_norm": 0.7672807574272156
},
{
"bits_per_byte": 1.6467005173771254,
"learning_rate": 0.00018501559244109864,
"loss_nats": 1.1414058208465576,
"step": 3840,
"unclipped_grad_norm": 0.7319741249084473
},
{
"bits_per_byte": 1.5964396195583748,
"learning_rate": 0.00018446354454099178,
"loss_nats": 1.106567621231079,
"step": 3850,
"unclipped_grad_norm": 0.7756319642066956
},
{
"bits_per_byte": 1.6360811046342996,
"learning_rate": 0.00018391116406261048,
"loss_nats": 1.1340450048446655,
"step": 3860,
"unclipped_grad_norm": 0.766754150390625
},
{
"bits_per_byte": 1.7433616464680584,
"learning_rate": 0.00018335846044461305,
"loss_nats": 1.2084062099456787,
"step": 3870,
"unclipped_grad_norm": 0.7916337847709656
},
{
"bits_per_byte": 1.6358671582166262,
"learning_rate": 0.0001828054431311793,
"loss_nats": 1.1338967084884644,
"step": 3880,
"unclipped_grad_norm": 0.7497678399085999
},
{
"bits_per_byte": 1.600699113872294,
"learning_rate": 0.00018225212157184934,
"loss_nats": 1.1095200777053833,
"step": 3890,
"unclipped_grad_norm": 0.7555440664291382
},
{
"bits_per_byte": 1.6778517349279058,
"learning_rate": 0.00018169850522136192,
"loss_nats": 1.1629981994628906,
"step": 3900,
"unclipped_grad_norm": 0.826478123664856
},
{
"bits_per_byte": 1.6805711246033486,
"learning_rate": 0.000181144603539493,
"loss_nats": 1.1648831367492676,
"step": 3910,
"unclipped_grad_norm": 0.7849931120872498
},
{
"bits_per_byte": 1.6375147520118918,
"learning_rate": 0.00018059042599089412,
"loss_nats": 1.1350387334823608,
"step": 3920,
"unclipped_grad_norm": 0.7588120102882385
},
{
"bits_per_byte": 1.673005091843941,
"learning_rate": 0.00018003598204493048,
"loss_nats": 1.15963876247406,
"step": 3930,
"unclipped_grad_norm": 0.742434024810791
},
{
"bits_per_byte": 1.6996766772876981,
"learning_rate": 0.00017948128117551936,
"loss_nats": 1.1781260967254639,
"step": 3940,
"unclipped_grad_norm": 0.7588658928871155
},
{
"bits_per_byte": 1.7339400928884918,
"learning_rate": 0.00017892633286096816,
"loss_nats": 1.2018756866455078,
"step": 3950,
"unclipped_grad_norm": 0.7937244176864624
},
{
"bits_per_byte": 1.713467450112434,
"learning_rate": 0.00017837114658381247,
"loss_nats": 1.1876851320266724,
"step": 3960,
"unclipped_grad_norm": 0.7884036302566528
},
{
"bits_per_byte": 1.6516962694193882,
"learning_rate": 0.00017781573183065386,
"loss_nats": 1.1448686122894287,
"step": 3970,
"unclipped_grad_norm": 0.8004661798477173
},
{
"bits_per_byte": 1.6740500584305835,
"learning_rate": 0.0001772600980919981,
"loss_nats": 1.1603630781173706,
"step": 3980,
"unclipped_grad_norm": 0.7692103385925293
},
{
"bits_per_byte": 1.691941929547794,
"learning_rate": 0.0001767042548620927,
"loss_nats": 1.172764778137207,
"step": 3990,
"unclipped_grad_norm": 0.7891059517860413
},
{
"bits_per_byte": 1.5813677479673616,
"learning_rate": 0.00017614821163876486,
"loss_nats": 1.0961205959320068,
"step": 4000,
"unclipped_grad_norm": 0.7436221837997437
},
{
"bits_per_byte": 1.6793110077204767,
"learning_rate": 0.00017559197792325906,
"loss_nats": 1.164009690284729,
"step": 4010,
"unclipped_grad_norm": 0.8044730424880981
},
{
"bits_per_byte": 1.6584589712166156,
"learning_rate": 0.0001750355632200748,
"loss_nats": 1.1495561599731445,
"step": 4020,
"unclipped_grad_norm": 0.8183668851852417
},
{
"bits_per_byte": 1.6850787898824628,
"learning_rate": 0.00017447897703680418,
"loss_nats": 1.1680076122283936,
"step": 4030,
"unclipped_grad_norm": 0.793128252029419
},
{
"bits_per_byte": 1.617821878574846,
"learning_rate": 0.0001739222288839694,
"loss_nats": 1.1213886737823486,
"step": 4040,
"unclipped_grad_norm": 0.7582541108131409
},
{
"bits_per_byte": 1.6450140555027646,
"learning_rate": 0.00017336532827486026,
"loss_nats": 1.1402368545532227,
"step": 4050,
"unclipped_grad_norm": 0.7559582591056824
},
{
"bits_per_byte": 1.6581645369183384,
"learning_rate": 0.00017280828472537163,
"loss_nats": 1.1493520736694336,
"step": 4060,
"unclipped_grad_norm": 0.743245005607605
},
{
"bits_per_byte": 1.680131880913045,
"learning_rate": 0.0001722511077538408,
"loss_nats": 1.1645786762237549,
"step": 4070,
"unclipped_grad_norm": 0.7624887824058533
},
{
"bits_per_byte": 1.7564567494700543,
"learning_rate": 0.00017169380688088497,
"loss_nats": 1.2174830436706543,
"step": 4080,
"unclipped_grad_norm": 0.8132302761077881
},
{
"bits_per_byte": 1.6669108866106175,
"learning_rate": 0.0001711363916292384,
"loss_nats": 1.1554145812988281,
"step": 4090,
"unclipped_grad_norm": 0.8276432752609253
},
{
"bits_per_byte": 1.638365894151012,
"learning_rate": 0.00017057887152358974,
"loss_nats": 1.1356287002563477,
"step": 4100,
"unclipped_grad_norm": 0.7751885652542114
},
{
"bits_per_byte": 1.6303674970073352,
"learning_rate": 0.0001700212560904194,
"loss_nats": 1.1300846338272095,
"step": 4110,
"unclipped_grad_norm": 0.7898787260055542
},
{
"bits_per_byte": 1.6896222275529564,
"learning_rate": 0.00016946355485783656,
"loss_nats": 1.171156883239746,
"step": 4120,
"unclipped_grad_norm": 0.8148247599601746
},
{
"bits_per_byte": 1.6213521664491068,
"learning_rate": 0.0001689057773554166,
"loss_nats": 1.1238356828689575,
"step": 4130,
"unclipped_grad_norm": 0.8133904933929443
},
{
"bits_per_byte": 1.6881359534842002,
"learning_rate": 0.00016834793311403806,
"loss_nats": 1.1701266765594482,
"step": 4140,
"unclipped_grad_norm": 0.8217440247535706
},
{
"bits_per_byte": 1.6129334437067215,
"learning_rate": 0.00016779003166571981,
"loss_nats": 1.1180002689361572,
"step": 4150,
"unclipped_grad_norm": 0.7947918176651001
},
{
"bits_per_byte": 1.651558683298698,
"learning_rate": 0.00016723208254345838,
"loss_nats": 1.144773244857788,
"step": 4160,
"unclipped_grad_norm": 0.7860805988311768
},
{
"bits_per_byte": 1.6828089628563754,
"learning_rate": 0.00016667409528106481,
"loss_nats": 1.1664342880249023,
"step": 4170,
"unclipped_grad_norm": 0.8207855224609375
},
{
"bits_per_byte": 1.7311515661874024,
"learning_rate": 0.00016611607941300184,
"loss_nats": 1.1999428272247314,
"step": 4180,
"unclipped_grad_norm": 0.8373809456825256
},
{
"bits_per_byte": 1.6433404923272186,
"learning_rate": 0.0001655580444742211,
"loss_nats": 1.139076828956604,
"step": 4190,
"unclipped_grad_norm": 0.7621071338653564
},
{
"bits_per_byte": 1.682679287937625,
"learning_rate": 0.00016499999999999997,
"loss_nats": 1.166344404220581,
"step": 4200,
"unclipped_grad_norm": 0.7672834396362305
},
{
"bits_per_byte": 1.661693964879345,
"learning_rate": 0.0001644419555257789,
"loss_nats": 1.1517984867095947,
"step": 4210,
"unclipped_grad_norm": 0.7330533862113953
},
{
"bits_per_byte": 1.6972419188994332,
"learning_rate": 0.0001638839205869981,
"loss_nats": 1.1764384508132935,
"step": 4220,
"unclipped_grad_norm": 0.7866858243942261
},
{
"bits_per_byte": 1.6670219874030747,
"learning_rate": 0.0001633259047189352,
"loss_nats": 1.155491590499878,
"step": 4230,
"unclipped_grad_norm": 0.7902593612670898
},
{
"bits_per_byte": 1.6114956687455084,
"learning_rate": 0.0001627679174565416,
"loss_nats": 1.1170036792755127,
"step": 4240,
"unclipped_grad_norm": 0.740251362323761
},
{
"bits_per_byte": 1.6335233786506678,
"learning_rate": 0.00016220996833428016,
"loss_nats": 1.1322721242904663,
"step": 4250,
"unclipped_grad_norm": 0.7777352333068848
},
{
"bits_per_byte": 1.6115895712728796,
"learning_rate": 0.00016165206688596191,
"loss_nats": 1.1170687675476074,
"step": 4260,
"unclipped_grad_norm": 0.774424135684967
},
{
"bits_per_byte": 1.619611874005026,
"learning_rate": 0.00016109422264458335,
"loss_nats": 1.1226294040679932,
"step": 4270,
"unclipped_grad_norm": 0.7803477048873901
},
{
"bits_per_byte": 1.6299772683725275,
"learning_rate": 0.00016053644514216336,
"loss_nats": 1.1298141479492188,
"step": 4280,
"unclipped_grad_norm": 0.7580373287200928
},
{
"bits_per_byte": 1.6795242662075467,
"learning_rate": 0.00015997874390958058,
"loss_nats": 1.164157509803772,
"step": 4290,
"unclipped_grad_norm": 0.7638103365898132
},
{
"bits_per_byte": 1.600082040120998,
"learning_rate": 0.00015942112847641018,
"loss_nats": 1.109092354774475,
"step": 4300,
"unclipped_grad_norm": 0.7825598120689392
},
{
"bits_per_byte": 1.6771565810531184,
"learning_rate": 0.00015886360837076156,
"loss_nats": 1.1625163555145264,
"step": 4310,
"unclipped_grad_norm": 0.7940689921379089
},
{
"bits_per_byte": 1.6577068910843924,
"learning_rate": 0.00015830619311911495,
"loss_nats": 1.149034857749939,
"step": 4320,
"unclipped_grad_norm": 0.7976187467575073
},
{
"bits_per_byte": 1.6195226150092283,
"learning_rate": 0.00015774889224615915,
"loss_nats": 1.1225675344467163,
"step": 4330,
"unclipped_grad_norm": 0.7892199158668518
},
{
"bits_per_byte": 1.655279183984813,
"learning_rate": 0.00015719171527462834,
"loss_nats": 1.1473520994186401,
"step": 4340,
"unclipped_grad_norm": 0.8144169449806213
},
{
"bits_per_byte": 1.6394932404274176,
"learning_rate": 0.0001566346717251397,
"loss_nats": 1.136410117149353,
"step": 4350,
"unclipped_grad_norm": 0.8119622468948364
},
{
"bits_per_byte": 1.6287904160989233,
"learning_rate": 0.00015607777111603056,
"loss_nats": 1.1289914846420288,
"step": 4360,
"unclipped_grad_norm": 0.7554351091384888
},
{
"bits_per_byte": 1.6321829458698431,
"learning_rate": 0.00015552102296319576,
"loss_nats": 1.1313430070877075,
"step": 4370,
"unclipped_grad_norm": 0.808085024356842
},
{
"bits_per_byte": 1.6820371047193032,
"learning_rate": 0.00015496443677992516,
"loss_nats": 1.1658992767333984,
"step": 4380,
"unclipped_grad_norm": 0.7708722949028015
},
{
"bits_per_byte": 1.6522423143358775,
"learning_rate": 0.0001544080220767409,
"loss_nats": 1.1452471017837524,
"step": 4390,
"unclipped_grad_norm": 0.8123121857643127
},
{
"bits_per_byte": 1.6025224739367414,
"learning_rate": 0.00015385178836123511,
"loss_nats": 1.1107839345932007,
"step": 4400,
"unclipped_grad_norm": 0.7717524766921997
},
{
"bits_per_byte": 1.6853694405624209,
"learning_rate": 0.00015329574513790726,
"loss_nats": 1.1682090759277344,
"step": 4410,
"unclipped_grad_norm": 0.81525057554245
},
{
"bits_per_byte": 1.6142167782474597,
"learning_rate": 0.00015273990190800186,
"loss_nats": 1.1188898086547852,
"step": 4420,
"unclipped_grad_norm": 0.7956154346466064
},
{
"bits_per_byte": 1.6755413199962148,
"learning_rate": 0.0001521842681693461,
"loss_nats": 1.1613967418670654,
"step": 4430,
"unclipped_grad_norm": 0.7907513380050659
},
{
"bits_per_byte": 1.6011154838700328,
"learning_rate": 0.00015162885341618752,
"loss_nats": 1.1098086833953857,
"step": 4440,
"unclipped_grad_norm": 0.7607184648513794
},
{
"bits_per_byte": 1.5990663105850023,
"learning_rate": 0.00015107366713903178,
"loss_nats": 1.1083883047103882,
"step": 4450,
"unclipped_grad_norm": 0.7442869544029236
},
{
"bits_per_byte": 1.5859970049806358,
"learning_rate": 0.0001505187188244806,
"loss_nats": 1.0993293523788452,
"step": 4460,
"unclipped_grad_norm": 0.7537500262260437
},
{
"bits_per_byte": 1.6451918855637568,
"learning_rate": 0.00014996401795506952,
"loss_nats": 1.1403601169586182,
"step": 4470,
"unclipped_grad_norm": 0.7931647896766663
},
{
"bits_per_byte": 1.6505943765753102,
"learning_rate": 0.00014940957400910585,
"loss_nats": 1.1441048383712769,
"step": 4480,
"unclipped_grad_norm": 0.9106901288032532
},
{
"bits_per_byte": 1.6301040195862135,
"learning_rate": 0.00014885539646050699,
"loss_nats": 1.1299020051956177,
"step": 4490,
"unclipped_grad_norm": 0.7690432667732239
},
{
"bits_per_byte": 1.6520481459230536,
"learning_rate": 0.00014830149477863805,
"loss_nats": 1.1451125144958496,
"step": 4500,
"unclipped_grad_norm": 0.7990603446960449
},
{
"bits_per_byte": 1.5968599451570833,
"learning_rate": 0.00014774787842815066,
"loss_nats": 1.1068589687347412,
"step": 4510,
"unclipped_grad_norm": 0.8130239248275757
},
{
"bits_per_byte": 1.6430763269754933,
"learning_rate": 0.00014719455686882066,
"loss_nats": 1.138893723487854,
"step": 4520,
"unclipped_grad_norm": 0.7745712399482727
},
{
"bits_per_byte": 1.660709880151108,
"learning_rate": 0.00014664153955538695,
"loss_nats": 1.1511163711547852,
"step": 4530,
"unclipped_grad_norm": 0.7695059776306152
},
{
"bits_per_byte": 1.707295852686222,
"learning_rate": 0.00014608883593738946,
"loss_nats": 1.1834073066711426,
"step": 4540,
"unclipped_grad_norm": 0.8436031937599182
},
{
"bits_per_byte": 1.5623325361872151,
"learning_rate": 0.00014553645545900817,
"loss_nats": 1.0829263925552368,
"step": 4550,
"unclipped_grad_norm": 0.7549967169761658
},
{
"bits_per_byte": 1.6257084869954619,
"learning_rate": 0.0001449844075589013,
"loss_nats": 1.1268552541732788,
"step": 4560,
"unclipped_grad_norm": 0.8449703454971313
},
{
"bits_per_byte": 1.6175644205465043,
"learning_rate": 0.0001444327016700444,
"loss_nats": 1.1212102174758911,
"step": 4570,
"unclipped_grad_norm": 0.7528937458992004
},
{
"bits_per_byte": 1.570274179056107,
"learning_rate": 0.0001438813472195688,
"loss_nats": 1.0884311199188232,
"step": 4580,
"unclipped_grad_norm": 0.790579617023468
},
{
"bits_per_byte": 1.609637052237634,
"learning_rate": 0.00014333035362860093,
"loss_nats": 1.1157153844833374,
"step": 4590,
"unclipped_grad_norm": 0.7935193181037903
},
{
"bits_per_byte": 1.6088013885370918,
"learning_rate": 0.00014277973031210088,
"loss_nats": 1.1151361465454102,
"step": 4600,
"unclipped_grad_norm": 0.7915687561035156
},
{
"bits_per_byte": 1.6615916352020816,
"learning_rate": 0.00014222948667870207,
"loss_nats": 1.151727557182312,
"step": 4610,
"unclipped_grad_norm": 0.8493216633796692
},
{
"bits_per_byte": 1.6675298521710726,
"learning_rate": 0.00014167963213054991,
"loss_nats": 1.1558436155319214,
"step": 4620,
"unclipped_grad_norm": 0.8179065585136414
},
{
"bits_per_byte": 1.6286588493710132,
"learning_rate": 0.00014113017606314152,
"loss_nats": 1.1289002895355225,
"step": 4630,
"unclipped_grad_norm": 0.8027536273002625
},
{
"bits_per_byte": 1.7031861552612044,
"learning_rate": 0.00014058112786516518,
"loss_nats": 1.180558681488037,
"step": 4640,
"unclipped_grad_norm": 0.8179929256439209
},
{
"bits_per_byte": 1.6059254946493637,
"learning_rate": 0.0001400324969183396,
"loss_nats": 1.113142728805542,
"step": 4650,
"unclipped_grad_norm": 0.8228299021720886
},
{
"bits_per_byte": 1.5739184914278899,
"learning_rate": 0.00013948429259725406,
"loss_nats": 1.0909571647644043,
"step": 4660,
"unclipped_grad_norm": 0.820624589920044
},
{
"bits_per_byte": 1.7237126286069824,
"learning_rate": 0.00013893652426920776,
"loss_nats": 1.194786548614502,
"step": 4670,
"unclipped_grad_norm": 0.8582949638366699
},
{
"bits_per_byte": 1.7009104808249877,
"learning_rate": 0.00013838920129405016,
"loss_nats": 1.1789813041687012,
"step": 4680,
"unclipped_grad_norm": 0.8158925175666809
},
{
"bits_per_byte": 1.6679322915740917,
"learning_rate": 0.0001378423330240207,
"loss_nats": 1.1561225652694702,
"step": 4690,
"unclipped_grad_norm": 0.849236249923706
},
{
"bits_per_byte": 1.589747430648001,
"learning_rate": 0.00013729592880358936,
"loss_nats": 1.101928949356079,
"step": 4700,
"unclipped_grad_norm": 0.7403190732002258
},
{
"bits_per_byte": 1.6742342518496576,
"learning_rate": 0.00013674999796929659,
"loss_nats": 1.1604907512664795,
"step": 4710,
"unclipped_grad_norm": 0.8582251071929932
},
{
"bits_per_byte": 1.607032031025015,
"learning_rate": 0.00013620454984959417,
"loss_nats": 1.1139097213745117,
"step": 4720,
"unclipped_grad_norm": 0.8400667905807495
},
{
"bits_per_byte": 1.5176990826873464,
"learning_rate": 0.00013565959376468542,
"loss_nats": 1.0519888401031494,
"step": 4730,
"unclipped_grad_norm": 0.7827285528182983
},
{
"bits_per_byte": 1.5868581221135059,
"learning_rate": 0.00013511513902636634,
"loss_nats": 1.099926233291626,
"step": 4740,
"unclipped_grad_norm": 0.7815871238708496
},
{
"bits_per_byte": 1.6377919880450826,
"learning_rate": 0.0001345711949378662,
"loss_nats": 1.1352308988571167,
"step": 4750,
"unclipped_grad_norm": 0.7846289873123169
},
{
"bits_per_byte": 1.6125860387519786,
"learning_rate": 0.00013402777079368853,
"loss_nats": 1.1177594661712646,
"step": 4760,
"unclipped_grad_norm": 0.8307337164878845
},
{
"bits_per_byte": 1.5936545325103024,
"learning_rate": 0.0001334848758794528,
"loss_nats": 1.1046371459960938,
"step": 4770,
"unclipped_grad_norm": 0.8213374614715576
},
{
"bits_per_byte": 1.6121588338472355,
"learning_rate": 0.00013294251947173504,
"loss_nats": 1.1174633502960205,
"step": 4780,
"unclipped_grad_norm": 0.8224621415138245
},
{
"bits_per_byte": 1.5705015401205475,
"learning_rate": 0.00013240071083790996,
"loss_nats": 1.0885887145996094,
"step": 4790,
"unclipped_grad_norm": 0.8122685551643372
},
{
"bits_per_byte": 1.5722774329733569,
"learning_rate": 0.0001318594592359921,
"loss_nats": 1.0898196697235107,
"step": 4800,
"unclipped_grad_norm": 0.8455460667610168
},
{
"bits_per_byte": 1.6480244398234671,
"learning_rate": 0.0001313187739144781,
"loss_nats": 1.1423234939575195,
"step": 4810,
"unclipped_grad_norm": 0.8817458748817444
},
{
"bits_per_byte": 1.574972745077679,
"learning_rate": 0.0001307786641121882,
"loss_nats": 1.0916879177093506,
"step": 4820,
"unclipped_grad_norm": 0.7954387664794922
},
{
"bits_per_byte": 1.608869665649484,
"learning_rate": 0.00013023913905810876,
"loss_nats": 1.1151834726333618,
"step": 4830,
"unclipped_grad_norm": 0.8687117695808411
},
{
"bits_per_byte": 1.5985519104762718,
"learning_rate": 0.00012970020797123432,
"loss_nats": 1.1080317497253418,
"step": 4840,
"unclipped_grad_norm": 0.8505465388298035
},
{
"bits_per_byte": 1.6228387844831649,
"learning_rate": 0.00012916188006041027,
"loss_nats": 1.1248661279678345,
"step": 4850,
"unclipped_grad_norm": 0.7912812232971191
},
{
"bits_per_byte": 1.5432041098103009,
"learning_rate": 0.00012862416452417516,
"loss_nats": 1.0696675777435303,
"step": 4860,
"unclipped_grad_norm": 0.7933070063591003
},
{
"bits_per_byte": 1.635918581029234,
"learning_rate": 0.00012808707055060405,
"loss_nats": 1.13393235206604,
"step": 4870,
"unclipped_grad_norm": 0.8805916905403137
},
{
"bits_per_byte": 1.4766798445953597,
"learning_rate": 0.000127550607317151,
"loss_nats": 1.0235564708709717,
"step": 4880,
"unclipped_grad_norm": 0.8066675066947937
},
{
"bits_per_byte": 1.5344280071194227,
"learning_rate": 0.00012701478399049248,
"loss_nats": 1.0635844469070435,
"step": 4890,
"unclipped_grad_norm": 0.8088182806968689
},
{
"bits_per_byte": 1.6101516243290155,
"learning_rate": 0.0001264796097263708,
"loss_nats": 1.1160720586776733,
"step": 4900,
"unclipped_grad_norm": 0.8653903603553772
},
{
"bits_per_byte": 1.6779198400576476,
"learning_rate": 0.0001259450936694375,
"loss_nats": 1.1630454063415527,
"step": 4910,
"unclipped_grad_norm": 0.8391038179397583
},
{
"bits_per_byte": 1.6284904783558185,
"learning_rate": 0.00012541124495309727,
"loss_nats": 1.1287835836410522,
"step": 4920,
"unclipped_grad_norm": 0.8723531365394592
},
{
"bits_per_byte": 1.5831391692712484,
"learning_rate": 0.00012487807269935167,
"loss_nats": 1.0973484516143799,
"step": 4930,
"unclipped_grad_norm": 0.7553638815879822
},
{
"bits_per_byte": 1.5755332365368409,
"learning_rate": 0.00012434558601864349,
"loss_nats": 1.0920764207839966,
"step": 4940,
"unclipped_grad_norm": 0.8158645033836365
},
{
"bits_per_byte": 1.530142715407874,
"learning_rate": 0.0001238137940097009,
"loss_nats": 1.0606141090393066,
"step": 4950,
"unclipped_grad_norm": 0.7773033380508423
},
{
"bits_per_byte": 1.5220387209165676,
"learning_rate": 0.0001232827057593821,
"loss_nats": 1.0549968481063843,
"step": 4960,
"unclipped_grad_norm": 0.8325244784355164
},
{
"bits_per_byte": 1.6031579498316795,
"learning_rate": 0.00012275233034251992,
"loss_nats": 1.1112244129180908,
"step": 4970,
"unclipped_grad_norm": 0.8591333627700806
},
{
"bits_per_byte": 1.6532000857185325,
"learning_rate": 0.0001222226768217669,
"loss_nats": 1.1459109783172607,
"step": 4980,
"unclipped_grad_norm": 0.8351151347160339
},
{
"bits_per_byte": 1.585272098107249,
"learning_rate": 0.00012169375424744034,
"loss_nats": 1.0988268852233887,
"step": 4990,
"unclipped_grad_norm": 0.8779996037483215
},
{
"bits_per_byte": 1.5394700224947675,
"learning_rate": 0.00012116557165736772,
"loss_nats": 1.0670793056488037,
"step": 5000,
"unclipped_grad_norm": 0.7928086519241333
},
{
"bits_per_byte": 1.5552827953456976,
"learning_rate": 0.00012063813807673214,
"loss_nats": 1.0780398845672607,
"step": 5010,
"unclipped_grad_norm": 0.8201308846473694
},
{
"bits_per_byte": 1.5898205232746179,
"learning_rate": 0.00012011146251791832,
"loss_nats": 1.1019796133041382,
"step": 5020,
"unclipped_grad_norm": 0.7774789929389954
},
{
"bits_per_byte": 1.5697566832596608,
"learning_rate": 0.0001195855539803584,
"loss_nats": 1.088072419166565,
"step": 5030,
"unclipped_grad_norm": 0.7855319380760193
},
{
"bits_per_byte": 1.539816567536256,
"learning_rate": 0.00011906042145037817,
"loss_nats": 1.0673195123672485,
"step": 5040,
"unclipped_grad_norm": 0.7920348048210144
},
{
"bits_per_byte": 1.5092143186070297,
"learning_rate": 0.0001185360739010438,
"loss_nats": 1.0461076498031616,
"step": 5050,
"unclipped_grad_norm": 0.8045132160186768
},
{
"bits_per_byte": 1.5339026001210367,
"learning_rate": 0.00011801252029200808,
"loss_nats": 1.0632202625274658,
"step": 5060,
"unclipped_grad_norm": 0.8044583797454834
},
{
"bits_per_byte": 1.6138325690054323,
"learning_rate": 0.00011748976956935777,
"loss_nats": 1.1186234951019287,
"step": 5070,
"unclipped_grad_norm": 0.8723475337028503
},
{
"bits_per_byte": 1.574335549356232,
"learning_rate": 0.00011696783066546024,
"loss_nats": 1.091246247291565,
"step": 5080,
"unclipped_grad_norm": 0.8741123080253601
},
{
"bits_per_byte": 1.5594206979254568,
"learning_rate": 0.00011644671249881147,
"loss_nats": 1.0809080600738525,
"step": 5090,
"unclipped_grad_norm": 0.8465037941932678
},
{
"bits_per_byte": 1.5640659493252613,
"learning_rate": 0.00011592642397388293,
"loss_nats": 1.0841279029846191,
"step": 5100,
"unclipped_grad_norm": 0.783299446105957
},
{
"bits_per_byte": 1.488738924108558,
"learning_rate": 0.00011540697398097015,
"loss_nats": 1.0319151878356934,
"step": 5110,
"unclipped_grad_norm": 0.8348572254180908
},
{
"bits_per_byte": 1.5847128105266433,
"learning_rate": 0.00011488837139604016,
"loss_nats": 1.0984392166137695,
"step": 5120,
"unclipped_grad_norm": 0.8110998868942261
},
{
"bits_per_byte": 1.590178763136365,
"learning_rate": 0.00011437062508058039,
"loss_nats": 1.1022279262542725,
"step": 5130,
"unclipped_grad_norm": 0.8208310604095459
},
{
"bits_per_byte": 1.5494388248693793,
"learning_rate": 0.00011385374388144673,
"loss_nats": 1.0739891529083252,
"step": 5140,
"unclipped_grad_norm": 0.8678956627845764
},
{
"bits_per_byte": 1.6273239200350162,
"learning_rate": 0.00011333773663071286,
"loss_nats": 1.1279749870300293,
"step": 5150,
"unclipped_grad_norm": 0.848126232624054
},
{
"bits_per_byte": 1.5085432423033631,
"learning_rate": 0.00011282261214551897,
"loss_nats": 1.0456424951553345,
"step": 5160,
"unclipped_grad_norm": 0.7922812104225159
},
{
"bits_per_byte": 1.6700107019097687,
"learning_rate": 0.00011230837922792112,
"loss_nats": 1.1575632095336914,
"step": 5170,
"unclipped_grad_norm": 0.8759046196937561
},
{
"bits_per_byte": 1.5869253673299932,
"learning_rate": 0.0001117950466647412,
"loss_nats": 1.0999728441238403,
"step": 5180,
"unclipped_grad_norm": 0.8647724390029907
},
{
"bits_per_byte": 1.5237559676854329,
"learning_rate": 0.00011128262322741627,
"loss_nats": 1.0561871528625488,
"step": 5190,
"unclipped_grad_norm": 0.813337504863739
},
{
"bits_per_byte": 1.556985939537192,
"learning_rate": 0.00011077111767184913,
"loss_nats": 1.0792204141616821,
"step": 5200,
"unclipped_grad_norm": 0.8388134241104126
},
{
"bits_per_byte": 1.5602121620847276,
"learning_rate": 0.00011026053873825837,
"loss_nats": 1.0814566612243652,
"step": 5210,
"unclipped_grad_norm": 0.8162683248519897
},
{
"bits_per_byte": 1.6432956048553435,
"learning_rate": 0.0001097508951510293,
"loss_nats": 1.1390457153320312,
"step": 5220,
"unclipped_grad_norm": 0.8667425513267517
},
{
"bits_per_byte": 1.5501931407760636,
"learning_rate": 0.0001092421956185646,
"loss_nats": 1.074512004852295,
"step": 5230,
"unclipped_grad_norm": 0.8668702840805054
},
{
"bits_per_byte": 1.5510165937083948,
"learning_rate": 0.00010873444883313581,
"loss_nats": 1.075082778930664,
"step": 5240,
"unclipped_grad_norm": 0.8049055337905884
},
{
"bits_per_byte": 1.5393878147876552,
"learning_rate": 0.00010822766347073447,
"loss_nats": 1.0670223236083984,
"step": 5250,
"unclipped_grad_norm": 0.8209428191184998
},
{
"bits_per_byte": 1.540701762240247,
"learning_rate": 0.00010772184819092424,
"loss_nats": 1.0679330825805664,
"step": 5260,
"unclipped_grad_norm": 0.8524174690246582
},
{
"bits_per_byte": 1.5376201771020872,
"learning_rate": 0.0001072170116366926,
"loss_nats": 1.0657970905303955,
"step": 5270,
"unclipped_grad_norm": 0.8064137697219849
},
{
"bits_per_byte": 1.5550475230793173,
"learning_rate": 0.00010671316243430345,
"loss_nats": 1.0778768062591553,
"step": 5280,
"unclipped_grad_norm": 0.8169121146202087
},
{
"bits_per_byte": 1.5423493560355128,
"learning_rate": 0.00010621030919314944,
"loss_nats": 1.069075107574463,
"step": 5290,
"unclipped_grad_norm": 0.8263237476348877
},
{
"bits_per_byte": 1.5542627662434303,
"learning_rate": 0.000105708460505605,
"loss_nats": 1.077332854270935,
"step": 5300,
"unclipped_grad_norm": 0.8492859601974487
},
{
"bits_per_byte": 1.5509104804128124,
"learning_rate": 0.00010520762494687964,
"loss_nats": 1.0750092267990112,
"step": 5310,
"unclipped_grad_norm": 0.8885292410850525
},
{
"bits_per_byte": 1.5517654061702515,
"learning_rate": 0.00010470781107487111,
"loss_nats": 1.0756018161773682,
"step": 5320,
"unclipped_grad_norm": 0.8326203227043152
},
{
"bits_per_byte": 1.5477100552629064,
"learning_rate": 0.00010420902743001955,
"loss_nats": 1.0727908611297607,
"step": 5330,
"unclipped_grad_norm": 0.8806997537612915
},
{
"bits_per_byte": 1.503639328996661,
"learning_rate": 0.00010371128253516117,
"loss_nats": 1.0422433614730835,
"step": 5340,
"unclipped_grad_norm": 0.8234722018241882
},
{
"bits_per_byte": 1.5869049013945407,
"learning_rate": 0.000103214584895383,
"loss_nats": 1.0999586582183838,
"step": 5350,
"unclipped_grad_norm": 0.8706242442131042
},
{
"bits_per_byte": 1.5072631754329913,
"learning_rate": 0.00010271894299787715,
"loss_nats": 1.044755220413208,
"step": 5360,
"unclipped_grad_norm": 0.8708282709121704
},
{
"bits_per_byte": 1.471025742965594,
"learning_rate": 0.00010222436531179622,
"loss_nats": 1.0196373462677002,
"step": 5370,
"unclipped_grad_norm": 0.8303158283233643
},
{
"bits_per_byte": 1.5968417149960918,
"learning_rate": 0.0001017308602881082,
"loss_nats": 1.1068463325500488,
"step": 5380,
"unclipped_grad_norm": 0.883955180644989
},
{
"bits_per_byte": 1.531507741707772,
"learning_rate": 0.00010123843635945242,
"loss_nats": 1.0615602731704712,
"step": 5390,
"unclipped_grad_norm": 0.8507125377655029
},
{
"bits_per_byte": 1.5483677169198058,
"learning_rate": 0.00010074710193999505,
"loss_nats": 1.073246717453003,
"step": 5400,
"unclipped_grad_norm": 0.8522453904151917
},
{
"bits_per_byte": 1.571479089508052,
"learning_rate": 0.0001002568654252858,
"loss_nats": 1.089266300201416,
"step": 5410,
"unclipped_grad_norm": 0.8313266038894653
},
{
"bits_per_byte": 1.5408727129952047,
"learning_rate": 9.976773519211404e-05,
"loss_nats": 1.0680515766143799,
"step": 5420,
"unclipped_grad_norm": 0.8902517557144165
},
{
"bits_per_byte": 1.5778336764747818,
"learning_rate": 9.92797195983659e-05,
"loss_nats": 1.0936709642410278,
"step": 5430,
"unclipped_grad_norm": 0.8956695795059204
},
{
"bits_per_byte": 1.513927503153925,
"learning_rate": 9.879282698288142e-05,
"loss_nats": 1.0493745803833008,
"step": 5440,
"unclipped_grad_norm": 0.897573709487915
},
{
"bits_per_byte": 1.5078418970531444,
"learning_rate": 9.830706566531197e-05,
"loss_nats": 1.0451563596725464,
"step": 5450,
"unclipped_grad_norm": 0.8440007567405701
},
{
"bits_per_byte": 1.5030907043404085,
"learning_rate": 9.782244394597826e-05,
"loss_nats": 1.0418630838394165,
"step": 5460,
"unclipped_grad_norm": 0.8436890244483948
},
{
"bits_per_byte": 1.5464133060754008,
"learning_rate": 9.733897010572829e-05,
"loss_nats": 1.0718920230865479,
"step": 5470,
"unclipped_grad_norm": 0.8681005239486694
},
{
"bits_per_byte": 1.553998600891705,
"learning_rate": 9.685665240579612e-05,
"loss_nats": 1.077149748802185,
"step": 5480,
"unclipped_grad_norm": 0.8523774147033691
},
{
"bits_per_byte": 1.543775436176467,
"learning_rate": 9.637549908766041e-05,
"loss_nats": 1.070063591003418,
"step": 5490,
"unclipped_grad_norm": 0.8906207084655762
},
{
"bits_per_byte": 1.5444891641775478,
"learning_rate": 9.589551837290389e-05,
"loss_nats": 1.0705583095550537,
"step": 5500,
"unclipped_grad_norm": 0.8658390045166016
},
{
"bits_per_byte": 1.6370650173798855,
"learning_rate": 9.541671846307257e-05,
"loss_nats": 1.1347270011901855,
"step": 5510,
"unclipped_grad_norm": 0.9161565899848938
},
{
"bits_per_byte": 1.5520985365649729,
"learning_rate": 9.493910753953597e-05,
"loss_nats": 1.075832724571228,
"step": 5520,
"unclipped_grad_norm": 0.845907986164093
},
{
"bits_per_byte": 1.5215033389244317,
"learning_rate": 9.446269376334689e-05,
"loss_nats": 1.0546257495880127,
"step": 5530,
"unclipped_grad_norm": 0.8585748672485352
},
{
"bits_per_byte": 1.5943249208833656,
"learning_rate": 9.39874852751024e-05,
"loss_nats": 1.1051018238067627,
"step": 5540,
"unclipped_grad_norm": 0.9360165596008301
},
{
"bits_per_byte": 1.5176287417831436,
"learning_rate": 9.351349019480433e-05,
"loss_nats": 1.0519400835037231,
"step": 5550,
"unclipped_grad_norm": 0.8308430910110474
},
{
"bits_per_byte": 1.5164790377621256,
"learning_rate": 9.304071662172086e-05,
"loss_nats": 1.0511431694030762,
"step": 5560,
"unclipped_grad_norm": 0.894058346748352
},
{
"bits_per_byte": 1.4905304673825959,
"learning_rate": 9.256917263424787e-05,
"loss_nats": 1.0331569910049438,
"step": 5570,
"unclipped_grad_norm": 0.8399901390075684
},
{
"bits_per_byte": 1.5655759569998369,
"learning_rate": 9.209886628977114e-05,
"loss_nats": 1.085174560546875,
"step": 5580,
"unclipped_grad_norm": 0.8579773902893066
},
{
"bits_per_byte": 1.482571282283316,
"learning_rate": 9.162980562452843e-05,
"loss_nats": 1.0276401042938232,
"step": 5590,
"unclipped_grad_norm": 0.8161733746528625
},
{
"bits_per_byte": 1.5496305855250914,
"learning_rate": 9.116199865347238e-05,
"loss_nats": 1.0741220712661743,
"step": 5600,
"unclipped_grad_norm": 0.8440633416175842
},
{
"bits_per_byte": 1.5466313800766949,
"learning_rate": 9.069545337013333e-05,
"loss_nats": 1.0720431804656982,
"step": 5610,
"unclipped_grad_norm": 0.8384022116661072
},
{
"bits_per_byte": 1.55366271877457,
"learning_rate": 9.0230177746483e-05,
"loss_nats": 1.0769169330596924,
"step": 5620,
"unclipped_grad_norm": 0.8643615245819092
},
{
"bits_per_byte": 1.5359743031333302,
"learning_rate": 8.976617973279797e-05,
"loss_nats": 1.0646562576293945,
"step": 5630,
"unclipped_grad_norm": 0.8559057712554932
},
{
"bits_per_byte": 1.5154779267514533,
"learning_rate": 8.930346725752419e-05,
"loss_nats": 1.050449252128601,
"step": 5640,
"unclipped_grad_norm": 0.8564271330833435
},
{
"bits_per_byte": 1.5399061704973558,
"learning_rate": 8.88420482271411e-05,
"loss_nats": 1.0673816204071045,
"step": 5650,
"unclipped_grad_norm": 0.899230420589447
},
{
"bits_per_byte": 1.6017691899259623,
"learning_rate": 8.838193052602691e-05,
"loss_nats": 1.1102617979049683,
"step": 5660,
"unclipped_grad_norm": 0.8796955943107605
},
{
"bits_per_byte": 1.5203328250026595,
"learning_rate": 8.792312201632352e-05,
"loss_nats": 1.05381441116333,
"step": 5670,
"unclipped_grad_norm": 0.8509314060211182
},
{
"bits_per_byte": 1.5301421994599214,
"learning_rate": 8.746563053780261e-05,
"loss_nats": 1.060613751411438,
"step": 5680,
"unclipped_grad_norm": 0.8624072074890137
},
{
"bits_per_byte": 1.6028342784827556,
"learning_rate": 8.700946390773114e-05,
"loss_nats": 1.1110000610351562,
"step": 5690,
"unclipped_grad_norm": 0.8914096355438232
},
{
"bits_per_byte": 1.526354281574668,
"learning_rate": 8.655462992073833e-05,
"loss_nats": 1.057988166809082,
"step": 5700,
"unclipped_grad_norm": 0.8579604625701904
},
{
"bits_per_byte": 1.5347007716036911,
"learning_rate": 8.610113634868199e-05,
"loss_nats": 1.063773512840271,
"step": 5710,
"unclipped_grad_norm": 0.8810272216796875
},
{
"bits_per_byte": 1.5491378552303696,
"learning_rate": 8.564899094051613e-05,
"loss_nats": 1.0737805366516113,
"step": 5720,
"unclipped_grad_norm": 0.8394830226898193
},
{
"bits_per_byte": 1.5456280332915613,
"learning_rate": 8.519820142215816e-05,
"loss_nats": 1.071347713470459,
"step": 5730,
"unclipped_grad_norm": 0.865943968296051
},
{
"bits_per_byte": 1.5433466834278662,
"learning_rate": 8.474877549635728e-05,
"loss_nats": 1.0697664022445679,
"step": 5740,
"unclipped_grad_norm": 0.886944591999054
},
{
"bits_per_byte": 1.5488365416260579,
"learning_rate": 8.430072084256247e-05,
"loss_nats": 1.0735716819763184,
"step": 5750,
"unclipped_grad_norm": 0.9145897626876831
},
{
"bits_per_byte": 1.5636783004302166,
"learning_rate": 8.385404511679161e-05,
"loss_nats": 1.0838592052459717,
"step": 5760,
"unclipped_grad_norm": 0.9009037017822266
},
{
"bits_per_byte": 1.511110599315443,
"learning_rate": 8.340875595150034e-05,
"loss_nats": 1.0474220514297485,
"step": 5770,
"unclipped_grad_norm": 0.844070315361023
},
{
"bits_per_byte": 1.494012772097266,
"learning_rate": 8.296486095545203e-05,
"loss_nats": 1.035570740699768,
"step": 5780,
"unclipped_grad_norm": 0.8943845629692078
},
{
"bits_per_byte": 1.5135893852623288,
"learning_rate": 8.252236771358729e-05,
"loss_nats": 1.049140214920044,
"step": 5790,
"unclipped_grad_norm": 0.8841454386711121
},
{
"bits_per_byte": 1.4905120652389536,
"learning_rate": 8.208128378689484e-05,
"loss_nats": 1.033144235610962,
"step": 5800,
"unclipped_grad_norm": 0.8580894470214844
},
{
"bits_per_byte": 1.5735937881830608,
"learning_rate": 8.16416167122819e-05,
"loss_nats": 1.0907320976257324,
"step": 5810,
"unclipped_grad_norm": 0.933175265789032
},
{
"bits_per_byte": 1.5042643139498963,
"learning_rate": 8.120337400244578e-05,
"loss_nats": 1.042676568031311,
"step": 5820,
"unclipped_grad_norm": 0.828717052936554
},
{
"bits_per_byte": 1.5156406223391694,
"learning_rate": 8.076656314574521e-05,
"loss_nats": 1.0505620241165161,
"step": 5830,
"unclipped_grad_norm": 0.9511673450469971
},
{
"bits_per_byte": 1.5092205099824607,
"learning_rate": 8.03311916060725e-05,
"loss_nats": 1.0461119413375854,
"step": 5840,
"unclipped_grad_norm": 0.8395604491233826
},
{
"bits_per_byte": 1.477936693807865,
"learning_rate": 7.989726682272606e-05,
"loss_nats": 1.0244276523590088,
"step": 5850,
"unclipped_grad_norm": 0.9143134355545044
},
{
"bits_per_byte": 1.5336375748560571,
"learning_rate": 7.946479621028318e-05,
"loss_nats": 1.063036561012268,
"step": 5860,
"unclipped_grad_norm": 0.9014923572540283
},
{
"bits_per_byte": 1.5402322496033916,
"learning_rate": 7.903378715847343e-05,
"loss_nats": 1.0676076412200928,
"step": 5870,
"unclipped_grad_norm": 0.8728002905845642
},
{
"bits_per_byte": 1.555285031120159,
"learning_rate": 7.860424703205224e-05,
"loss_nats": 1.078041434288025,
"step": 5880,
"unclipped_grad_norm": 0.8936865925788879
},
{
"bits_per_byte": 1.564054942435606,
"learning_rate": 7.817618317067534e-05,
"loss_nats": 1.084120273590088,
"step": 5890,
"unclipped_grad_norm": 0.8752698302268982
},
{
"bits_per_byte": 1.5458715607251832,
"learning_rate": 7.774960288877295e-05,
"loss_nats": 1.071516513824463,
"step": 5900,
"unclipped_grad_norm": 0.8957662582397461
},
{
"bits_per_byte": 1.5568008862048637,
"learning_rate": 7.732451347542522e-05,
"loss_nats": 1.0790921449661255,
"step": 5910,
"unclipped_grad_norm": 0.8808028101921082
},
{
"bits_per_byte": 1.4896764015384112,
"learning_rate": 7.690092219423722e-05,
"loss_nats": 1.0325649976730347,
"step": 5920,
"unclipped_grad_norm": 0.8660398721694946
},
{
"bits_per_byte": 1.5908711652887386,
"learning_rate": 7.647883628321536e-05,
"loss_nats": 1.102707862854004,
"step": 5930,
"unclipped_grad_norm": 0.9604227542877197
},
{
"bits_per_byte": 1.4574508083676907,
"learning_rate": 7.605826295464316e-05,
"loss_nats": 1.010227918624878,
"step": 5940,
"unclipped_grad_norm": 0.8380697965621948
},
{
"bits_per_byte": 1.5131755950043528,
"learning_rate": 7.563920939495851e-05,
"loss_nats": 1.0488533973693848,
"step": 5950,
"unclipped_grad_norm": 0.8392192125320435
},
{
"bits_per_byte": 1.5452194025131114,
"learning_rate": 7.522168276463047e-05,
"loss_nats": 1.0710644721984863,
"step": 5960,
"unclipped_grad_norm": 0.932554304599762
},
{
"bits_per_byte": 1.5213482105733536,
"learning_rate": 7.480569019803719e-05,
"loss_nats": 1.054518222808838,
"step": 5970,
"unclipped_grad_norm": 0.927780270576477
},
{
"bits_per_byte": 1.5601101763727658,
"learning_rate": 7.439123880334388e-05,
"loss_nats": 1.0813859701156616,
"step": 5980,
"unclipped_grad_norm": 0.8920019268989563
},
{
"bits_per_byte": 1.5382338112003657,
"learning_rate": 7.39783356623813e-05,
"loss_nats": 1.0662224292755127,
"step": 5990,
"unclipped_grad_norm": 0.9537093639373779
},
{
"bits_per_byte": 1.4675097296513546,
"learning_rate": 7.356698783052497e-05,
"loss_nats": 1.017200231552124,
"step": 6000,
"unclipped_grad_norm": 0.8773758411407471
},
{
"bits_per_byte": 1.506333093257125,
"learning_rate": 7.315720233657431e-05,
"loss_nats": 1.0441105365753174,
"step": 6010,
"unclipped_grad_norm": 0.8785988092422485
},
{
"bits_per_byte": 1.4794742187065786,
"learning_rate": 7.274898618263286e-05,
"loss_nats": 1.0254933834075928,
"step": 6020,
"unclipped_grad_norm": 0.8875186443328857
},
{
"bits_per_byte": 1.5245675538148544,
"learning_rate": 7.234234634398831e-05,
"loss_nats": 1.056749701499939,
"step": 6030,
"unclipped_grad_norm": 0.88789963722229
},
{
"bits_per_byte": 1.5628784091210537,
"learning_rate": 7.193728976899362e-05,
"loss_nats": 1.083304762840271,
"step": 6040,
"unclipped_grad_norm": 0.9149079322814941
},
{
"bits_per_byte": 1.4881958028971334,
"learning_rate": 7.153382337894797e-05,
"loss_nats": 1.031538724899292,
"step": 6050,
"unclipped_grad_norm": 0.9299342632293701
},
{
"bits_per_byte": 1.4785307218839452,
"learning_rate": 7.113195406797887e-05,
"loss_nats": 1.0248394012451172,
"step": 6060,
"unclipped_grad_norm": 0.8783867359161377
},
{
"bits_per_byte": 1.5509557118499895,
"learning_rate": 7.073168870292397e-05,
"loss_nats": 1.075040578842163,
"step": 6070,
"unclipped_grad_norm": 0.9337034821510315
},
{
"bits_per_byte": 1.4914187577743023,
"learning_rate": 7.033303412321404e-05,
"loss_nats": 1.0337727069854736,
"step": 6080,
"unclipped_grad_norm": 0.9420101046562195
},
{
"bits_per_byte": 1.4968499698885498,
"learning_rate": 6.993599714075586e-05,
"loss_nats": 1.0375373363494873,
"step": 6090,
"unclipped_grad_norm": 0.8664546012878418
},
{
"bits_per_byte": 1.4906561867003767,
"learning_rate": 6.954058453981609e-05,
"loss_nats": 1.0332441329956055,
"step": 6100,
"unclipped_grad_norm": 0.8687640428543091
},
{
"bits_per_byte": 1.5538661742505406,
"learning_rate": 6.914680307690504e-05,
"loss_nats": 1.077057957649231,
"step": 6110,
"unclipped_grad_norm": 0.8986173868179321
},
{
"bits_per_byte": 1.5476213122150613,
"learning_rate": 6.875465948066139e-05,
"loss_nats": 1.0727293491363525,
"step": 6120,
"unclipped_grad_norm": 0.9189077019691467
},
{
"bits_per_byte": 1.4789527673091625,
"learning_rate": 6.836416045173733e-05,
"loss_nats": 1.0251319408416748,
"step": 6130,
"unclipped_grad_norm": 0.8851786851882935
},
{
"bits_per_byte": 1.4768556108645414,
"learning_rate": 6.797531266268377e-05,
"loss_nats": 1.0236783027648926,
"step": 6140,
"unclipped_grad_norm": 0.9043483138084412
},
{
"bits_per_byte": 1.552988202817886,
"learning_rate": 6.758812275783663e-05,
"loss_nats": 1.0764493942260742,
"step": 6150,
"unclipped_grad_norm": 0.9005390405654907
},
{
"bits_per_byte": 1.3994871515820966,
"learning_rate": 6.720259735320298e-05,
"loss_nats": 0.970050573348999,
"step": 6160,
"unclipped_grad_norm": 0.8554620742797852
},
{
"bits_per_byte": 1.5302097886417105,
"learning_rate": 6.681874303634842e-05,
"loss_nats": 1.0606606006622314,
"step": 6170,
"unclipped_grad_norm": 0.8954014778137207
},
{
"bits_per_byte": 1.4938230752333643,
"learning_rate": 6.643656636628402e-05,
"loss_nats": 1.0354392528533936,
"step": 6180,
"unclipped_grad_norm": 0.892278790473938
},
{
"bits_per_byte": 1.516749394489282,
"learning_rate": 6.605607387335466e-05,
"loss_nats": 1.05133056640625,
"step": 6190,
"unclipped_grad_norm": 0.9185703992843628
},
{
"bits_per_byte": 1.5086576107661869,
"learning_rate": 6.567727205912724e-05,
"loss_nats": 1.0457217693328857,
"step": 6200,
"unclipped_grad_norm": 0.8927735090255737
},
{
"bits_per_byte": 1.5088584865023946,
"learning_rate": 6.530016739627959e-05,
"loss_nats": 1.045861005783081,
"step": 6210,
"unclipped_grad_norm": 0.8906724452972412
},
{
"bits_per_byte": 1.5118191678369979,
"learning_rate": 6.492476632848993e-05,
"loss_nats": 1.0479131937026978,
"step": 6220,
"unclipped_grad_norm": 0.9077256917953491
},
{
"bits_per_byte": 1.4471287536282067,
"learning_rate": 6.455107527032678e-05,
"loss_nats": 1.0030732154846191,
"step": 6230,
"unclipped_grad_norm": 0.8839235305786133
},
{
"bits_per_byte": 1.5076786855174757,
"learning_rate": 6.417910060713926e-05,
"loss_nats": 1.0450432300567627,
"step": 6240,
"unclipped_grad_norm": 0.9515005946159363
},
{
"bits_per_byte": 1.5408861276419719,
"learning_rate": 6.380884869494799e-05,
"loss_nats": 1.0680608749389648,
"step": 6250,
"unclipped_grad_norm": 0.901561975479126
},
{
"bits_per_byte": 1.4200397662908057,
"learning_rate": 6.34403258603367e-05,
"loss_nats": 0.9842965602874756,
"step": 6260,
"unclipped_grad_norm": 0.889621376991272
},
{
"bits_per_byte": 1.5216213190229235,
"learning_rate": 6.30735384003437e-05,
"loss_nats": 1.0547075271606445,
"step": 6270,
"unclipped_grad_norm": 0.9411186575889587
},
{
"bits_per_byte": 1.5224643779774532,
"learning_rate": 6.270849258235484e-05,
"loss_nats": 1.0552918910980225,
"step": 6280,
"unclipped_grad_norm": 0.9406285881996155
},
{
"bits_per_byte": 1.452789390598705,
"learning_rate": 6.234519464399581e-05,
"loss_nats": 1.0069968700408936,
"step": 6290,
"unclipped_grad_norm": 0.8839805126190186
},
{
"bits_per_byte": 1.5074908804627336,
"learning_rate": 6.198365079302614e-05,
"loss_nats": 1.0449130535125732,
"step": 6300,
"unclipped_grad_norm": 0.9184122681617737
},
{
"bits_per_byte": 1.478450577968643,
"learning_rate": 6.16238672072326e-05,
"loss_nats": 1.0247838497161865,
"step": 6310,
"unclipped_grad_norm": 0.8948619365692139
},
{
"bits_per_byte": 1.4214500240278807,
"learning_rate": 6.126585003432406e-05,
"loss_nats": 0.985274076461792,
"step": 6320,
"unclipped_grad_norm": 0.8749227523803711
},
{
"bits_per_byte": 1.5185149683830395,
"learning_rate": 6.090960539182614e-05,
"loss_nats": 1.0525543689727783,
"step": 6330,
"unclipped_grad_norm": 0.9816636443138123
},
{
"bits_per_byte": 1.4759477144506365,
"learning_rate": 6.0555139366976935e-05,
"loss_nats": 1.023048996925354,
"step": 6340,
"unclipped_grad_norm": 0.8950188159942627
},
{
"bits_per_byte": 1.587707716408768,
"learning_rate": 6.020245801662274e-05,
"loss_nats": 1.1005151271820068,
"step": 6350,
"unclipped_grad_norm": 1.01009202003479
},
{
"bits_per_byte": 1.4638088349874379,
"learning_rate": 5.985156736711483e-05,
"loss_nats": 1.0146349668502808,
"step": 6360,
"unclipped_grad_norm": 0.9816907644271851
},
{
"bits_per_byte": 1.4382974445064014,
"learning_rate": 5.950247341420618e-05,
"loss_nats": 0.9969518184661865,
"step": 6370,
"unclipped_grad_norm": 0.8970034122467041
},
{
"bits_per_byte": 1.4818757844432269,
"learning_rate": 5.915518212294935e-05,
"loss_nats": 1.0271580219268799,
"step": 6380,
"unclipped_grad_norm": 0.9058489799499512
},
{
"bits_per_byte": 1.527050123380059,
"learning_rate": 5.880969942759425e-05,
"loss_nats": 1.0584704875946045,
"step": 6390,
"unclipped_grad_norm": 0.929339587688446
},
{
"bits_per_byte": 1.5156593684481134,
"learning_rate": 5.846603123148688e-05,
"loss_nats": 1.0505750179290771,
"step": 6400,
"unclipped_grad_norm": 0.9009133577346802
},
{
"bits_per_byte": 1.4969940913499729,
"learning_rate": 5.812418340696858e-05,
"loss_nats": 1.0376372337341309,
"step": 6410,
"unclipped_grad_norm": 0.957128643989563
},
{
"bits_per_byte": 1.4783625228514012,
"learning_rate": 5.778416179527541e-05,
"loss_nats": 1.0247228145599365,
"step": 6420,
"unclipped_grad_norm": 0.9414587616920471
},
{
"bits_per_byte": 1.523227980947284,
"learning_rate": 5.7445972206438635e-05,
"loss_nats": 1.055821180343628,
"step": 6430,
"unclipped_grad_norm": 0.9234302639961243
},
{
"bits_per_byte": 1.4623229048839832,
"learning_rate": 5.710962041918516e-05,
"loss_nats": 1.013604998588562,
"step": 6440,
"unclipped_grad_norm": 0.9177728295326233
},
{
"bits_per_byte": 1.5306256426914968,
"learning_rate": 5.677511218083908e-05,
"loss_nats": 1.0609488487243652,
"step": 6450,
"unclipped_grad_norm": 0.9453822374343872
},
{
"bits_per_byte": 1.5847898587542297,
"learning_rate": 5.644245320722316e-05,
"loss_nats": 1.0984926223754883,
"step": 6460,
"unclipped_grad_norm": 0.9647909998893738
},
{
"bits_per_byte": 1.5690988496201106,
"learning_rate": 5.61116491825615e-05,
"loss_nats": 1.0876164436340332,
"step": 6470,
"unclipped_grad_norm": 0.9592546820640564
},
{
"bits_per_byte": 1.4544314809491432,
"learning_rate": 5.5782705759382104e-05,
"loss_nats": 1.0081350803375244,
"step": 6480,
"unclipped_grad_norm": 0.9334172010421753
},
{
"bits_per_byte": 1.5573938823850386,
"learning_rate": 5.5455628558420546e-05,
"loss_nats": 1.0795031785964966,
"step": 6490,
"unclipped_grad_norm": 0.967822790145874
},
{
"bits_per_byte": 1.5280063469188563,
"learning_rate": 5.513042316852368e-05,
"loss_nats": 1.0591332912445068,
"step": 6500,
"unclipped_grad_norm": 0.9625112414360046
},
{
"bits_per_byte": 1.4687796495453256,
"learning_rate": 5.480709514655443e-05,
"loss_nats": 1.018080472946167,
"step": 6510,
"unclipped_grad_norm": 0.9510544538497925
},
{
"bits_per_byte": 1.4284500618832987,
"learning_rate": 5.448565001729653e-05,
"loss_nats": 0.9901261329650879,
"step": 6520,
"unclipped_grad_norm": 0.8985839486122131
},
{
"bits_per_byte": 1.4546447394362132,
"learning_rate": 5.41660932733604e-05,
"loss_nats": 1.0082828998565674,
"step": 6530,
"unclipped_grad_norm": 0.907886803150177
},
{
"bits_per_byte": 1.4970680438898438,
"learning_rate": 5.384843037508902e-05,
"loss_nats": 1.0376884937286377,
"step": 6540,
"unclipped_grad_norm": 0.9381445050239563
},
{
"bits_per_byte": 1.4280130539674563,
"learning_rate": 5.353266675046498e-05,
"loss_nats": 0.9898232221603394,
"step": 6550,
"unclipped_grad_norm": 0.9031082391738892
},
{
"bits_per_byte": 1.5210671909218436,
"learning_rate": 5.321880779501729e-05,
"loss_nats": 1.054323434829712,
"step": 6560,
"unclipped_grad_norm": 0.9433532357215881
},
{
"bits_per_byte": 1.479243074023819,
"learning_rate": 5.290685887172962e-05,
"loss_nats": 1.0253331661224365,
"step": 6570,
"unclipped_grad_norm": 0.9311387538909912
},
{
"bits_per_byte": 1.5051876888023787,
"learning_rate": 5.2596825310948295e-05,
"loss_nats": 1.0433166027069092,
"step": 6580,
"unclipped_grad_norm": 0.9569925665855408
},
{
"bits_per_byte": 1.494823842278735,
"learning_rate": 5.228871241029154e-05,
"loss_nats": 1.0361329317092896,
"step": 6590,
"unclipped_grad_norm": 0.8958868384361267
},
{
"bits_per_byte": 1.4302109062541577,
"learning_rate": 5.198252543455865e-05,
"loss_nats": 0.9913466572761536,
"step": 6600,
"unclipped_grad_norm": 0.9136191010475159
},
{
"bits_per_byte": 1.436879103584736,
"learning_rate": 5.167826961564027e-05,
"loss_nats": 0.9959686994552612,
"step": 6610,
"unclipped_grad_norm": 0.8710504770278931
},
{
"bits_per_byte": 1.4278971376607748,
"learning_rate": 5.137595015242884e-05,
"loss_nats": 0.9897428750991821,
"step": 6620,
"unclipped_grad_norm": 0.9145034551620483
},
{
"bits_per_byte": 1.4582168190946336,
"learning_rate": 5.1075572210729925e-05,
"loss_nats": 1.010758876800537,
"step": 6630,
"unclipped_grad_norm": 0.8999127149581909
},
{
"bits_per_byte": 1.5240701799885592,
"learning_rate": 5.07771409231737e-05,
"loss_nats": 1.056404948234558,
"step": 6640,
"unclipped_grad_norm": 0.9801992774009705
},
{
"bits_per_byte": 1.5169175935218258,
"learning_rate": 5.048066138912756e-05,
"loss_nats": 1.0514471530914307,
"step": 6650,
"unclipped_grad_norm": 0.8882671594619751
},
{
"bits_per_byte": 1.5152221885496202,
"learning_rate": 5.018613867460863e-05,
"loss_nats": 1.050271987915039,
"step": 6660,
"unclipped_grad_norm": 0.9029646515846252
},
{
"bits_per_byte": 1.4896082964086694,
"learning_rate": 4.989357781219755e-05,
"loss_nats": 1.0325177907943726,
"step": 6670,
"unclipped_grad_norm": 0.9797500371932983
},
{
"bits_per_byte": 1.4764526555135697,
"learning_rate": 4.960298380095222e-05,
"loss_nats": 1.023398995399475,
"step": 6680,
"unclipped_grad_norm": 0.9356769919395447
},
{
"bits_per_byte": 1.4958203097578342,
"learning_rate": 4.931436160632256e-05,
"loss_nats": 1.0368236303329468,
"step": 6690,
"unclipped_grad_norm": 0.939789891242981
},
{
"bits_per_byte": 1.5480882451121538,
"learning_rate": 4.9027716160065485e-05,
"loss_nats": 1.073053002357483,
"step": 6700,
"unclipped_grad_norm": 1.0080333948135376
},
{
"bits_per_byte": 1.525887864625528,
"learning_rate": 4.874305236016084e-05,
"loss_nats": 1.0576648712158203,
"step": 6710,
"unclipped_grad_norm": 0.9197862148284912
},
{
"bits_per_byte": 1.534906462854123,
"learning_rate": 4.846037507072752e-05,
"loss_nats": 1.0639160871505737,
"step": 6720,
"unclipped_grad_norm": 0.9543269872665405
},
{
"bits_per_byte": 1.4820493149379474,
"learning_rate": 4.817968912194049e-05,
"loss_nats": 1.0272783041000366,
"step": 6730,
"unclipped_grad_norm": 0.9817740321159363
},
{
"bits_per_byte": 1.4384499931177168,
"learning_rate": 4.790099930994813e-05,
"loss_nats": 0.9970575571060181,
"step": 6740,
"unclipped_grad_norm": 0.9710240364074707
},
{
"bits_per_byte": 1.511069151496585,
"learning_rate": 4.762431039679047e-05,
"loss_nats": 1.0473933219909668,
"step": 6750,
"unclipped_grad_norm": 0.9612263441085815
},
{
"bits_per_byte": 1.4780942299160553,
"learning_rate": 4.734962711031755e-05,
"loss_nats": 1.0245368480682373,
"step": 6760,
"unclipped_grad_norm": 0.9238441586494446
},
{
"bits_per_byte": 1.3767874194686909,
"learning_rate": 4.707695414410892e-05,
"loss_nats": 0.9543163180351257,
"step": 6770,
"unclipped_grad_norm": 0.9215719699859619
},
{
"bits_per_byte": 1.4785716537548506,
"learning_rate": 4.680629615739322e-05,
"loss_nats": 1.0248677730560303,
"step": 6780,
"unclipped_grad_norm": 0.9403936266899109
},
{
"bits_per_byte": 1.4459957319243224,
"learning_rate": 4.653765777496863e-05,
"loss_nats": 1.0022878646850586,
"step": 6790,
"unclipped_grad_norm": 0.9651554822921753
},
{
"bits_per_byte": 1.5175859181030786,
"learning_rate": 4.6271043587123986e-05,
"loss_nats": 1.051910400390625,
"step": 6800,
"unclipped_grad_norm": 0.9926899075508118
},
{
"bits_per_byte": 1.5110634760691066,
"learning_rate": 4.600645814956006e-05,
"loss_nats": 1.0473893880844116,
"step": 6810,
"unclipped_grad_norm": 0.9793192148208618
},
{
"bits_per_byte": 1.5052492585913877,
"learning_rate": 4.574390598331204e-05,
"loss_nats": 1.0433592796325684,
"step": 6820,
"unclipped_grad_norm": 1.0081095695495605
},
{
"bits_per_byte": 1.466343515295854,
"learning_rate": 4.548339157467199e-05,
"loss_nats": 1.0163918733596802,
"step": 6830,
"unclipped_grad_norm": 0.960361659526825
},
{
"bits_per_byte": 1.4376153613130795,
"learning_rate": 4.52249193751124e-05,
"loss_nats": 0.9964790344238281,
"step": 6840,
"unclipped_grad_norm": 0.9702624678611755
},
{
"bits_per_byte": 1.4922177891702109,
"learning_rate": 4.4968493801209985e-05,
"loss_nats": 1.0343265533447266,
"step": 6850,
"unclipped_grad_norm": 0.9893983006477356
},
{
"bits_per_byte": 1.4659078832412185,
"learning_rate": 4.471411923457034e-05,
"loss_nats": 1.016089916229248,
"step": 6860,
"unclipped_grad_norm": 0.9693014621734619
},
{
"bits_per_byte": 1.413224953750367,
"learning_rate": 4.446180002175291e-05,
"loss_nats": 0.9795728921890259,
"step": 6870,
"unclipped_grad_norm": 0.9252057075500488
},
{
"bits_per_byte": 1.4200107012228098,
"learning_rate": 4.421154047419693e-05,
"loss_nats": 0.9842764139175415,
"step": 6880,
"unclipped_grad_norm": 0.8884434700012207
},
{
"bits_per_byte": 1.4366857090938405,
"learning_rate": 4.396334486814753e-05,
"loss_nats": 0.9958346486091614,
"step": 6890,
"unclipped_grad_norm": 0.9679457545280457
},
{
"bits_per_byte": 1.5416700245646047,
"learning_rate": 4.371721744458285e-05,
"loss_nats": 1.0686042308807373,
"step": 6900,
"unclipped_grad_norm": 0.9985603094100952
},
{
"bits_per_byte": 1.4893800754309745,
"learning_rate": 4.347316240914145e-05,
"loss_nats": 1.0323596000671387,
"step": 6910,
"unclipped_grad_norm": 0.9936915636062622
},
{
"bits_per_byte": 1.4499423897963222,
"learning_rate": 4.3231183932050546e-05,
"loss_nats": 1.00502347946167,
"step": 6920,
"unclipped_grad_norm": 0.923648476600647
},
{
"bits_per_byte": 1.4406846496917027,
"learning_rate": 4.299128614805464e-05,
"loss_nats": 0.9986065030097961,
"step": 6930,
"unclipped_grad_norm": 0.9748247861862183
},
{
"bits_per_byte": 1.408974058568991,
"learning_rate": 4.2753473156344965e-05,
"loss_nats": 0.9766263961791992,
"step": 6940,
"unclipped_grad_norm": 0.9357670545578003
},
{
"bits_per_byte": 1.434796565665438,
"learning_rate": 4.251774902048942e-05,
"loss_nats": 0.9945251941680908,
"step": 6950,
"unclipped_grad_norm": 0.9624595642089844
},
{
"bits_per_byte": 1.49675830313564,
"learning_rate": 4.228411776836306e-05,
"loss_nats": 1.0374737977981567,
"step": 6960,
"unclipped_grad_norm": 0.9387544989585876
},
{
"bits_per_byte": 1.4929531869853003,
"learning_rate": 4.2052583392079427e-05,
"loss_nats": 1.0348362922668457,
"step": 6970,
"unclipped_grad_norm": 0.9343914985656738
},
{
"bits_per_byte": 1.4660633555575984,
"learning_rate": 4.182314984792214e-05,
"loss_nats": 1.016197681427002,
"step": 6980,
"unclipped_grad_norm": 0.9326976537704468
},
{
"bits_per_byte": 1.492015881538098,
"learning_rate": 4.159582105627748e-05,
"loss_nats": 1.034186601638794,
"step": 6990,
"unclipped_grad_norm": 1.0069916248321533
},
{
"bits_per_byte": 1.52302985693349,
"learning_rate": 4.137060090156724e-05,
"loss_nats": 1.0556838512420654,
"step": 7000,
"unclipped_grad_norm": 0.9719612002372742
},
{
"bits_per_byte": 1.512785538352196,
"learning_rate": 4.114749323218249e-05,
"loss_nats": 1.0485830307006836,
"step": 7010,
"unclipped_grad_norm": 0.9873834848403931
},
{
"bits_per_byte": 1.4473323810868282,
"learning_rate": 4.0926501860417706e-05,
"loss_nats": 1.0032143592834473,
"step": 7020,
"unclipped_grad_norm": 0.939332127571106
},
{
"bits_per_byte": 1.5166639191118032,
"learning_rate": 4.070763056240571e-05,
"loss_nats": 1.0512713193893433,
"step": 7030,
"unclipped_grad_norm": 0.9837853312492371
},
{
"bits_per_byte": 1.5072451172546506,
"learning_rate": 4.049088307805306e-05,
"loss_nats": 1.0447427034378052,
"step": 7040,
"unclipped_grad_norm": 0.9734936952590942
},
{
"bits_per_byte": 1.4796651194490362,
"learning_rate": 4.027626311097629e-05,
"loss_nats": 1.0256257057189941,
"step": 7050,
"unclipped_grad_norm": 1.005554437637329
},
{
"bits_per_byte": 1.4927048440374544,
"learning_rate": 4.006377432843846e-05,
"loss_nats": 1.0346641540527344,
"step": 7060,
"unclipped_grad_norm": 0.9903765320777893
},
{
"bits_per_byte": 1.4561590466770602,
"learning_rate": 3.985342036128656e-05,
"loss_nats": 1.009332537651062,
"step": 7070,
"unclipped_grad_norm": 1.009922981262207
},
{
"bits_per_byte": 1.4353391709289103,
"learning_rate": 3.964520480388956e-05,
"loss_nats": 0.9949012994766235,
"step": 7080,
"unclipped_grad_norm": 0.9552373290061951
},
{
"bits_per_byte": 1.4780041110070032,
"learning_rate": 3.943913121407678e-05,
"loss_nats": 1.0244743824005127,
"step": 7090,
"unclipped_grad_norm": 1.0046122074127197
},
{
"bits_per_byte": 1.4728850474040718,
"learning_rate": 3.923520311307735e-05,
"loss_nats": 1.0209261178970337,
"step": 7100,
"unclipped_grad_norm": 0.9868813157081604
},
{
"bits_per_byte": 1.4862149067244954,
"learning_rate": 3.9033423985459816e-05,
"loss_nats": 1.030165672302246,
"step": 7110,
"unclipped_grad_norm": 0.9593554735183716
},
{
"bits_per_byte": 1.5328834309320238,
"learning_rate": 3.88337972790728e-05,
"loss_nats": 1.062513828277588,
"step": 7120,
"unclipped_grad_norm": 1.0129024982452393
},
{
"bits_per_byte": 1.4811555211014134,
"learning_rate": 3.8636326404985845e-05,
"loss_nats": 1.0266587734222412,
"step": 7130,
"unclipped_grad_norm": 1.0546875
},
{
"bits_per_byte": 1.4724865636020226,
"learning_rate": 3.844101473743143e-05,
"loss_nats": 1.0206499099731445,
"step": 7140,
"unclipped_grad_norm": 0.9776664972305298
},
{
"bits_per_byte": 1.4357171887955067,
"learning_rate": 3.824786561374701e-05,
"loss_nats": 0.9951633214950562,
"step": 7150,
"unclipped_grad_norm": 0.9291411638259888
},
{
"bits_per_byte": 1.5232358921492237,
"learning_rate": 3.805688233431824e-05,
"loss_nats": 1.0558266639709473,
"step": 7160,
"unclipped_grad_norm": 1.0189123153686523
},
{
"bits_per_byte": 1.4618840051589814,
"learning_rate": 3.786806816252236e-05,
"loss_nats": 1.0133007764816284,
"step": 7170,
"unclipped_grad_norm": 0.9592419266700745
},
{
"bits_per_byte": 1.4762450724539784,
"learning_rate": 3.768142632467269e-05,
"loss_nats": 1.0232551097869873,
"step": 7180,
"unclipped_grad_norm": 0.9680334329605103
},
{
"bits_per_byte": 1.4441101141402626,
"learning_rate": 3.7496960009963236e-05,
"loss_nats": 1.0009808540344238,
"step": 7190,
"unclipped_grad_norm": 1.0148857831954956
},
{
"bits_per_byte": 1.472608843266786,
"learning_rate": 3.7314672370414325e-05,
"loss_nats": 1.0207346677780151,
"step": 7200,
"unclipped_grad_norm": 0.9935786128044128
},
{
"bits_per_byte": 1.5299951542934338,
"learning_rate": 3.713456652081882e-05,
"loss_nats": 1.060511827468872,
"step": 7210,
"unclipped_grad_norm": 0.9725626111030579
},
{
"bits_per_byte": 1.5238232129019202,
"learning_rate": 3.695664553868873e-05,
"loss_nats": 1.0562337636947632,
"step": 7220,
"unclipped_grad_norm": 1.0141370296478271
},
{
"bits_per_byte": 1.4390708504873315,
"learning_rate": 3.678091246420277e-05,
"loss_nats": 0.9974879026412964,
"step": 7230,
"unclipped_grad_norm": 1.0297694206237793
},
{
"bits_per_byte": 1.4632025961431465,
"learning_rate": 3.660737030015427e-05,
"loss_nats": 1.0142147541046143,
"step": 7240,
"unclipped_grad_norm": 0.9345177412033081
},
{
"bits_per_byte": 1.5031187375124992,
"learning_rate": 3.6436022011900056e-05,
"loss_nats": 1.0418825149536133,
"step": 7250,
"unclipped_grad_norm": 1.0408201217651367
},
{
"bits_per_byte": 1.3820238612308362,
"learning_rate": 3.626687052730954e-05,
"loss_nats": 0.9579459428787231,
"step": 7260,
"unclipped_grad_norm": 0.9544695615768433
},
{
"bits_per_byte": 1.4274823155068936,
"learning_rate": 3.6099918736714955e-05,
"loss_nats": 0.9894553422927856,
"step": 7270,
"unclipped_grad_norm": 0.9940797686576843
},
{
"bits_per_byte": 1.4990891840027836,
"learning_rate": 3.5935169492861716e-05,
"loss_nats": 1.0390894412994385,
"step": 7280,
"unclipped_grad_norm": 1.013985514640808
},
{
"bits_per_byte": 1.4370805812602216,
"learning_rate": 3.5772625610859886e-05,
"loss_nats": 0.99610835313797,
"step": 7290,
"unclipped_grad_norm": 0.9820091724395752
},
{
"bits_per_byte": 1.4451605841717328,
"learning_rate": 3.5612289868135887e-05,
"loss_nats": 1.001708984375,
"step": 7300,
"unclipped_grad_norm": 0.9996907114982605
},
{
"bits_per_byte": 1.4502488628801598,
"learning_rate": 3.545416500438522e-05,
"loss_nats": 1.0052359104156494,
"step": 7310,
"unclipped_grad_norm": 0.9723894000053406
},
{
"bits_per_byte": 1.4141343980081293,
"learning_rate": 3.52982537215255e-05,
"loss_nats": 0.9802032709121704,
"step": 7320,
"unclipped_grad_norm": 0.9815637469291687
},
{
"bits_per_byte": 1.429772436485783,
"learning_rate": 3.5144558683650396e-05,
"loss_nats": 0.9910427331924438,
"step": 7330,
"unclipped_grad_norm": 1.0089058876037598
},
{
"bits_per_byte": 1.4033304478809279,
"learning_rate": 3.499308251698404e-05,
"loss_nats": 0.9727145433425903,
"step": 7340,
"unclipped_grad_norm": 0.9830491542816162
},
{
"bits_per_byte": 1.4800763299672492,
"learning_rate": 3.484382780983616e-05,
"loss_nats": 1.02591073513031,
"step": 7350,
"unclipped_grad_norm": 0.9842677712440491
},
{
"bits_per_byte": 1.4409101189469837,
"learning_rate": 3.469679711255795e-05,
"loss_nats": 0.9987627863883972,
"step": 7360,
"unclipped_grad_norm": 1.0266367197036743
},
{
"bits_per_byte": 1.4700960047550296,
"learning_rate": 3.4551992937498284e-05,
"loss_nats": 1.0189929008483887,
"step": 7370,
"unclipped_grad_norm": 0.9935671091079712
},
{
"bits_per_byte": 1.4111584102175994,
"learning_rate": 3.440941775896104e-05,
"loss_nats": 0.9781404733657837,
"step": 7380,
"unclipped_grad_norm": 0.9791932702064514
},
{
"bits_per_byte": 1.4805810990475317,
"learning_rate": 3.4269074013162596e-05,
"loss_nats": 1.0262606143951416,
"step": 7390,
"unclipped_grad_norm": 0.9806758761405945
},
{
"bits_per_byte": 1.422500752033327,
"learning_rate": 3.41309640981904e-05,
"loss_nats": 0.9860023856163025,
"step": 7400,
"unclipped_grad_norm": 1.014562726020813
},
{
"bits_per_byte": 1.419007526420327,
"learning_rate": 3.3995090373961795e-05,
"loss_nats": 0.9835810661315918,
"step": 7410,
"unclipped_grad_norm": 0.9609370231628418
},
{
"bits_per_byte": 1.483305992167802,
"learning_rate": 3.3861455162183876e-05,
"loss_nats": 1.0281493663787842,
"step": 7420,
"unclipped_grad_norm": 0.9924190640449524
},
{
"bits_per_byte": 1.4892452410326982,
"learning_rate": 3.3730060746313664e-05,
"loss_nats": 1.0322661399841309,
"step": 7430,
"unclipped_grad_norm": 1.0244429111480713
},
{
"bits_per_byte": 1.50888548777858,
"learning_rate": 3.3600909371519225e-05,
"loss_nats": 1.0458797216415405,
"step": 7440,
"unclipped_grad_norm": 1.0394790172576904
},
{
"bits_per_byte": 1.4553271665948369,
"learning_rate": 3.3474003244641194e-05,
"loss_nats": 1.0087559223175049,
"step": 7450,
"unclipped_grad_norm": 0.99248868227005
},
{
"bits_per_byte": 1.4425173828106221,
"learning_rate": 3.334934453415514e-05,
"loss_nats": 0.999876856803894,
"step": 7460,
"unclipped_grad_norm": 1.0004469156265259
},
{
"bits_per_byte": 1.437790267669007,
"learning_rate": 3.3226935370134486e-05,
"loss_nats": 0.9966002702713013,
"step": 7470,
"unclipped_grad_norm": 1.0109968185424805
},
{
"bits_per_byte": 1.4682612938356252,
"learning_rate": 3.31067778442141e-05,
"loss_nats": 1.017721176147461,
"step": 7480,
"unclipped_grad_norm": 1.0299010276794434
},
{
"bits_per_byte": 1.5222762289574092,
"learning_rate": 3.298887400955457e-05,
"loss_nats": 1.055161476135254,
"step": 7490,
"unclipped_grad_norm": 1.0327476263046265
},
{
"bits_per_byte": 1.4747042798848986,
"learning_rate": 3.287322588080714e-05,
"loss_nats": 1.0221871137619019,
"step": 7500,
"unclipped_grad_norm": 0.9886637926101685
},
{
"bits_per_byte": 1.4352613487793948,
"learning_rate": 3.275983543407924e-05,
"loss_nats": 0.9948473572731018,
"step": 7510,
"unclipped_grad_norm": 0.9710773825645447
},
{
"bits_per_byte": 1.4570311706995855,
"learning_rate": 3.2648704606900735e-05,
"loss_nats": 1.009937047958374,
"step": 7520,
"unclipped_grad_norm": 1.0144232511520386
},
{
"bits_per_byte": 1.502727820947088,
"learning_rate": 3.253983529819086e-05,
"loss_nats": 1.0416115522384644,
"step": 7530,
"unclipped_grad_norm": 1.0307852029800415
},
{
"bits_per_byte": 1.4572943041554056,
"learning_rate": 3.243322936822571e-05,
"loss_nats": 1.0101194381713867,
"step": 7540,
"unclipped_grad_norm": 0.9732069969177246
},
{
"bits_per_byte": 1.4747278415080667,
"learning_rate": 3.232888863860648e-05,
"loss_nats": 1.0222034454345703,
"step": 7550,
"unclipped_grad_norm": 1.0250486135482788
},
{
"bits_per_byte": 1.4513507557242378,
"learning_rate": 3.222681489222838e-05,
"loss_nats": 1.0059996843338013,
"step": 7560,
"unclipped_grad_norm": 1.0031380653381348
},
{
"bits_per_byte": 1.4127096077370564,
"learning_rate": 3.212700987325009e-05,
"loss_nats": 0.979215681552887,
"step": 7570,
"unclipped_grad_norm": 0.9720339775085449
},
{
"bits_per_byte": 1.587541409185384,
"learning_rate": 3.202947528706403e-05,
"loss_nats": 1.1003998517990112,
"step": 7580,
"unclipped_grad_norm": 1.080464482307434
},
{
"bits_per_byte": 1.4421111597892842,
"learning_rate": 3.1934212800267116e-05,
"loss_nats": 0.9995952844619751,
"step": 7590,
"unclipped_grad_norm": 0.9815735220909119
},
{
"bits_per_byte": 1.4267507013101233,
"learning_rate": 3.1841224040632484e-05,
"loss_nats": 0.9889482259750366,
"step": 7600,
"unclipped_grad_norm": 1.029758334159851
},
{
"bits_per_byte": 1.4539179407536669,
"learning_rate": 3.175051059708143e-05,
"loss_nats": 1.0077791213989258,
"step": 7610,
"unclipped_grad_norm": 1.031221628189087
},
{
"bits_per_byte": 1.4028547438686412,
"learning_rate": 3.1662074019656414e-05,
"loss_nats": 0.9723848104476929,
"step": 7620,
"unclipped_grad_norm": 0.9562389850616455
},
{
"bits_per_byte": 1.3938100042771153,
"learning_rate": 3.157591581949455e-05,
"loss_nats": 0.9661154747009277,
"step": 7630,
"unclipped_grad_norm": 1.023301124572754
},
{
"bits_per_byte": 1.4184527103886437,
"learning_rate": 3.149203746880175e-05,
"loss_nats": 0.983196496963501,
"step": 7640,
"unclipped_grad_norm": 0.9981763958930969
},
{
"bits_per_byte": 1.5608753271864544,
"learning_rate": 3.141044040082763e-05,
"loss_nats": 1.081916332244873,
"step": 7650,
"unclipped_grad_norm": 1.0295648574829102
},
{
"bits_per_byte": 1.4665378556913289,
"learning_rate": 3.133112600984091e-05,
"loss_nats": 1.0165265798568726,
"step": 7660,
"unclipped_grad_norm": 1.0587892532348633
},
{
"bits_per_byte": 1.4931012640476933,
"learning_rate": 3.125409565110571e-05,
"loss_nats": 1.034938931465149,
"step": 7670,
"unclipped_grad_norm": 1.0342050790786743
},
{
"bits_per_byte": 1.4465245785757257,
"learning_rate": 3.117935064085835e-05,
"loss_nats": 1.0026544332504272,
"step": 7680,
"unclipped_grad_norm": 1.0567405223846436
},
{
"bits_per_byte": 1.4541416901824393,
"learning_rate": 3.110689225628479e-05,
"loss_nats": 1.0079342126846313,
"step": 7690,
"unclipped_grad_norm": 1.0190099477767944
},
{
"bits_per_byte": 1.3949941048283057,
"learning_rate": 3.1036721735498926e-05,
"loss_nats": 0.9669362306594849,
"step": 7700,
"unclipped_grad_norm": 1.002891182899475
},
{
"bits_per_byte": 1.4762163513512843,
"learning_rate": 3.096884027752132e-05,
"loss_nats": 1.0232352018356323,
"step": 7710,
"unclipped_grad_norm": 0.9984713792800903
},
{
"bits_per_byte": 1.480895139368007,
"learning_rate": 3.090324904225885e-05,
"loss_nats": 1.0264782905578613,
"step": 7720,
"unclipped_grad_norm": 1.0342158079147339
},
{
"bits_per_byte": 1.486845911070511,
"learning_rate": 3.083994915048472e-05,
"loss_nats": 1.030603051185608,
"step": 7730,
"unclipped_grad_norm": 1.027107834815979
},
{
"bits_per_byte": 1.4387832095037634,
"learning_rate": 3.077894168381942e-05,
"loss_nats": 0.9972885251045227,
"step": 7740,
"unclipped_grad_norm": 1.0455139875411987
},
{
"bits_per_byte": 1.4381068877292456,
"learning_rate": 3.0720227684712254e-05,
"loss_nats": 0.9968197345733643,
"step": 7750,
"unclipped_grad_norm": 1.0159157514572144
},
{
"bits_per_byte": 1.4538223183997872,
"learning_rate": 3.0663808156423456e-05,
"loss_nats": 1.0077128410339355,
"step": 7760,
"unclipped_grad_norm": 0.9867706894874573
},
{
"bits_per_byte": 1.3986834766546148,
"learning_rate": 3.060968406300711e-05,
"loss_nats": 0.9694935083389282,
"step": 7770,
"unclipped_grad_norm": 0.9943742156028748
},
{
"bits_per_byte": 1.4760859885019304,
"learning_rate": 3.055785632929463e-05,
"loss_nats": 1.0231448411941528,
"step": 7780,
"unclipped_grad_norm": 1.0422483682632446
},
{
"bits_per_byte": 1.4162891686407895,
"learning_rate": 3.0508325840878984e-05,
"loss_nats": 0.9816968441009521,
"step": 7790,
"unclipped_grad_norm": 0.9937946796417236
},
{
"bits_per_byte": 1.45477768202533,
"learning_rate": 3.046109344409957e-05,
"loss_nats": 1.0083750486373901,
"step": 7800,
"unclipped_grad_norm": 1.0428839921951294
},
{
"bits_per_byte": 1.4292601001688625,
"learning_rate": 3.0416159946027727e-05,
"loss_nats": 0.9906876087188721,
"step": 7810,
"unclipped_grad_norm": 1.028254508972168
},
{
"bits_per_byte": 1.4070698666586379,
"learning_rate": 3.0373526114452946e-05,
"loss_nats": 0.975306510925293,
"step": 7820,
"unclipped_grad_norm": 1.0253064632415771
},
{
"bits_per_byte": 1.4530960356651936,
"learning_rate": 3.0333192677869803e-05,
"loss_nats": 1.0072094202041626,
"step": 7830,
"unclipped_grad_norm": 1.0885143280029297
},
{
"bits_per_byte": 1.4330722676078875,
"learning_rate": 3.029516032546544e-05,
"loss_nats": 0.9933300018310547,
"step": 7840,
"unclipped_grad_norm": 1.0074925422668457
},
{
"bits_per_byte": 1.3858685333908742,
"learning_rate": 3.02594297071078e-05,
"loss_nats": 0.9606108665466309,
"step": 7850,
"unclipped_grad_norm": 0.9880329966545105
},
{
"bits_per_byte": 1.4517502714221922,
"learning_rate": 3.0226001433334603e-05,
"loss_nats": 1.0062766075134277,
"step": 7860,
"unclipped_grad_norm": 1.0082415342330933
},
{
"bits_per_byte": 1.4675547891058807,
"learning_rate": 3.019487607534278e-05,
"loss_nats": 1.0172314643859863,
"step": 7870,
"unclipped_grad_norm": 1.023923397064209
},
{
"bits_per_byte": 1.3867994754799946,
"learning_rate": 3.016605416497886e-05,
"loss_nats": 0.9612561464309692,
"step": 7880,
"unclipped_grad_norm": 1.0151392221450806
},
{
"bits_per_byte": 1.4568015738606837,
"learning_rate": 3.0139536194729734e-05,
"loss_nats": 1.0097779035568237,
"step": 7890,
"unclipped_grad_norm": 1.0505098104476929
},
{
"bits_per_byte": 1.446776189193938,
"learning_rate": 3.0115322617714362e-05,
"loss_nats": 1.00282883644104,
"step": 7900,
"unclipped_grad_norm": 1.0279992818832397
},
{
"bits_per_byte": 1.461830518554563,
"learning_rate": 3.0093413847675965e-05,
"loss_nats": 1.0132637023925781,
"step": 7910,
"unclipped_grad_norm": 1.0125948190689087
},
{
"bits_per_byte": 1.4593548282953928,
"learning_rate": 3.0073810258974985e-05,
"loss_nats": 1.0115476846694946,
"step": 7920,
"unclipped_grad_norm": 1.0411789417266846
},
{
"bits_per_byte": 1.4317973602170415,
"learning_rate": 3.0056512186582658e-05,
"loss_nats": 0.9924463033676147,
"step": 7930,
"unclipped_grad_norm": 1.0330348014831543
},
{
"bits_per_byte": 1.431196452834927,
"learning_rate": 3.0041519926075316e-05,
"loss_nats": 0.9920297861099243,
"step": 7940,
"unclipped_grad_norm": 1.0142537355422974
},
{
"bits_per_byte": 1.49434246283897,
"learning_rate": 3.0028833733629313e-05,
"loss_nats": 1.035799264907837,
"step": 7950,
"unclipped_grad_norm": 1.0716009140014648
},
{
"bits_per_byte": 1.4186909923514142,
"learning_rate": 3.0018453826016686e-05,
"loss_nats": 0.9833616614341736,
"step": 7960,
"unclipped_grad_norm": 1.0115852355957031
},
{
"bits_per_byte": 1.4420433126335188,
"learning_rate": 3.001038038060143e-05,
"loss_nats": 0.9995482563972473,
"step": 7970,
"unclipped_grad_norm": 1.0335451364517212
},
{
"bits_per_byte": 1.4872165336831205,
"learning_rate": 3.0004613535336433e-05,
"loss_nats": 1.0308599472045898,
"step": 7980,
"unclipped_grad_norm": 1.0748283863067627
},
{
"bits_per_byte": 1.5234063269562288,
"learning_rate": 3.000115338876117e-05,
"loss_nats": 1.055944800376892,
"step": 7990,
"unclipped_grad_norm": 1.076164722442627
},
{
"bits_per_byte": 1.4309937712808851,
"learning_rate": 3e-05,
"loss_nats": 0.9918892979621887,
"step": 8000,
"unclipped_grad_norm": 1.0322442054748535
}
]
}
Reference in New Issue View Git Blame Copy Permalink