wuyang/llm-atlas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
39a9ad6215b3f821a34f9ae9e3425be42026b9af
llm-atlas/experiments/k3/attnres_gradient/results/raw/formal-depth-32-block-seed-2026073003.json
T
wuyang 26fc824409 research: audit AttnRes gradient scale study
2026-07-30 09:49:53 +08:00

13073 lines
366 KiB
JSON
Raw Blame History

{
"architecture": "block",
"batch_size": 32,
"canonical_sha256_without_self": "513b85666fcdbf75424596e968cd73733314ec188345e453998de08b18dcef65",
"depth": 32,
"diagnostics": [
{
"activation_grad_rms_by_block": [
0.00027819047681987286,
0.0002560806169640273,
0.0002331122086616233,
0.00022053472639527172,
0.00020666928321588784,
0.00019684624567162246,
0.0001890837011160329,
0.00018263237143401057,
0.00017469045997131616,
0.0001688334159553051,
0.0001649101759539917,
0.00016157960635609925,
0.00015717606584075838,
0.00015400603297166526,
0.00015046694898046553,
0.00014767753600608557,
0.00014588078192900866,
0.00014366162940859795,
0.00014116008242126554,
0.00013849612150806934,
0.0001361419854220003,
0.00013430001854430884,
0.00013224364374764264,
0.00013045054220128804,
0.00012888714263681322,
0.00012767451698891819,
0.0001263424928765744,
0.00012512138346210122,
0.00012419422273524106,
0.00012343283742666245,
0.0001227961911354214,
0.00012195061572128907
],
"activation_grad_statistics": {
"first_quartile_mean": 0.00022039370378479362,
"first_to_last_ratio": 1.7624457042066763,
"imbalance_abs_log_ratio": 0.5667024491058967,
"last_quartile_mean": 0.00012504992537287762,
"mean": 0.0001607882525149762,
"normalized": [
1.7301666786506156,
1.592657504255016,
1.4498087081325215,
1.3715848200709229,
1.2853506396348091,
1.224257634451794,
1.175979576607571,
1.1358564358860717,
1.0864628307036615,
1.050035766385231,
1.0256357250890054,
1.0049217142965674,
0.9775345112735686,
0.9578189361646354,
0.9358080993289649,
0.9184597363064851,
0.9072850761620224,
0.8934833681037547,
0.8779253472396409,
0.8613572157279928,
0.8467159963027754,
0.8352601414820391,
0.8224708066612338,
0.81131886291965,
0.8015955184587156,
0.7940537632065271,
0.7857694259399116,
0.7781749070905981,
0.7724085609032495,
0.7676732316943716,
0.7637136993200662,
0.75845476155001
],
"population_cv": 0.2498932502086342
},
"activation_output_rms_by_block": [
0.0034565660171210766,
0.004928664769977331,
0.0060537648387253284,
0.006960673723369837,
0.0036145171616226435,
0.0051584807224571705,
0.006370538379997015,
0.007546419743448496,
0.00395671371370554,
0.005665866658091545,
0.006755253300070763,
0.007946331985294819,
0.004383862018585205,
0.006652849726378918,
0.007932198233902454,
0.009347881190478802,
0.004880861379206181,
0.006906712893396616,
0.008132148534059525,
0.009816600009799004,
0.005453306715935469,
0.007622495759278536,
0.009448722936213017,
0.010670131072402,
0.0055701653473079205,
0.007738742511719465,
0.009509840048849583,
0.010865328833460808,
0.005987230688333511,
0.00815181527286768,
0.009874933399260044,
0.011666664853692055
],
"activation_output_statistics": {
"first_quartile_mean": 0.005511203169589862,
"first_to_last_ratio": 0.6356203088456116,
"imbalance_abs_log_ratio": 0.45315389258709987,
"last_quartile_mean": 0.008670590119436383,
"mean": 0.007157071326219011,
"normalized": [
0.4829581625738996,
0.6886426787339398,
0.8458438602600115,
0.9725589429115095,
0.5050274053272774,
0.7207530129760953,
0.8901040788372985,
1.0544005221525679,
0.5528397766850007,
0.7916459680002588,
0.9438571997073384,
1.1102770425361632,
0.6125217730505899,
0.9295491721602742,
1.1083022471557418,
1.3061042379491232,
0.6819634955048963,
0.9650194302374437,
1.1362396940587196,
1.3715945478756304,
0.7619466772614072,
1.0650300118366156,
1.3201940438400879,
1.490851576861243,
0.7782743937317399,
1.0812722353862223,
1.3287334463206348,
1.5181249897087195,
0.8365475786723071,
1.138987569259587,
1.3797450031110483,
1.6300892253166077
],
"population_cv": 0.3025375276561286
},
"bits_per_byte": 8.06433666787865,
"branch_output_rms_by_sublayer": [
0.0021227982360869646,
0.0027448106557130814,
0.00224129855632782,
0.0027243648655712605,
0.0024903982412070036,
0.0027103275060653687,
0.0023722455371171236,
0.0027234479784965515,
0.0023577234242111444,
0.0027595730498433113,
0.0023993616923689842,
0.0027272943407297134,
0.002688874490559101,
0.002728298306465149,
0.0029664887115359306,
0.0027007071767002344,
0.0029547461308538914,
0.002684502862393856,
0.003052416956052184,
0.002683063969016075,
0.0028214550111442804,
0.0027139110025018454,
0.0032544767018407583,
0.002666700165718794,
0.00348988501355052,
0.002679243916645646,
0.003914494998753071,
0.0026584581937640905,
0.0035071265883743763,
0.002684473292902112,
0.004211232997477055,
0.0026407993864268064,
0.004072706215083599,
0.002663099905475974,
0.004050548188388348,
0.0026283918414264917,
0.003929397091269493,
0.002597011858597398,
0.004410331137478352,
0.0026434636674821377,
0.004824656527489424,
0.0025708195753395557,
0.00433384021744132,
0.002691218862310052,
0.004475373309105635,
0.0026598076801747084,
0.004755842965096235,
0.0026596153620630503,
0.004843301139771938,
0.0026765079237520695,
0.0050001065246760845,
0.002582729561254382,
0.005346097983419895,
0.002614464145153761,
0.004958339501172304,
0.0025760054122656584,
0.0054801348596811295,
0.002634295728057623,
0.005455824080854654,
0.00259007909335196,
0.005286100786179304,
0.00257018837146461,
0.005402065813541412,
0.0025643527042120695
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.006159982634052032,
0.005514663551817784,
0.005191681274696802,
0.00489831575692433,
0.004666415075367917,
0.004399813239791133,
0.004483878567781538,
0.004868281936291656,
0.00443828064080646,
0.004266304440803418,
0.0039471188591477255,
0.004494388758593749,
0.0042334459038140095,
0.00446434979681851,
0.004322339302329385,
0.004638471384339534,
0.004165330760400444,
0.004499937764754925,
0.003981478974932938,
0.004372386299982334,
0.00458495381977442,
0.00429501466057645,
0.003963900484885007,
0.004271473918508079,
0.004041799650627023,
0.004206875397956982,
0.003955591472942362,
0.003916464042690177,
0.003989587235588314,
0.003922309599766117,
0.003679536678174494,
0.003763244111925236
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.005022879004590398,
"first_to_last_ratio": 1.2766484804448084,
"imbalance_abs_log_ratio": 0.24423826934058726,
"last_quartile_mean": 0.003934426023708838,
"mean": 0.0043936754999019155,
"normalized": [
1.4020112851278042,
1.255136741878387,
1.1816260155791434,
1.1148560600421402,
1.0620754936207943,
1.0013969488391563,
1.0205302070855338,
1.108020366183241,
1.0101521245493756,
0.9710103627130996,
0.8983637638318627,
1.0229223252136126,
0.9635317637610054,
1.0160854612313022,
0.9837638902613262,
1.0557155130011497,
0.9480287655502622,
1.0241852783291305,
0.9061841219320409,
0.995154580733134,
1.0435349219296635,
0.9775448051801577,
0.9021832597727114,
0.9721869351988865,
0.9199131002545027,
0.9574843199164109,
0.9002921296829196,
0.891386731400079,
0.9080295610536958,
0.892717179467095,
0.8374620925593245,
0.8565138941210488
],
"population_cv": 0.11556590700625036
},
"depth_weights": [
{
"entropy_mean": 0.0,
"mean_weights": [
1.0
],
"sources": 1
},
{
"entropy_mean": 0.6931471824645996,
"mean_weights": [
0.5,
0.5
],
"sources": 2
},
{
"entropy_mean": 0.6931471824645996,
"mean_weights": [
0.5,
0.5
],
"sources": 2
},
{
"entropy_mean": 0.6931471824645996,
"mean_weights": [
0.5,
0.5
],
"sources": 2
},
{
"entropy_mean": 0.6931471824645996,
"mean_weights": [
0.5,
0.5
],
"sources": 2
},
{
"entropy_mean": 0.6931471824645996,
"mean_weights": [
0.5,
0.5
],
"sources": 2
},
{
"entropy_mean": 0.6931471824645996,
"mean_weights": [
0.5,
0.5
],
"sources": 2
},
{
"entropy_mean": 0.6931471824645996,
"mean_weights": [
0.5,
0.5
],
"sources": 2
},
{
"entropy_mean": 0.6931471824645996,
"mean_weights": [
0.5,
0.5
],
"sources": 2
},
{
"entropy_mean": 1.0986123085021973,
"mean_weights": [
0.3333333432674408,
0.3333333432674408,
0.3333333432674408
],
"sources": 3
},
{
"entropy_mean": 1.0986123085021973,
"mean_weights": [
0.3333333432674408,
0.3333333432674408,
0.3333333432674408
],
"sources": 3
},
{
"entropy_mean": 1.0986123085021973,
"mean_weights": [
0.3333333432674408,
0.3333333432674408,
0.3333333432674408
],
"sources": 3
},
{
"entropy_mean": 1.0986123085021973,
"mean_weights": [
0.3333333432674408,
0.3333333432674408,
0.3333333432674408
],
"sources": 3
},
{
"entropy_mean": 1.0986123085021973,
"mean_weights": [
0.3333333432674408,
0.3333333432674408,
0.3333333432674408
],
"sources": 3
},
{
"entropy_mean": 1.0986123085021973,
"mean_weights": [
0.3333333432674408,
0.3333333432674408,
0.3333333432674408
],
"sources": 3
},
{
"entropy_mean": 1.0986123085021973,
"mean_weights": [
0.3333333432674408,
0.3333333432674408,
0.3333333432674408
],
"sources": 3
},
{
"entropy_mean": 1.0986123085021973,
"mean_weights": [
0.3333333432674408,
0.3333333432674408,
0.3333333432674408
],
"sources": 3
},
{
"entropy_mean": 1.3862943649291992,
"mean_weights": [
0.25,
0.25,
0.25,
0.25
],
"sources": 4
},
{
"entropy_mean": 1.3862943649291992,
"mean_weights": [
0.25,
0.25,
0.25,
0.25
],
"sources": 4
},
{
"entropy_mean": 1.3862943649291992,
"mean_weights": [
0.25,
0.25,
0.25,
0.25
],
"sources": 4
},
{
"entropy_mean": 1.3862943649291992,
"mean_weights": [
0.25,
0.25,
0.25,
0.25
],
"sources": 4
},
{
"entropy_mean": 1.3862943649291992,
"mean_weights": [
0.25,
0.25,
0.25,
0.25
],
"sources": 4
},
{
"entropy_mean": 1.3862943649291992,
"mean_weights": [
0.25,
0.25,
0.25,
0.25
],
"sources": 4
},
{
"entropy_mean": 1.3862943649291992,
"mean_weights": [
0.25,
0.25,
0.25,
0.25
],
"sources": 4
},
{
"entropy_mean": 1.3862943649291992,
"mean_weights": [
0.25,
0.25,
0.25,
0.25
],
"sources": 4
},
{
"entropy_mean": 1.6094379425048828,
"mean_weights": [
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343
],
"sources": 5
},
{
"entropy_mean": 1.6094379425048828,
"mean_weights": [
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343
],
"sources": 5
},
{
"entropy_mean": 1.6094379425048828,
"mean_weights": [
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343
],
"sources": 5
},
{
"entropy_mean": 1.6094379425048828,
"mean_weights": [
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343
],
"sources": 5
},
{
"entropy_mean": 1.6094379425048828,
"mean_weights": [
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343
],
"sources": 5
},
{
"entropy_mean": 1.6094379425048828,
"mean_weights": [
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343
],
"sources": 5
},
{
"entropy_mean": 1.6094379425048828,
"mean_weights": [
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343
],
"sources": 5
},
{
"entropy_mean": 1.6094379425048828,
"mean_weights": [
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343
],
"sources": 5
},
{
"entropy_mean": 1.791759729385376,
"mean_weights": [
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592
],
"sources": 6
},
{
"entropy_mean": 1.791759729385376,
"mean_weights": [
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592
],
"sources": 6
},
{
"entropy_mean": 1.791759729385376,
"mean_weights": [
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592
],
"sources": 6
},
{
"entropy_mean": 1.791759729385376,
"mean_weights": [
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592
],
"sources": 6
},
{
"entropy_mean": 1.791759729385376,
"mean_weights": [
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592
],
"sources": 6
},
{
"entropy_mean": 1.791759729385376,
"mean_weights": [
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592
],
"sources": 6
},
{
"entropy_mean": 1.791759729385376,
"mean_weights": [
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592
],
"sources": 6
},
{
"entropy_mean": 1.791759729385376,
"mean_weights": [
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592
],
"sources": 6
},
{
"entropy_mean": 1.9459102153778076,
"mean_weights": [
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548
],
"sources": 7
},
{
"entropy_mean": 1.9459102153778076,
"mean_weights": [
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548
],
"sources": 7
},
{
"entropy_mean": 1.9459102153778076,
"mean_weights": [
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548
],
"sources": 7
},
{
"entropy_mean": 1.9459102153778076,
"mean_weights": [
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548
],
"sources": 7
},
{
"entropy_mean": 1.9459102153778076,
"mean_weights": [
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548
],
"sources": 7
},
{
"entropy_mean": 1.9459102153778076,
"mean_weights": [
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548
],
"sources": 7
},
{
"entropy_mean": 1.9459102153778076,
"mean_weights": [
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548
],
"sources": 7
},
{
"entropy_mean": 1.9459102153778076,
"mean_weights": [
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548
],
"sources": 7
},
{
"entropy_mean": 2.079441547393799,
"mean_weights": [
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125
],
"sources": 8
},
{
"entropy_mean": 2.079441547393799,
"mean_weights": [
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125
],
"sources": 8
},
{
"entropy_mean": 2.079441547393799,
"mean_weights": [
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125
],
"sources": 8
},
{
"entropy_mean": 2.079441547393799,
"mean_weights": [
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125
],
"sources": 8
},
{
"entropy_mean": 2.079441547393799,
"mean_weights": [
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125
],
"sources": 8
},
{
"entropy_mean": 2.079441547393799,
"mean_weights": [
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125
],
"sources": 8
},
{
"entropy_mean": 2.079441547393799,
"mean_weights": [
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125
],
"sources": 8
},
{
"entropy_mean": 2.079441547393799,
"mean_weights": [
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125
],
"sources": 8
},
{
"entropy_mean": 2.1972246170043945,
"mean_weights": [
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519
],
"sources": 9
},
{
"entropy_mean": 2.1972246170043945,
"mean_weights": [
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519
],
"sources": 9
},
{
"entropy_mean": 2.1972246170043945,
"mean_weights": [
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519
],
"sources": 9
},
{
"entropy_mean": 2.1972246170043945,
"mean_weights": [
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519
],
"sources": 9
},
{
"entropy_mean": 2.1972246170043945,
"mean_weights": [
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519
],
"sources": 9
},
{
"entropy_mean": 2.1972246170043945,
"mean_weights": [
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519
],
"sources": 9
},
{
"entropy_mean": 2.1972246170043945,
"mean_weights": [
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519
],
"sources": 9
}
],
"layer_input_rms_by_sublayer": [
0.028097106143832207,
0.014079716056585312,
0.014136734418570995,
0.014167090877890587,
0.014224153943359852,
0.014255538582801819,
0.014337832108139992,
0.014355218037962914,
0.014408928342163563,
0.009655896574258804,
0.009704872034490108,
0.009734091348946095,
0.009773381985723972,
0.009828771464526653,
0.00987160298973322,
0.009915035218000412,
0.00994801800698042,
0.007479202933609486,
0.007515520788729191,
0.007559895049780607,
0.007602147292345762,
0.0076358383521437645,
0.007658426649868488,
0.007706487085670233,
0.007734050042927265,
0.0062490664422512054,
0.0062688966281712055,
0.006315518636256456,
0.006336068734526634,
0.0063744354993104935,
0.006399397272616625,
0.006439714692533016,
0.006462901830673218,
0.0054116579703986645,
0.005432929843664169,
0.005481995176523924,
0.005504137370735407,
0.005541813559830189,
0.005560056772083044,
0.005614723544567823,
0.005631812382489443,
0.004862982779741287,
0.004873365629464388,
0.004923288244754076,
0.004937449004501104,
0.0049925679340958595,
0.005005750339478254,
0.005004179198294878,
0.005022916477173567,
0.004448203835636377,
0.0044684382155537605,
0.004490054678171873,
0.004500201437622309,
0.00455933902412653,
0.004573327023535967,
0.004625064320862293,
0.00463611725717783,
0.004160413984209299,
0.004166214261204004,
0.004242690280079842,
0.0042496914975345135,
0.004270957782864571,
0.004283483140170574,
0.004303553607314825
],
"loss_nats": 5.5897722244262695,
"loss_scale": 1.0,
"output_weights": {
"entropy_mean": 2.1972246170043945,
"mean_weights": [
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519
],
"sources": 9
},
"step": 0,
"stream_state_rms_by_sublayer": [
0.0021227982360869646,
0.0034565660171210766,
0.00413061399012804,
0.004928664769977331,
0.005416445434093475,
0.0060537648387253284,
0.006388718727976084,
0.006960673723369837,
0.0023577234242111444,
0.0036145171616226435,
0.004388594534248114,
0.0051584807224571705,
0.00574506726115942,
0.006370538379997015,
0.007066869642585516,
0.007546419743448496,
0.0029547461308538914,
0.00395671371370554,
0.004991721361875534,
0.005665866658091545,
0.006190058775246143,
0.006755253300070763,
0.0074937604367733,
0.007946331985294819,
0.00348988501355052,
0.004383862018585205,
0.006057556718587875,
0.006652849726378918,
0.0074848332442343235,
0.007932198233902454,
0.00897275935858488,
0.009347881190478802,
0.004072706215083599,
0.004880861379206181,
0.0063449000008404255,
0.006906712893396616,
0.007735534105449915,
0.008132148534059525,
0.00946879480034113,
0.009816600009799004,
0.004824656527489424,
0.005453306715935469,
0.007074959110468626,
0.007622495759278536,
0.009095527231693268,
0.009448722936213017,
0.010346688330173492,
0.010670131072402,
0.004843301139771938,
0.0055701653473079205,
0.007334613241255283,
0.007738742511719465,
0.009146629832684994,
0.009509840048849583,
0.010587968863546848,
0.010865328833460808,
0.0054801348596811295,
0.005987230688333511,
0.0077637480571866035,
0.00815181527286768,
0.009536821395158768,
0.009874933399260044,
0.011367839761078358,
0.011666664853692055
]
},
{
"activation_grad_rms_by_block": [
6.886429764563218e-05,
6.52737362543121e-05,
6.186874816194177e-05,
5.901425174670294e-05,
5.411080928752199e-05,
5.1034174248343334e-05,
4.815354259335436e-05,
4.550318044493906e-05,
4.120725498069078e-05,
3.8442405639216304e-05,
3.604330777307041e-05,
3.37670098815579e-05,
3.108748569502495e-05,
2.9203254598542117e-05,
2.7565200070966966e-05,
2.5956482204492204e-05,
2.4566294086980633e-05,
2.329334893147461e-05,
2.2260202968027443e-05,
2.14147730730474e-05,
2.0902438336634077e-05,
2.0279332602513023e-05,
1.9777973648160696e-05,
1.9328235794091597e-05,
1.9035323930438608e-05,
1.8655482563190162e-05,
1.832095585996285e-05,
1.8063024981529452e-05,
1.7733544154907577e-05,
1.752249227138236e-05,
1.7320473489235155e-05,
1.7174717868329026e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 5.6727842547843466e-05,
"first_to_last_ratio": 3.155359202626439,
"imbalance_abs_log_ratio": 1.1491023413751247,
"last_quartile_mean": 1.79782518898719e-05,
"mean": 3.227324236831919e-05,
"normalized": [
2.133789250541258,
2.0225341944070556,
1.9170292050567193,
1.8285814320482991,
1.67664620337743,
1.581315371598383,
1.4920577871848402,
1.409935200362978,
1.2768241415105412,
1.1911541208190792,
1.1168170635514478,
1.0462850151897058,
0.9632588303411923,
0.904875136661487,
0.8541193275958588,
0.8042725273235084,
0.7611969633115008,
0.7217542218299198,
0.6897417592562383,
0.6635457580819046,
0.647670850610065,
0.628363657145899,
0.6128288389014056,
0.5988935221787641,
0.5898175247840765,
0.5780479801280578,
0.5676825294116558,
0.559690432569022,
0.5494813304632692,
0.5429417990112825,
0.5366821620079203,
0.5321658627392354
],
"population_cv": 0.49221598973047537
},
"activation_output_rms_by_block": [
0.0055193523876369,
0.009546536952257156,
0.013977172784507275,
0.018744563683867455,
0.00660830270498991,
0.011765030212700367,
0.017156247049570084,
0.022267481312155724,
0.007449587807059288,
0.013175037689507008,
0.019073888659477234,
0.025712018832564354,
0.008972279727458954,
0.01624389924108982,
0.023743286728858948,
0.03214457631111145,
0.00991375558078289,
0.019281653687357903,
0.02868141047656536,
0.03889070823788643,
0.0127727584913373,
0.022841427475214005,
0.034929387271404266,
0.04634999856352806,
0.014845813624560833,
0.029589947313070297,
0.046351123601198196,
0.06254568696022034,
0.01947367936372757,
0.0370744951069355,
0.05507569760084152,
0.07403328269720078
],
"activation_output_statistics": {
"first_quartile_mean": 0.013198085885960609,
"first_to_last_ratio": 0.31146869331458016,
"imbalance_abs_log_ratio": 1.1664564488083473,
"last_quartile_mean": 0.04237371578346938,
"mean": 0.0251484402542701,
"normalized": [
0.21947096248642048,
0.3796075166385794,
0.5557868656341027,
0.7453569085933552,
0.26277187126417706,
0.46782345520226504,
0.6821992487847045,
0.8854418439877121,
0.2962246458125521,
0.5238908479530829,
0.7584521531604159,
1.0224100808080359,
0.35677281122575777,
0.6459207440641045,
0.9441256192748351,
1.2781936369057099,
0.39420956053525336,
0.7667137004285568,
1.140484665711838,
1.5464461352144077,
0.5078946591595691,
0.9082641803733982,
1.3889285744261377,
1.8430565909811456,
0.5903274109431131,
1.1766116313335198,
1.8431013268637195,
2.487060283971303,
0.7743493897368454,
1.4742264224772512,
2.190024392923927,
2.9438518631242045
],
"population_cv": 0.6624204772463126
},
"bits_per_byte": 5.18286091123436,
"branch_output_rms_by_sublayer": [
0.002723898272961378,
0.004827107302844524,
0.0025367254856973886,
0.004943604115396738,
0.0027546093333512545,
0.00549353240057826,
0.002648916095495224,
0.005891259294003248,
0.0027809101156890392,
0.0059933834709227085,
0.0025255572982132435,
0.006044833920896053,
0.003397067543119192,
0.0061737713403999805,
0.0033452848438173532,
0.0059272525832057,
0.003559867851436138,
0.0063678608275949955,
0.0038245979230850935,
0.0061998614110052586,
0.003386941971257329,
0.006479958072304726,
0.00461281044408679,
0.006899545900523663,
0.004887574352324009,
0.007097913417965174,
0.005150536075234413,
0.006852242164313793,
0.0054018739610910416,
0.006816149223595858,
0.006125702988356352,
0.007103090640157461,
0.005625482182949781,
0.007109561935067177,
0.006412550341337919,
0.007626448292285204,
0.0072620403952896595,
0.007445307914167643,
0.008774910122156143,
0.007442969363182783,
0.008495661430060863,
0.007471293676644564,
0.007638709619641304,
0.007774693425744772,
0.009623398073017597,
0.007449399214237928,
0.008996927179396152,
0.007351893465965986,
0.00961991585791111,
0.008440978825092316,
0.011907265521585941,
0.008385274559259415,
0.014190610498189926,
0.008216616697609425,
0.013635334558784962,
0.007803319953382015,
0.014384749345481396,
0.007909612730145454,
0.01446358859539032,
0.007761957123875618,
0.013604369945824146,
0.008686874993145466,
0.015084062702953815,
0.008338849991559982
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.00023786681750428726,
0.00020576573809540593,
0.00019740566304489066,
0.0001813902134209802,
0.0001696969276866905,
0.00015923444695871368,
0.00017314685145041394,
0.00015953319119459735,
0.0001633125663305534,
0.0001601758879769712,
0.00014581270072009644,
0.00016837778263142155,
0.0001603117902727929,
0.00016340123818181664,
0.00015138579649161358,
0.00015756725288262655,
0.00013845240957374987,
0.0001515626583813202,
0.00014303549429743115,
0.00016115390114098118,
0.00015144047315466305,
0.00013355245347473064,
0.00014934284038847835,
0.00014161248575260865,
0.00014194691162247068,
0.00015146794722523853,
0.00015520640136475585,
0.0001429086966261467,
0.00014288517283478702,
0.00013975759356060232,
0.00013469418893153685,
0.00013332122361590492
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.00018550498116949745,
"first_to_last_ratio": 1.2992954513055368,
"imbalance_abs_log_ratio": 0.26182215701959494,
"last_quartile_mean": 0.00014277351697268036,
"mean": 0.00015833517864966492,
"normalized": [
1.5022992333914333,
1.2995579368416077,
1.2467580782011416,
1.1456090489045851,
1.0717575786626974,
1.0056795231275704,
1.0935463169149642,
1.0075663068381233,
1.0314357663491922,
1.0116253971038178,
0.9209115874541316,
1.0634262333071103,
1.0124837173898131,
1.0319957925671146,
0.9561096768430238,
0.9951499990489321,
0.874426079880151,
0.9572266862859976,
0.9033715407863586,
1.0178022503612605,
0.9564549991113648,
0.8434793494011273,
0.9432069465681839,
0.894384223141863,
0.8964963619142705,
0.9566285175348117,
0.9802395316594056,
0.9025707227220104,
0.9024221531396832,
0.8826692510944287,
0.8506902261408595,
0.8420189673129665
],
"population_cv": 0.13848454603612886
},
"depth_weights": [
{
"entropy_mean": 0.0,
"mean_weights": [
1.0
],
"sources": 1
},
{
"entropy_mean": 0.6920576691627502,
"mean_weights": [
0.5229578018188477,
0.47704219818115234
],
"sources": 2
},
{
"entropy_mean": 0.6922386288642883,
"mean_weights": [
0.5206724405288696,
0.479327529668808
],
"sources": 2
},
{
"entropy_mean": 0.6930604577064514,
"mean_weights": [
0.5023424625396729,
0.4976575970649719
],
"sources": 2
},
{
"entropy_mean": 0.6924410462379456,
"mean_weights": [
0.5176653861999512,
0.48233455419540405
],
"sources": 2
},
{
"entropy_mean": 0.6929901838302612,
"mean_weights": [
0.5013687014579773,
0.4986313283443451
],
"sources": 2
},
{
"entropy_mean": 0.6929076313972473,
"mean_weights": [
0.5098068714141846,
0.49019312858581543
],
"sources": 2
},
{
"entropy_mean": 0.6927430629730225,
"mean_weights": [
0.5127849578857422,
0.4872150421142578
],
"sources": 2
},
{
"entropy_mean": 0.6928876638412476,
"mean_weights": [
0.5100651383399963,
0.48993486166000366
],
"sources": 2
},
{
"entropy_mean": 1.098104476928711,
"mean_weights": [
0.33572155237197876,
0.32083213329315186,
0.34344637393951416
],
"sources": 3
},
{
"entropy_mean": 1.098517656326294,
"mean_weights": [
0.3379058539867401,
0.3309776186943054,
0.33111652731895447
],
"sources": 3
},
{
"entropy_mean": 1.0984238386154175,
"mean_weights": [
0.34072375297546387,
0.3312225341796875,
0.3280537724494934
],
"sources": 3
},
{
"entropy_mean": 1.0985376834869385,
"mean_weights": [
0.3372243344783783,
0.33110499382019043,
0.3316706418991089
],
"sources": 3
},
{
"entropy_mean": 1.0983879566192627,
"mean_weights": [
0.34103795886039734,
0.33302855491638184,
0.3259335160255432
],
"sources": 3
},
{
"entropy_mean": 1.0984759330749512,
"mean_weights": [
0.3378061056137085,
0.3331226110458374,
0.3290712833404541
],
"sources": 3
},
{
"entropy_mean": 1.0982115268707275,
"mean_weights": [
0.3427826464176178,
0.33246177434921265,
0.32475557923316956
],
"sources": 3
},
{
"entropy_mean": 1.098510980606079,
"mean_weights": [
0.336602121591568,
0.33530712127685547,
0.32809075713157654
],
"sources": 3
},
{
"entropy_mean": 1.3860652446746826,
"mean_weights": [
0.25563088059425354,
0.25276094675064087,
0.24706901609897614,
0.24453918635845184
],
"sources": 4
},
{
"entropy_mean": 1.3861669301986694,
"mean_weights": [
0.25406360626220703,
0.25270599126815796,
0.24658669531345367,
0.24664369225502014
],
"sources": 4
},
{
"entropy_mean": 1.386000394821167,
"mean_weights": [
0.2551618814468384,
0.2541123330593109,
0.2483193278312683,
0.24240648746490479
],
"sources": 4
},
{
"entropy_mean": 1.386232852935791,
"mean_weights": [
0.25183945894241333,
0.25246676802635193,
0.24847325682640076,
0.24722051620483398
],
"sources": 4
},
{
"entropy_mean": 1.385960340499878,
"mean_weights": [
0.25454509258270264,
0.25490614771842957,
0.24972698092460632,
0.24082177877426147
],
"sources": 4
},
{
"entropy_mean": 1.386134147644043,
"mean_weights": [
0.25200384855270386,
0.25437670946121216,
0.24930089712142944,
0.24431857466697693
],
"sources": 4
},
{
"entropy_mean": 1.3859115839004517,
"mean_weights": [
0.2541099786758423,
0.2563633620738983,
0.2499358206987381,
0.2395908385515213
],
"sources": 4
},
{
"entropy_mean": 1.3861732482910156,
"mean_weights": [
0.2492598295211792,
0.25510817766189575,
0.2504415512084961,
0.24519044160842896
],
"sources": 4
},
{
"entropy_mean": 1.6091704368591309,
"mean_weights": [
0.20294126868247986,
0.20561827719211578,
0.20076946914196014,
0.19495637714862823,
0.19571462273597717
],
"sources": 5
},
{
"entropy_mean": 1.6093817949295044,
"mean_weights": [
0.1999536156654358,
0.20290037989616394,
0.19905918836593628,
0.1980593204498291,
0.2000274658203125
],
"sources": 5
},
{
"entropy_mean": 1.6089024543762207,
"mean_weights": [
0.20434537529945374,
0.20689502358436584,
0.20309439301490784,
0.19485914707183838,
0.1908060759305954
],
"sources": 5
},
{
"entropy_mean": 1.609398365020752,
"mean_weights": [
0.20015040040016174,
0.2025333195924759,
0.2003234326839447,
0.19931097328662872,
0.19768190383911133
],
"sources": 5
},
{
"entropy_mean": 1.6085304021835327,
"mean_weights": [
0.20286625623703003,
0.21047379076480865,
0.20388861000537872,
0.19463706016540527,
0.18813428282737732
],
"sources": 5
},
{
"entropy_mean": 1.60928213596344,
"mean_weights": [
0.20026028156280518,
0.20464842021465302,
0.20178082585334778,
0.1985689103603363,
0.19474159181118011
],
"sources": 5
},
{
"entropy_mean": 1.6085617542266846,
"mean_weights": [
0.20295967161655426,
0.2110326737165451,
0.20332126319408417,
0.194858580827713,
0.18782781064510345
],
"sources": 5
},
{
"entropy_mean": 1.6093013286590576,
"mean_weights": [
0.20157885551452637,
0.20404376089572906,
0.20028473436832428,
0.19877153635025024,
0.19532111287117004
],
"sources": 5
},
{
"entropy_mean": 1.79081130027771,
"mean_weights": [
0.1695082187652588,
0.17698298394680023,
0.17063108086585999,
0.16265572607517242,
0.15738935768604279,
0.16283263266086578
],
"sources": 6
},
{
"entropy_mean": 1.7917133569717407,
"mean_weights": [
0.16738522052764893,
0.1689736694097519,
0.16775797307491302,
0.16576024889945984,
0.16475272178649902,
0.16537019610404968
],
"sources": 6
},
{
"entropy_mean": 1.7902145385742188,
"mean_weights": [
0.17057637870311737,
0.18038460612297058,
0.17244961857795715,
0.164292573928833,
0.15751495957374573,
0.15478186309337616
],
"sources": 6
},
{
"entropy_mean": 1.7917354106903076,
"mean_weights": [
0.16660897433757782,
0.16850554943084717,
0.1666441559791565,
0.16635674238204956,
0.16625551879405975,
0.1656290590763092
],
"sources": 6
},
{
"entropy_mean": 1.7898752689361572,
"mean_weights": [
0.17184747755527496,
0.180228590965271,
0.17470473051071167,
0.16429948806762695,
0.15632927417755127,
0.15259043872356415
],
"sources": 6
},
{
"entropy_mean": 1.7917178869247437,
"mean_weights": [
0.1659121811389923,
0.16851314902305603,
0.16693055629730225,
0.16679996252059937,
0.16575691103935242,
0.16608723998069763
],
"sources": 6
},
{
"entropy_mean": 1.7883799076080322,
"mean_weights": [
0.1728566288948059,
0.18512636423110962,
0.17585304379463196,
0.16398008167743683,
0.1537502557039261,
0.1484336107969284
],
"sources": 6
},
{
"entropy_mean": 1.7917424440383911,
"mean_weights": [
0.1667865514755249,
0.1680906116962433,
0.16619735956192017,
0.16709285974502563,
0.16617436707019806,
0.16565825045108795
],
"sources": 6
},
{
"entropy_mean": 1.9426109790802002,
"mean_weights": [
0.14892546832561493,
0.16000479459762573,
0.15354622900485992,
0.14328595995903015,
0.13467344641685486,
0.12974688410758972,
0.12981724739074707
],
"sources": 7
},
{
"entropy_mean": 1.9458978176116943,
"mean_weights": [
0.1419999748468399,
0.1434222161769867,
0.14279979467391968,
0.14312142133712769,
0.14279043674468994,
0.1428302675485611,
0.143035888671875
],
"sources": 7
},
{
"entropy_mean": 1.9415357112884521,
"mean_weights": [
0.1496729701757431,
0.16313126683235168,
0.15414540469646454,
0.1440294086933136,
0.1343868225812912,
0.12868303060531616,
0.1259511113166809
],
"sources": 7
},
{
"entropy_mean": 1.9458801746368408,
"mean_weights": [
0.1429852843284607,
0.1429750919342041,
0.14214137196540833,
0.1414642632007599,
0.1421438455581665,
0.14385518431663513,
0.14443498849868774
],
"sources": 7
},
{
"entropy_mean": 1.9424511194229126,
"mean_weights": [
0.14928653836250305,
0.16045087575912476,
0.15314346551895142,
0.1440504789352417,
0.1356543004512787,
0.13001246750354767,
0.1274019032716751
],
"sources": 7
},
{
"entropy_mean": 1.9458882808685303,
"mean_weights": [
0.14319148659706116,
0.14391475915908813,
0.14284171164035797,
0.1420421153306961,
0.14134715497493744,
0.14342516660690308,
0.1432376205921173
],
"sources": 7
},
{
"entropy_mean": 1.9395697116851807,
"mean_weights": [
0.15016257762908936,
0.16719450056552887,
0.15599417686462402,
0.14574657380580902,
0.13377557694911957,
0.12622776627540588,
0.12089881300926208
],
"sources": 7
},
{
"entropy_mean": 1.9458978176116943,
"mean_weights": [
0.1428529918193817,
0.14342239499092102,
0.14335417747497559,
0.14274317026138306,
0.1419316530227661,
0.14220726490020752,
0.14348836243152618
],
"sources": 7
},
{
"entropy_mean": 2.074862480163574,
"mean_weights": [
0.13159659504890442,
0.14486080408096313,
0.13573890924453735,
0.12842535972595215,
0.12030278891324997,
0.11495959758758545,
0.10915482044219971,
0.11496111750602722
],
"sources": 8
},
{
"entropy_mean": 2.079418659210205,
"mean_weights": [
0.12459750473499298,
0.12504151463508606,
0.12489251047372818,
0.125042125582695,
0.12461285293102264,
0.12601442635059357,
0.1261361837387085,
0.12366288155317307
],
"sources": 8
},
{
"entropy_mean": 2.0741164684295654,
"mean_weights": [
0.1323997974395752,
0.14626389741897583,
0.1369163542985916,
0.12836164236068726,
0.12002508342266083,
0.11502262949943542,
0.10908686369657516,
0.11192375421524048
],
"sources": 8
},
{
"entropy_mean": 2.0793213844299316,
"mean_weights": [
0.12429114431142807,
0.12243181467056274,
0.12357296049594879,
0.12375843524932861,
0.12469147145748138,
0.12560917437076569,
0.12766826152801514,
0.12797673046588898
],
"sources": 8
},
{
"entropy_mean": 2.0734808444976807,
"mean_weights": [
0.1330580711364746,
0.14781729876995087,
0.1374434530735016,
0.12885303795337677,
0.11949820816516876,
0.11495119333267212,
0.10982058197259903,
0.10855817794799805
],
"sources": 8
},
{
"entropy_mean": 2.07940673828125,
"mean_weights": [
0.12457560002803802,
0.12431882321834564,
0.12479400634765625,
0.12437158077955246,
0.12397614866495132,
0.12488298863172531,
0.12619814276695251,
0.12688270211219788
],
"sources": 8
},
{
"entropy_mean": 2.07299542427063,
"mean_weights": [
0.13274571299552917,
0.14826728403568268,
0.13701492547988892,
0.12918588519096375,
0.12125477194786072,
0.1169566735625267,
0.10889489948749542,
0.10567984730005264
],
"sources": 8
},
{
"entropy_mean": 2.079298734664917,
"mean_weights": [
0.12391229718923569,
0.12222036719322205,
0.12298265099525452,
0.12416766583919525,
0.125657856464386,
0.12553520500659943,
0.126623272895813,
0.1289007067680359
],
"sources": 8
},
{
"entropy_mean": 2.189734935760498,
"mean_weights": [
0.1203557699918747,
0.13466420769691467,
0.12461012601852417,
0.11655421555042267,
0.10905998200178146,
0.10487955063581467,
0.09900732338428497,
0.0950026735663414,
0.0958661437034607
],
"sources": 9
},
{
"entropy_mean": 2.197068691253662,
"mean_weights": [
0.1099812239408493,
0.10853179544210434,
0.10949578881263733,
0.11001666635274887,
0.1107073426246643,
0.11118210852146149,
0.11218173801898956,
0.11490923166275024,
0.11299411952495575
],
"sources": 9
},
{
"entropy_mean": 2.1907877922058105,
"mean_weights": [
0.11948458850383759,
0.13228623569011688,
0.12348943203687668,
0.11669500172138214,
0.11027230322360992,
0.1054498627781868,
0.10031206905841827,
0.09697622060775757,
0.09503427147865295
],
"sources": 9
},
{
"entropy_mean": 2.1971726417541504,
"mean_weights": [
0.11100216209888458,
0.11012113094329834,
0.1099993959069252,
0.11000204086303711,
0.10997181385755539,
0.11260797083377838,
0.11264219880104065,
0.11185945570468903,
0.11179381608963013
],
"sources": 9
},
{
"entropy_mean": 2.190375804901123,
"mean_weights": [
0.11983220279216766,
0.13351142406463623,
0.12386792153120041,
0.11615028977394104,
0.10967587679624557,
0.10584976524114609,
0.10062196850776672,
0.09596225619316101,
0.09452830255031586
],
"sources": 9
},
{
"entropy_mean": 2.1969356536865234,
"mean_weights": [
0.10960094630718231,
0.10724944621324539,
0.10892992466688156,
0.1096237450838089,
0.11032412201166153,
0.11112386733293533,
0.11386257410049438,
0.11429048329591751,
0.11499489098787308
],
"sources": 9
},
{
"entropy_mean": 2.188795566558838,
"mean_weights": [
0.12045153975486755,
0.13515985012054443,
0.12505559623241425,
0.11722971498966217,
0.11039333790540695,
0.10568177700042725,
0.09951679408550262,
0.09492656588554382,
0.09158482402563095
],
"sources": 9
}
],
"layer_input_rms_by_sublayer": [
0.028617795556783676,
0.014952295459806919,
0.015273182652890682,
0.014778315089643002,
0.015790818259119987,
0.01543855108320713,
0.01658804342150688,
0.016689632087945938,
0.017823126167058945,
0.01178102195262909,
0.012884888797998428,
0.012982882559299469,
0.013985250145196915,
0.014135614968836308,
0.015267650596797466,
0.015412751585245132,
0.016450626775622368,
0.012501097284257412,
0.013461960479617119,
0.013585822656750679,
0.014477741904556751,
0.014657454565167427,
0.01562186423689127,
0.01575874350965023,
0.016906939446926117,
0.013761760666966438,
0.014646334573626518,
0.014892088249325752,
0.015772713348269463,
0.016054842621088028,
0.016963548958301544,
0.017220331355929375,
0.018231164664030075,
0.015453077852725983,
0.016293151304125786,
0.016655873507261276,
0.017603756859898567,
0.017766626551747322,
0.018859725445508957,
0.018939858302474022,
0.020210443064570427,
0.017597980797290802,
0.018590020015835762,
0.018625281751155853,
0.019812360405921936,
0.01988193951547146,
0.021170100197196007,
0.0208931565284729,
0.022594628855586052,
0.019704710692167282,
0.021153371781110764,
0.021027294918894768,
0.022732628509402275,
0.02250491827726364,
0.024445844814181328,
0.023921677842736244,
0.02632913924753666,
0.022959763184189796,
0.02505173534154892,
0.02448389306664467,
0.026562821120023727,
0.025734128430485725,
0.028511708602309227,
0.027005694806575775
],
"loss_nats": 3.5924854278564453,
"loss_scale": 1.0,
"output_weights": {
"entropy_mean": 2.1922647953033447,
"mean_weights": [
0.09991011023521423,
0.0929517149925232,
0.10029369592666626,
0.10801893472671509,
0.11530699580907822,
0.11817377060651779,
0.12114627659320831,
0.12246126681566238,
0.12173722684383392
],
"sources": 9
},
"step": 100,
"stream_state_rms_by_sublayer": [
0.002723898272961378,
0.0055193523876369,
0.006246842443943024,
0.009546536952257156,
0.009986688382923603,
0.013977172784507275,
0.014312135986983776,
0.018744563683867455,
0.0027809101156890392,
0.00660830270498991,
0.007167908828705549,
0.011765030212700367,
0.012327533215284348,
0.017156247049570084,
0.017696699127554893,
0.022267481312155724,
0.003559867851436138,
0.007449587807059288,
0.008642485365271568,
0.013175037689507008,
0.014099182561039925,
0.019073888659477234,
0.020258057862520218,
0.025712018832564354,
0.004887574352324009,
0.008972279727458954,
0.011225050315260887,
0.01624389924108982,
0.01857108250260353,
0.023743286728858948,
0.026577936485409737,
0.03214457631111145,
0.005625482182949781,
0.00991375558078289,
0.013686468824744225,
0.019281653687357903,
0.023031942546367645,
0.02868141047656536,
0.03310495615005493,
0.03889070823788643,
0.008495661430060863,
0.0127727584913373,
0.017283262684941292,
0.022841427475214005,
0.029314789921045303,
0.034929387271404266,
0.04071469604969025,
0.04634999856352806,
0.00961991585791111,
0.014845813624560833,
0.023407738655805588,
0.029589947313070297,
0.0400763601064682,
0.046351123601198196,
0.05648425966501236,
0.06254568696022034,
0.014384749345481396,
0.01947367936372757,
0.03149676322937012,
0.0370744951069355,
0.04818879812955856,
0.05507569760084152,
0.06738673150539398,
0.07403328269720078
]
},
{
"activation_grad_rms_by_block": [
0.0001494410098530352,
0.00015486938355024904,
0.00014897517394274473,
0.0001431136333849281,
0.0001401594199705869,
0.00012453079398255795,
0.00011206007184227929,
0.00010208225285168737,
9.998067253036425e-05,
9.082697215490043e-05,
8.336143946507946e-05,
7.669178739888594e-05,
7.405834912788123e-05,
6.847811891930178e-05,
6.22319639660418e-05,
5.739524931414053e-05,
5.947135286987759e-05,
5.3890027629677206e-05,
4.890635682386346e-05,
4.4437034375732765e-05,
4.31341104558669e-05,
3.886142803821713e-05,
3.5836677852785215e-05,
3.340865805512294e-05,
3.223997919121757e-05,
3.0436871384154074e-05,
2.9292581530171447e-05,
2.8587397537194192e-05,
2.2913789507583715e-05,
2.2530648493557237e-05,
2.244284223706927e-05,
2.2519136109622195e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 0.00013440396742225857,
"first_to_last_ratio": 5.096772825661455,
"imbalance_abs_log_ratio": 1.6286075601589056,
"last_quartile_mean": 2.6370405748821213e-05,
"mean": 7.053641201082428e-05,
"normalized": [
2.1186363977530145,
2.195594858531836,
2.112032207136982,
2.0289327073081966,
1.9870506023056362,
1.7654824002594274,
1.5886840377574483,
1.4472277500593902,
1.4174334883240383,
1.287660792002894,
1.1818213754945048,
1.0872652182409996,
1.0499307664886113,
0.9708194245660434,
0.8822672176250175,
0.8136967514782698,
0.8431298271983457,
0.7640029609304118,
0.6933490863748281,
0.6299871670381194,
0.6115155169679979,
0.5509413780821966,
0.5080592679889337,
0.47363704932987183,
0.4570686014801849,
0.43150580695093105,
0.4152831239229507,
0.4052856775987878,
0.32485051130850606,
0.319418692435047,
0.31817385655546626,
0.3192554805051111
],
"population_cv": 0.606636113722199
},
"activation_output_rms_by_block": [
0.021243074908852577,
0.03624089062213898,
0.04951142147183418,
0.05851924046874046,
0.017167888581752777,
0.02674153819680214,
0.03352004662156105,
0.03946679085493088,
0.020600929856300354,
0.027865095064044,
0.035815127193927765,
0.04293550178408623,
0.021564343944191933,
0.029616132378578186,
0.035480253398418427,
0.04259813204407692,
0.021436138078570366,
0.03586751967668533,
0.04331304505467415,
0.051551222801208496,
0.025608548894524574,
0.04091007262468338,
0.05466190353035927,
0.07478146255016327,
0.02793189324438572,
0.05498877540230751,
0.08125203847885132,
0.10916247963905334,
0.026290444657206535,
0.05761713907122612,
0.08842677623033524,
0.12436132878065109
],
"activation_output_statistics": {
"first_quartile_mean": 0.03530136146582663,
"first_to_last_ratio": 0.495430868506741,
"imbalance_abs_log_ratio": 0.7023274536001348,
"last_quartile_mean": 0.07125385943800211,
"mean": 0.04553272487828508,
"normalized": [
0.4665452148018396,
0.7959306349228079,
1.0873810342821493,
1.2852127920121195,
0.37704505117242126,
0.5873037088881845,
0.7361748437231577,
0.8667785853016872,
0.452442279950724,
0.6119795188741951,
0.7865799222354093,
0.9429591991003927,
0.4736009980038805,
0.6504361963345259,
0.7792253482140983,
0.9355498085815704,
0.47078531179216626,
0.7877305777891374,
0.9512508897821419,
1.1321796123340224,
0.5624207416309823,
0.8984762658953829,
1.2004970859195814,
1.6423673907077576,
0.6134465556157976,
1.2076759198861848,
1.784475642431869,
2.397451062523908,
0.5773966905667149,
1.2654006371295428,
1.9420488553389141,
2.7312516242567333
],
"population_cv": 0.5563959277170655
},
"bits_per_byte": 3.0745070201808953,
"branch_output_rms_by_sublayer": [
0.01049753837287426,
0.01706349104642868,
0.013886291533708572,
0.012710319831967354,
0.01245343592017889,
0.013922877609729767,
0.01395952794700861,
0.011416541412472725,
0.014544809237122536,
0.008325628004968166,
0.014345151372253895,
0.00801781378686428,
0.015135552734136581,
0.007913725450634956,
0.014776745811104774,
0.008828459307551384,
0.01795734278857708,
0.008378393016755581,
0.013709688559174538,
0.008136541582643986,
0.016144096851348877,
0.008199729025363922,
0.018440017476677895,
0.009040302596986294,
0.018456056714057922,
0.00936646107584238,
0.012218938209116459,
0.009016151539981365,
0.01701919361948967,
0.009497544728219509,
0.01305371057242155,
0.009566655382514,
0.017418840900063515,
0.011108046397566795,
0.026864025741815567,
0.0114391278475523,
0.035265542566776276,
0.012216469272971153,
0.027374446392059326,
0.01392423827201128,
0.01969025284051895,
0.015176027081906796,
0.018602686002850533,
0.017532862722873688,
0.0133047541603446,
0.018514130264520645,
0.015662170946598053,
0.022848375141620636,
0.012774706818163395,
0.0230264812707901,
0.016374029219150543,
0.02426965720951557,
0.010044055990874767,
0.025484448298811913,
0.006929588038474321,
0.027965376153588295,
0.005331806372851133,
0.02452327311038971,
0.01223068032413721,
0.02779858186841011,
0.00525518087670207,
0.03124845214188099,
0.010387555696070194,
0.03395458683371544
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.0004975869912568777,
0.0004115293940014644,
0.00043817390286628833,
0.00039988528646951293,
0.0003968514715139995,
0.00031053290415162903,
0.00024154094413850278,
0.00022869177850848996,
0.0002476899080691786,
0.00021028907646624228,
0.0002010354933691209,
0.0001845208571901889,
0.0002273114134138899,
0.0001715437435350543,
0.00015668460984383524,
0.00015894955615086492,
0.00016556491904516564,
0.0001408361881569501,
0.00014198814555526783,
0.00012254378127881199,
0.00011450446233020633,
0.0001100879077446283,
0.00010549256620556044,
9.975908855450905e-05,
0.00010071115476761671,
9.794352566848755e-05,
9.525064309078406e-05,
8.226123370955787e-05,
6.340888944803813e-05,
6.739220497758678e-05,
6.650164980556211e-05,
7.228417264196778e-05
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.0003655990841133456,
"first_to_last_ratio": 4.5292712934136725,
"imbalance_abs_log_ratio": 1.5105610641741143,
"last_quartile_mean": 8.071918426370012e-05,
"mean": 0.0001915421207476825,
"normalized": [
2.5977941004023166,
2.148505991241321,
2.28761121134026,
2.0877146233348847,
2.071875731379053,
1.621225153712757,
1.2610330469123472,
1.1939503312158923,
1.293135458155751,
1.0978738026152224,
1.0495628459389565,
0.963343501000792,
1.186743743499249,
0.8955927963282188,
0.8180164719499744,
0.8298412669255573,
0.8643786463201141,
0.7352752888356756,
0.7412894093529857,
0.6397745874404216,
0.5978030413531993,
0.574745164744451,
0.5507538801062211,
0.5208206328983963,
0.5257911647552604,
0.5113419716048152,
0.4972830138821386,
0.4294681158820424,
0.33104410246958865,
0.3518401316353921,
0.3471907356250086,
0.3773800371417385
],
"population_cv": 0.6210714087578444
},
"depth_weights": [
{
"entropy_mean": 0.0,
"mean_weights": [
1.0
],
"sources": 1
},
{
"entropy_mean": 0.6506600379943848,
"mean_weights": [
0.3747701942920685,
0.6252298355102539
],
"sources": 2
},
{
"entropy_mean": 0.6806463003158569,
"mean_weights": [
0.45339685678482056,
0.5466030836105347
],
"sources": 2
},
{
"entropy_mean": 0.6869370937347412,
"mean_weights": [
0.4668571949005127,
0.5331428050994873
],
"sources": 2
},
{
"entropy_mean": 0.6791321635246277,
"mean_weights": [
0.4727538824081421,
0.5272461175918579
],
"sources": 2
},
{
"entropy_mean": 0.6804035902023315,
"mean_weights": [
0.4487338662147522,
0.5512661933898926
],
"sources": 2
},
{
"entropy_mean": 0.682766318321228,
"mean_weights": [
0.4646179676055908,
0.5353820323944092
],
"sources": 2
},
{
"entropy_mean": 0.6873732805252075,
"mean_weights": [
0.4883750379085541,
0.5116249322891235
],
"sources": 2
},
{
"entropy_mean": 0.6835889220237732,
"mean_weights": [
0.46665582060813904,
0.5333442091941833
],
"sources": 2
},
{
"entropy_mean": 1.0856965780258179,
"mean_weights": [
0.27463093400001526,
0.3524433374404907,
0.372925728559494
],
"sources": 3
},
{
"entropy_mean": 1.077350378036499,
"mean_weights": [
0.2904614210128784,
0.3131614923477173,
0.3963770568370819
],
"sources": 3
},
{
"entropy_mean": 1.0850789546966553,
"mean_weights": [
0.27783387899398804,
0.34068456292152405,
0.3814815878868103
],
"sources": 3
},
{
"entropy_mean": 1.062077522277832,
"mean_weights": [
0.28390786051750183,
0.29115602374076843,
0.42493611574172974
],
"sources": 3
},
{
"entropy_mean": 1.0936429500579834,
"mean_weights": [
0.30466002225875854,
0.3333379328250885,
0.36200207471847534
],
"sources": 3
},
{
"entropy_mean": 1.0627415180206299,
"mean_weights": [
0.2850727438926697,
0.29934102296829224,
0.4155862331390381
],
"sources": 3
},
{
"entropy_mean": 1.087097406387329,
"mean_weights": [
0.28865373134613037,
0.3320052921772003,
0.3793409764766693
],
"sources": 3
},
{
"entropy_mean": 1.0500720739364624,
"mean_weights": [
0.2745887339115143,
0.286259263753891,
0.4391520023345947
],
"sources": 3
},
{
"entropy_mean": 1.3743348121643066,
"mean_weights": [
0.2166871875524521,
0.23293715715408325,
0.2495419979095459,
0.30083364248275757
],
"sources": 4
},
{
"entropy_mean": 1.3609715700149536,
"mean_weights": [
0.23144879937171936,
0.21839870512485504,
0.24986028671264648,
0.3002921938896179
],
"sources": 4
},
{
"entropy_mean": 1.3810515403747559,
"mean_weights": [
0.2385442852973938,
0.23454180359840393,
0.24874086678028107,
0.27817302942276
],
"sources": 4
},
{
"entropy_mean": 1.348030686378479,
"mean_weights": [
0.23874622583389282,
0.21300309896469116,
0.22721335291862488,
0.32103732228279114
],
"sources": 4
},
{
"entropy_mean": 1.3825427293777466,
"mean_weights": [
0.24534732103347778,
0.2375141680240631,
0.25072187185287476,
0.26641663908958435
],
"sources": 4
},
{
"entropy_mean": 1.3499886989593506,
"mean_weights": [
0.24629786610603333,
0.22150886058807373,
0.2235339879989624,
0.30865928530693054
],
"sources": 4
},
{
"entropy_mean": 1.3807358741760254,
"mean_weights": [
0.24433431029319763,
0.23110222816467285,
0.23992134630680084,
0.2846421003341675
],
"sources": 4
},
{
"entropy_mean": 1.3399771451950073,
"mean_weights": [
0.24593281745910645,
0.21928201615810394,
0.21277041733264923,
0.3220147490501404
],
"sources": 4
},
{
"entropy_mean": 1.6075894832611084,
"mean_weights": [
0.19650746881961823,
0.19567455351352692,
0.18970930576324463,
0.21128275990486145,
0.20682592689990997
],
"sources": 5
},
{
"entropy_mean": 1.5706448554992676,
"mean_weights": [
0.1975565403699875,
0.1772933304309845,
0.16386017203330994,
0.21728253364562988,
0.2440074384212494
],
"sources": 5
},
{
"entropy_mean": 1.6042816638946533,
"mean_weights": [
0.21478912234306335,
0.18552109599113464,
0.17930489778518677,
0.20310312509536743,
0.2172817587852478
],
"sources": 5
},
{
"entropy_mean": 1.5764570236206055,
"mean_weights": [
0.20157726109027863,
0.17631688714027405,
0.1703747808933258,
0.19828173518180847,
0.25344932079315186
],
"sources": 5
},
{
"entropy_mean": 1.6011263132095337,
"mean_weights": [
0.20813238620758057,
0.18490281701087952,
0.17333751916885376,
0.20533671975135803,
0.22829055786132812
],
"sources": 5
},
{
"entropy_mean": 1.5817476511001587,
"mean_weights": [
0.20727968215942383,
0.18177060782909393,
0.17236655950546265,
0.18925222754478455,
0.24933092296123505
],
"sources": 5
},
{
"entropy_mean": 1.603710412979126,
"mean_weights": [
0.2016504406929016,
0.1979413628578186,
0.17889465391635895,
0.19759604334831238,
0.22391751408576965
],
"sources": 5
},
{
"entropy_mean": 1.5885388851165771,
"mean_weights": [
0.20988406240940094,
0.18461555242538452,
0.17252585291862488,
0.19482356309890747,
0.2381509691476822
],
"sources": 5
},
{
"entropy_mean": 1.783994197845459,
"mean_weights": [
0.16140900552272797,
0.16211673617362976,
0.15125815570354462,
0.1585075855255127,
0.17266258597373962,
0.19404590129852295
],
"sources": 6
},
{
"entropy_mean": 1.765566349029541,
"mean_weights": [
0.1630604863166809,
0.1509958952665329,
0.14303800463676453,
0.14991191029548645,
0.16761469841003418,
0.22537899017333984
],
"sources": 6
},
{
"entropy_mean": 1.7848252058029175,
"mean_weights": [
0.16126418113708496,
0.1664336919784546,
0.15361233055591583,
0.1546509563922882,
0.16693615913391113,
0.19710269570350647
],
"sources": 6
},
{
"entropy_mean": 1.7731435298919678,
"mean_weights": [
0.16123875975608826,
0.15981441736221313,
0.1509810984134674,
0.14913737773895264,
0.16297483444213867,
0.2158534824848175
],
"sources": 6
},
{
"entropy_mean": 1.7815005779266357,
"mean_weights": [
0.15527218580245972,
0.17254948616027832,
0.15193141996860504,
0.15522462129592896,
0.1650869995355606,
0.19993525743484497
],
"sources": 6
},
{
"entropy_mean": 1.7801456451416016,
"mean_weights": [
0.1661754697561264,
0.16477231681346893,
0.15300315618515015,
0.15292593836784363,
0.1582789272069931,
0.2048441767692566
],
"sources": 6
},
{
"entropy_mean": 1.7743862867355347,
"mean_weights": [
0.15205824375152588,
0.1771804541349411,
0.15399661660194397,
0.15122173726558685,
0.15813913941383362,
0.20740382373332977
],
"sources": 6
},
{
"entropy_mean": 1.781898856163025,
"mean_weights": [
0.17058727145195007,
0.1644459068775177,
0.15141303837299347,
0.15486885607242584,
0.15909278392791748,
0.19959214329719543
],
"sources": 6
},
{
"entropy_mean": 1.9389331340789795,
"mean_weights": [
0.12895603477954865,
0.15703952312469482,
0.13832643628120422,
0.13419194519519806,
0.1312965601682663,
0.15331263840198517,
0.15687686204910278
],
"sources": 7
},
{
"entropy_mean": 1.941878080368042,
"mean_weights": [
0.14007402956485748,
0.15248695015907288,
0.14026451110839844,
0.13737857341766357,
0.1305495798587799,
0.14631187915802002,
0.1529344916343689
],
"sources": 7
},
{
"entropy_mean": 1.936349630355835,
"mean_weights": [
0.12599323689937592,
0.16029544174671173,
0.1399340182542801,
0.13338115811347961,
0.12978437542915344,
0.14873966574668884,
0.16187208890914917
],
"sources": 7
},
{
"entropy_mean": 1.9385986328125,
"mean_weights": [
0.1397915780544281,
0.15300613641738892,
0.14345945417881012,
0.13018865883350372,
0.1257915198802948,
0.14249205589294434,
0.1652706116437912
],
"sources": 7
},
{
"entropy_mean": 1.939380407333374,
"mean_weights": [
0.12855647504329681,
0.15473899245262146,
0.1372847855091095,
0.13797816634178162,
0.13512183725833893,
0.15090687572956085,
0.15541282296180725
],
"sources": 7
},
{
"entropy_mean": 1.9388402700424194,
"mean_weights": [
0.14282521605491638,
0.15391701459884644,
0.1462559849023819,
0.13105326890945435,
0.12675246596336365,
0.13615548610687256,
0.16304057836532593
],
"sources": 7
},
{
"entropy_mean": 1.9313902854919434,
"mean_weights": [
0.11694864928722382,
0.17515113949775696,
0.15024805068969727,
0.1394161432981491,
0.13194763660430908,
0.1451682150363922,
0.14112016558647156
],
"sources": 7
},
{
"entropy_mean": 1.9366493225097656,
"mean_weights": [
0.14114168286323547,
0.16076743602752686,
0.14226865768432617,
0.12737974524497986,
0.12711969017982483,
0.1412842869758606,
0.16003848612308502
],
"sources": 7
},
{
"entropy_mean": 2.064785957336426,
"mean_weights": [
0.106104776263237,
0.15484434366226196,
0.13487765192985535,
0.12357068061828613,
0.12023378908634186,
0.12704278528690338,
0.11963208019733429,
0.11369390785694122
],
"sources": 8
},
{
"entropy_mean": 2.062044143676758,
"mean_weights": [
0.11719764769077301,
0.14862585067749023,
0.12573552131652832,
0.12179906666278839,
0.11081404983997345,
0.1260838657617569,
0.12823458015918732,
0.12150939553976059
],
"sources": 8
},
{
"entropy_mean": 2.054962158203125,
"mean_weights": [
0.10514847934246063,
0.16456037759780884,
0.14191412925720215,
0.1262327879667282,
0.1181878000497818,
0.12522782385349274,
0.1187765821814537,
0.09995201230049133
],
"sources": 8
},
{
"entropy_mean": 2.0610406398773193,
"mean_weights": [
0.11198848485946655,
0.15103809535503387,
0.13588255643844604,
0.12388211488723755,
0.11551118642091751,
0.12874530255794525,
0.12429477274417877,
0.10865749418735504
],
"sources": 8
},
{
"entropy_mean": 2.0338308811187744,
"mean_weights": [
0.0961676836013794,
0.17635446786880493,
0.14848342537879944,
0.13749772310256958,
0.12027012556791306,
0.12292921543121338,
0.10873906314373016,
0.08955828845500946
],
"sources": 8
},
{
"entropy_mean": 2.068699598312378,
"mean_weights": [
0.11418519914150238,
0.14292436838150024,
0.1359846293926239,
0.13056153059005737,
0.12100572884082794,
0.11985942721366882,
0.12221725285053253,
0.1132618635892868
],
"sources": 8
},
{
"entropy_mean": 2.036879539489746,
"mean_weights": [
0.09746269136667252,
0.17786666750907898,
0.15074899792671204,
0.13200224936008453,
0.11842608451843262,
0.12164927273988724,
0.1129462867975235,
0.08889776468276978
],
"sources": 8
},
{
"entropy_mean": 2.06752347946167,
"mean_weights": [
0.11505836993455887,
0.14705273509025574,
0.1377231776714325,
0.12483140081167221,
0.11869609355926514,
0.1215740442276001,
0.12386028468608856,
0.1112038791179657
],
"sources": 8
},
{
"entropy_mean": 2.1517844200134277,
"mean_weights": [
0.08786984533071518,
0.16287443041801453,
0.13886377215385437,
0.12039706110954285,
0.10935483872890472,
0.1119438111782074,
0.10067413747310638,
0.07631799578666687,
0.0917041003704071
],
"sources": 9
},
{
"entropy_mean": 2.177518129348755,
"mean_weights": [
0.10418819636106491,
0.13366681337356567,
0.12014994025230408,
0.11398349702358246,
0.10732515156269073,
0.12029910087585449,
0.11035183072090149,
0.09321349114179611,
0.09682197868824005
],
"sources": 9
},
{
"entropy_mean": 2.1535391807556152,
"mean_weights": [
0.08976081013679504,
0.163266122341156,
0.13892531394958496,
0.12143255770206451,
0.10986581444740295,
0.11029069125652313,
0.10418520867824554,
0.08120252937078476,
0.0810709297657013
],
"sources": 9
},
{
"entropy_mean": 2.191593647003174,
"mean_weights": [
0.10739251971244812,
0.12338980287313461,
0.11759916692972183,
0.1066875234246254,
0.1060015857219696,
0.1124890148639679,
0.11740898340940475,
0.10738331079483032,
0.10164809972047806
],
"sources": 9
},
{
"entropy_mean": 2.1465134620666504,
"mean_weights": [
0.08895281702280045,
0.1655423492193222,
0.14156755805015564,
0.12290289252996445,
0.11169517785310745,
0.11284246295690536,
0.1037452220916748,
0.0791342705488205,
0.07361726462841034
],
"sources": 9
},
{
"entropy_mean": 2.1857833862304688,
"mean_weights": [
0.10998444259166718,
0.12038750946521759,
0.11211743205785751,
0.11047163605690002,
0.10957512259483337,
0.12139561027288437,
0.11392621695995331,
0.10294321179389954,
0.09919881820678711
],
"sources": 9
},
{
"entropy_mean": 2.1400556564331055,
"mean_weights": [
0.08778931945562363,
0.16802933812141418,
0.1466498225927353,
0.12668846547603607,
0.11206841468811035,
0.11208955943584442,
0.10068599879741669,
0.07686632871627808,
0.0691327303647995
],
"sources": 9
}
],
"layer_input_rms_by_sublayer": [
0.04377923160791397,
0.017109708860516548,
0.023813940584659576,
0.02606838569045067,
0.029579024761915207,
0.030872320756316185,
0.034570932388305664,
0.034764163196086884,
0.037983279675245285,
0.025188947096467018,
0.02627667225897312,
0.026965361088514328,
0.02788778953254223,
0.028485212475061417,
0.03023744560778141,
0.030349206179380417,
0.031768180429935455,
0.022602196782827377,
0.02424551732838154,
0.023387571796774864,
0.025113960728049278,
0.02450813725590706,
0.026638666167855263,
0.025169003754854202,
0.027698444202542305,
0.020798001438379288,
0.022435806691646576,
0.02118327096104622,
0.022681118920445442,
0.021654823794960976,
0.023235918954014778,
0.022377775982022285,
0.023750189691781998,
0.018887808546423912,
0.019677624106407166,
0.01950881816446781,
0.020757248625159264,
0.020426612347364426,
0.02137788012623787,
0.021234560757875443,
0.02207154594361782,
0.01860968954861164,
0.019271831959486008,
0.019469313323497772,
0.020465917885303497,
0.020844420418143272,
0.021911408752202988,
0.02213398739695549,
0.024193286895751953,
0.0209572184830904,
0.02259150519967079,
0.022487536072731018,
0.024306390434503555,
0.023487187922000885,
0.02611992135643959,
0.024889562278985977,
0.02851196378469467,
0.023824406787753105,
0.027104467153549194,
0.025868752971291542,
0.03068944811820984,
0.0271493811160326,
0.03286270797252655,
0.028410611674189568
],
"loss_nats": 2.1310858726501465,
"loss_scale": 1.0,
"output_weights": {
"entropy_mean": 2.1613364219665527,
"mean_weights": [
0.1255333125591278,
0.08712290227413177,
0.09080196917057037,
0.08838342130184174,
0.09530800580978394,
0.09807588160037994,
0.13817133009433746,
0.15537576377391815,
0.12122740596532822
],
"sources": 9
},
"step": 500,
"stream_state_rms_by_sublayer": [
0.01049753837287426,
0.021243074908852577,
0.028129518032073975,
0.03624089062213898,
0.040982723236083984,
0.04951142147183418,
0.05135732889175415,
0.05851924046874046,
0.014544809237122536,
0.017167888581752777,
0.023477500304579735,
0.02674153819680214,
0.029818039387464523,
0.03352004662156105,
0.03540094941854477,
0.03946679085493088,
0.01795734278857708,
0.020600929856300354,
0.024303091689944267,
0.027865095064044,
0.031844362616539,
0.035815127193927765,
0.03825921565294266,
0.04293550178408623,
0.018456056714057922,
0.021564343944191933,
0.02560909278690815,
0.029616132378578186,
0.0305727981030941,
0.035480253398418427,
0.037264734506607056,
0.04259813204407692,
0.017418840900063515,
0.021436138078570366,
0.02998829260468483,
0.03586751967668533,
0.03708711639046669,
0.04331304505467415,
0.04352690279483795,
0.051551222801208496,
0.01969025284051895,
0.025608548894524574,
0.030230700969696045,
0.04091007262468338,
0.041474223136901855,
0.05466190353035927,
0.056570712476968765,
0.07478146255016327,
0.012774706818163395,
0.02793189324438572,
0.03524070605635643,
0.05498877540230751,
0.05920406058430672,
0.08125203847885132,
0.08430205285549164,
0.10916247963905334,
0.005331806372851133,
0.026290444657206535,
0.032364487648010254,
0.05761713907122612,
0.059356335550546646,
0.08842677623033524,
0.09272631257772446,
0.12436132878065109
]
},
{
"activation_grad_rms_by_block": [
6.294298509601504e-05,
6.491570093203336e-05,
6.380436389008537e-05,
6.04347005719319e-05,
7.516580808442086e-05,
7.751413795631379e-05,
6.965622014831752e-05,
6.359309190884233e-05,
6.543538620462641e-05,
6.0226317145861685e-05,
5.608194260275923e-05,
5.3055387979839e-05,
4.572631951305084e-05,
4.3727712181862444e-05,
4.314268517191522e-05,
4.164052370470017e-05,
7.694400119362399e-05,
7.326623017434031e-05,
6.749018211849034e-05,
6.173704605316743e-05,
0.00010233377543045208,
8.934178913477808e-05,
7.613583147758618e-05,
6.664563989033923e-05,
5.163387322681956e-05,
4.2875533836195245e-05,
3.8466609112219885e-05,
3.59200312232133e-05,
2.308324837940745e-05,
2.0679148292401806e-05,
1.9936938770115376e-05,
1.9895105651812628e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 6.725337607349502e-05,
"first_to_last_ratio": 2.130880302861834,
"imbalance_abs_log_ratio": 0.7565351820700446,
"last_quartile_mean": 3.1561311061523156e-05,
"mean": 5.6670258345548064e-05,
"normalized": [
1.1106881622493918,
1.145498588275503,
1.1258880010928651,
1.0664271230850946,
1.3263713679598177,
1.367809856868302,
1.2291495077292092,
1.122159908308202,
1.154668923611267,
1.062750001572791,
0.9896186154790125,
0.9362122130506743,
0.8068839078557516,
0.7716166020495587,
0.7612932503122151,
0.7347861986228512,
1.357749257546284,
1.2928515294142118,
1.190927729797305,
1.089408233799192,
1.8057756999530474,
1.5765198843812338,
1.3434883429213678,
1.176024987992221,
0.9111282484717285,
0.7565791137700626,
0.678779490957478,
0.6338427293588494,
0.4073256246452393,
0.3649030178459797,
0.3518060328673549,
0.35106784815593767
],
"population_cv": 0.34914284904214865
},
"activation_output_rms_by_block": [
0.10241243988275528,
0.1819411665201187,
0.24157075583934784,
0.26444777846336365,
0.10636615753173828,
0.13034582138061523,
0.14029134809970856,
0.15042904019355774,
0.04812335968017578,
0.07954440265893936,
0.10673297941684723,
0.14089107513427734,
0.06885477155447006,
0.10946234315633774,
0.13876035809516907,
0.1687854677438736,
0.05059739947319031,
0.07094386219978333,
0.09166282415390015,
0.12053482979536057,
0.048727843910455704,
0.07109228521585464,
0.09690473228693008,
0.11853098124265671,
0.05259029567241669,
0.08889466524124146,
0.12455403059720993,
0.1728687882423401,
0.06489895284175873,
0.14679722487926483,
0.23576539754867554,
0.35208994150161743
],
"activation_output_statistics": {
"first_quartile_mean": 0.16472556348890066,
"first_to_last_ratio": 1.0640676779683806,
"imbalance_abs_log_ratio": 0.0620989960097457,
"last_quartile_mean": 0.15480741206556559,
"mean": 0.127700416254811,
"normalized": [
0.8019742056157706,
1.4247499879489567,
1.8916990478505735,
2.0708450780276983,
0.8329350886335192,
1.020715712629516,
1.098597422108428,
1.177983945591802,
0.3768457542389787,
0.6228985385624584,
0.8358076077361742,
1.1032937813855357,
0.5391898755997675,
0.8571807858317286,
1.086608502655851,
1.3217299731199175,
0.396219534415841,
0.5555491851978249,
0.7177958133746243,
0.9438875246486875,
0.3815793662976406,
0.5567114603134765,
0.7588442945524017,
0.9281957312194052,
0.411825561848436,
0.6961188369493074,
0.9753611949758815,
1.3537057537651325,
0.5082125395132666,
1.1495438244018596,
1.8462382853806345,
2.7571557856089037
],
"population_cv": 0.532159231456062
},
"bits_per_byte": 2.0533380169036586,
"branch_output_rms_by_sublayer": [
0.05045261234045029,
0.0860176682472229,
0.09517527371644974,
0.031032679602503777,
0.08140470087528229,
0.011454286053776741,
0.035610511898994446,
0.03601690009236336,
0.1042611226439476,
0.01171922218054533,
0.050615254789590836,
0.008982792496681213,
0.04264022409915924,
0.01206296682357788,
0.03225839138031006,
0.026524480432271957,
0.040291935205459595,
0.024220434948801994,
0.036216676235198975,
0.026388581842184067,
0.02871432714164257,
0.03031591884791851,
0.03194425627589226,
0.032942064106464386,
0.05663999170064926,
0.029961559921503067,
0.03894488140940666,
0.02937687374651432,
0.03595346212387085,
0.031781863421201706,
0.03414257988333702,
0.035546254366636276,
0.03680795058608055,
0.030760737136006355,
0.022694969549775124,
0.03042135201394558,
0.022243596613407135,
0.03508196026086807,
0.02950950525701046,
0.04756028205156326,
0.035728659480810165,
0.0295625701546669,
0.024367405101656914,
0.03350908309221268,
0.0301151555031538,
0.038334235548973083,
0.022455131635069847,
0.043223608285188675,
0.025910304859280586,
0.04338310658931732,
0.025968369096517563,
0.046161677688360214,
0.01597527042031288,
0.050398919731378555,
0.015805471688508987,
0.062427736818790436,
0.013562481850385666,
0.0627857968211174,
0.02184087596833706,
0.08928166329860687,
0.012465676292777061,
0.09919292479753494,
0.01876053772866726,
0.13455425202846527
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.00034319970120439075,
0.0002207216513437987,
0.00019628105547348242,
0.00016658832628724645,
0.00021517433675904697,
0.00019244240293185932,
0.0001402257608174488,
0.00015224450647613049,
0.00018347185792801386,
0.00016427150761043105,
0.00015319316983403424,
0.00015483500137268905,
0.00013970482137555046,
0.00012521709576159806,
0.00014171058058500498,
0.00014304258084243227,
0.00024403444477986432,
0.0002443248488541129,
0.0002486173108247806,
0.0002822915928542698,
0.00039666535280576766,
0.0003607137256748422,
0.0003593407909791253,
0.00030791631238161,
0.0002432000733486066,
0.00020051667741228585,
0.0001766506293272091,
0.00017705575210645458,
0.00010859706957547057,
0.0001081942080930384,
0.00010514694748959741,
0.00011321633644894032
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.0002033597176616755,
"first_to_last_ratio": 1.3198987369921267,
"imbalance_abs_log_ratio": 0.2775550192556754,
"last_quartile_mean": 0.00015407221172520036,
"mean": 0.00020340020092372293,
"normalized": [
1.6873124984428802,
1.085159455799021,
0.9649993194799731,
0.8190175109498476,
1.0578865496781857,
0.9461269067478802,
0.6894081725614166,
0.7484973258862397,
0.9020239758603661,
0.8076270665633928,
0.7531613495872759,
0.7612332764152662,
0.6868470175599342,
0.6156193317063423,
0.6967081642075066,
0.703256831570558,
1.1997748461978148,
1.2012025934317396,
1.2223061220967748,
1.3878629006867693,
1.9501718828415566,
1.7734187283822467,
1.7666688102922847,
1.5138446814862372,
1.1956727291523621,
0.985823398903532,
0.8684879785023163,
0.8704797306117324,
0.5339083692262209,
0.5319277345926137,
0.516946133839015,
0.5566186067406962
],
"population_cv": 0.38898494230064556
},
"depth_weights": [
{
"entropy_mean": 0.0,
"mean_weights": [
1.0
],
"sources": 1
},
{
"entropy_mean": 0.39672940969467163,
"mean_weights": [
0.854300856590271,
0.1456991732120514
],
"sources": 2
},
{
"entropy_mean": 0.6258258819580078,
"mean_weights": [
0.5948886871337891,
0.40511131286621094
],
"sources": 2
},
{
"entropy_mean": 0.5053832530975342,
"mean_weights": [
0.778947651386261,
0.221052348613739
],
"sources": 2
},
{
"entropy_mean": 0.6737473011016846,
"mean_weights": [
0.5406605005264282,
0.4593394994735718
],
"sources": 2
},
{
"entropy_mean": 0.6529731750488281,
"mean_weights": [
0.5619670748710632,
0.43803298473358154
],
"sources": 2
},
{
"entropy_mean": 0.6891955733299255,
"mean_weights": [
0.5099825263023376,
0.49001747369766235
],
"sources": 2
},
{
"entropy_mean": 0.48565131425857544,
"mean_weights": [
0.7992967367172241,
0.20070326328277588
],
"sources": 2
},
{
"entropy_mean": 0.6875144243240356,
"mean_weights": [
0.5191382169723511,
0.4808618128299713
],
"sources": 2
},
{
"entropy_mean": 1.048142671585083,
"mean_weights": [
0.3059995770454407,
0.2586204409599304,
0.4353799819946289
],
"sources": 3
},
{
"entropy_mean": 1.078475832939148,
"mean_weights": [
0.2919672429561615,
0.28997206687927246,
0.4180607199668884
],
"sources": 3
},
{
"entropy_mean": 1.0040717124938965,
"mean_weights": [
0.27878478169441223,
0.21860124170780182,
0.5026139616966248
],
"sources": 3
},
{
"entropy_mean": 1.0727062225341797,
"mean_weights": [
0.3306647837162018,
0.2443503737449646,
0.42498481273651123
],
"sources": 3
},
{
"entropy_mean": 1.036304235458374,
"mean_weights": [
0.3365095853805542,
0.20819777250289917,
0.455292671918869
],
"sources": 3
},
{
"entropy_mean": 1.071068525314331,
"mean_weights": [
0.3291535973548889,
0.24366843700408936,
0.42717796564102173
],
"sources": 3
},
{
"entropy_mean": 1.0272725820541382,
"mean_weights": [
0.29237720370292664,
0.2188395857810974,
0.48878318071365356
],
"sources": 3
},
{
"entropy_mean": 1.0856674909591675,
"mean_weights": [
0.3748946189880371,
0.26831528544425964,
0.35679009556770325
],
"sources": 3
},
{
"entropy_mean": 1.3326265811920166,
"mean_weights": [
0.26391589641571045,
0.14978256821632385,
0.2545396089553833,
0.3317619264125824
],
"sources": 4
},
{
"entropy_mean": 1.3771977424621582,
"mean_weights": [
0.27525436878204346,
0.2067744880914688,
0.23979949951171875,
0.2781716585159302
],
"sources": 4
},
{
"entropy_mean": 1.3576545715332031,
"mean_weights": [
0.2848266065120697,
0.17341844737529755,
0.23106196522712708,
0.3106929659843445
],
"sources": 4
},
{
"entropy_mean": 1.3772028684616089,
"mean_weights": [
0.2702856659889221,
0.21318203210830688,
0.23082655668258667,
0.2857057452201843
],
"sources": 4
},
{
"entropy_mean": 1.3277652263641357,
"mean_weights": [
0.3119839131832123,
0.16646917164325714,
0.1912841796875,
0.3302627205848694
],
"sources": 4
},
{
"entropy_mean": 1.3750126361846924,
"mean_weights": [
0.2621780037879944,
0.21051618456840515,
0.23754313588142395,
0.2897626757621765
],
"sources": 4
},
{
"entropy_mean": 1.3568847179412842,
"mean_weights": [
0.2628105878829956,
0.1954234540462494,
0.21842113137245178,
0.3233448266983032
],
"sources": 4
},
{
"entropy_mean": 1.3733241558074951,
"mean_weights": [
0.25842776894569397,
0.22964225709438324,
0.21348324418067932,
0.2984467148780823
],
"sources": 4
},
{
"entropy_mean": 1.5855276584625244,
"mean_weights": [
0.18060272932052612,
0.1563272476196289,
0.18859915435314178,
0.24991817772388458,
0.2245527058839798
],
"sources": 5
},
{
"entropy_mean": 1.5906484127044678,
"mean_weights": [
0.19773373007774353,
0.18790484964847565,
0.15935488045215607,
0.18813443183898926,
0.2668721079826355
],
"sources": 5
},
{
"entropy_mean": 1.5916568040847778,
"mean_weights": [
0.2148856818675995,
0.14808836579322815,
0.18413493037223816,
0.22754491865634918,
0.22534608840942383
],
"sources": 5
},
{
"entropy_mean": 1.5474352836608887,
"mean_weights": [
0.19825294613838196,
0.1729484647512436,
0.11353601515293121,
0.19613605737686157,
0.3191264867782593
],
"sources": 5
},
{
"entropy_mean": 1.5919160842895508,
"mean_weights": [
0.21136541664600372,
0.1572093367576599,
0.18091553449630737,
0.2257765829563141,
0.2247331142425537
],
"sources": 5
},
{
"entropy_mean": 1.542572021484375,
"mean_weights": [
0.2053474485874176,
0.19789552688598633,
0.10896679759025574,
0.1649853140115738,
0.32280486822128296
],
"sources": 5
},
{
"entropy_mean": 1.5774884223937988,
"mean_weights": [
0.1874420940876007,
0.14777487516403198,
0.18392466008663177,
0.23523849248886108,
0.24561989307403564
],
"sources": 5
},
{
"entropy_mean": 1.585869312286377,
"mean_weights": [
0.2123720645904541,
0.19153764843940735,
0.14032581448554993,
0.1918477714061737,
0.26391667127609253
],
"sources": 5
},
{
"entropy_mean": 1.747511625289917,
"mean_weights": [
0.13209694623947144,
0.11971477419137955,
0.14703533053398132,
0.17972922325134277,
0.15732964873313904,
0.2640940546989441
],
"sources": 6
},
{
"entropy_mean": 1.6791677474975586,
"mean_weights": [
0.1429731547832489,
0.128341943025589,
0.08682706952095032,
0.12974600493907928,
0.16802877187728882,
0.3440830111503601
],
"sources": 6
},
{
"entropy_mean": 1.7079570293426514,
"mean_weights": [
0.12207645177841187,
0.11454501748085022,
0.14774824678897858,
0.1675407588481903,
0.1342848241329193,
0.3138047158718109
],
"sources": 6
},
{
"entropy_mean": 1.689455270767212,
"mean_weights": [
0.14791087806224823,
0.15263086557388306,
0.0942135751247406,
0.12123478949069977,
0.14674627780914307,
0.3372636139392853
],
"sources": 6
},
{
"entropy_mean": 1.6673545837402344,
"mean_weights": [
0.119766965508461,
0.10584045946598053,
0.1391725242137909,
0.15867865085601807,
0.1261359602212906,
0.3504054546356201
],
"sources": 6
},
{
"entropy_mean": 1.7276721000671387,
"mean_weights": [
0.16197484731674194,
0.16130125522613525,
0.10335841774940491,
0.12991490960121155,
0.15279719233512878,
0.2906533479690552
],
"sources": 6
},
{
"entropy_mean": 1.655259609222412,
"mean_weights": [
0.12291313707828522,
0.10535868257284164,
0.14431551098823547,
0.1644165962934494,
0.11736815422773361,
0.34562790393829346
],
"sources": 6
},
{
"entropy_mean": 1.7456457614898682,
"mean_weights": [
0.17144063115119934,
0.17272350192070007,
0.09372985363006592,
0.13482090830802917,
0.1837429702281952,
0.2435421198606491
],
"sources": 6
},
{
"entropy_mean": 1.8196223974227905,
"mean_weights": [
0.097611203789711,
0.08662202209234238,
0.10460364818572998,
0.12231925129890442,
0.09176328778266907,
0.22007866203784943,
0.27700191736221313
],
"sources": 7
},
{
"entropy_mean": 1.8201665878295898,
"mean_weights": [
0.12032124400138855,
0.12977322936058044,
0.07435770332813263,
0.09230788052082062,
0.1133352667093277,
0.15118572115898132,
0.31871896982192993
],
"sources": 7
},
{
"entropy_mean": 1.7483060359954834,
"mean_weights": [
0.09602338075637817,
0.07951871305704117,
0.09892410784959793,
0.11023538559675217,
0.08284324407577515,
0.17450374364852905,
0.35795143246650696
],
"sources": 7
},
{
"entropy_mean": 1.851926565170288,
"mean_weights": [
0.14006076753139496,
0.14811867475509644,
0.08932580053806305,
0.09198777377605438,
0.11175547540187836,
0.12843111157417297,
0.29032039642333984
],
"sources": 7
},
{
"entropy_mean": 1.7523608207702637,
"mean_weights": [
0.10785886645317078,
0.08607444912195206,
0.09595336019992828,
0.10649852454662323,
0.0838143527507782,
0.14955362677574158,
0.3702468276023865
],
"sources": 7
},
{
"entropy_mean": 1.8693451881408691,
"mean_weights": [
0.1378144919872284,
0.1244172528386116,
0.09968579560518265,
0.10974414646625519,
0.11655184626579285,
0.130243182182312,
0.2815432548522949
],
"sources": 7
},
{
"entropy_mean": 1.7898914813995361,
"mean_weights": [
0.10115306824445724,
0.09615937620401382,
0.11218561232089996,
0.1163020133972168,
0.09238721430301666,
0.14564841985702515,
0.33616429567337036
],
"sources": 7
},
{
"entropy_mean": 1.9094455242156982,
"mean_weights": [
0.14588291943073273,
0.14179158210754395,
0.1023377850651741,
0.11511939018964767,
0.13270339369773865,
0.13753530383110046,
0.22462961077690125
],
"sources": 7
},
{
"entropy_mean": 1.9082129001617432,
"mean_weights": [
0.09145341068506241,
0.07698438316583633,
0.08525429666042328,
0.08438773453235626,
0.06885559856891632,
0.11482041329145432,
0.2519546449184418,
0.2262895107269287
],
"sources": 8
},
{
"entropy_mean": 2.025789499282837,
"mean_weights": [
0.11047068238258362,
0.12069687247276306,
0.09073689579963684,
0.09690164029598236,
0.10987945646047592,
0.11353722959756851,
0.15697816014289856,
0.20079906284809113
],
"sources": 8
},
{
"entropy_mean": 1.9302079677581787,
"mean_weights": [
0.0971018448472023,
0.08240298926830292,
0.0830029845237732,
0.08926018327474594,
0.07480385154485703,
0.10335960984230042,
0.22211775183677673,
0.24795079231262207
],
"sources": 8
},
{
"entropy_mean": 2.010532855987549,
"mean_weights": [
0.10700424760580063,
0.09611141681671143,
0.09769981354475021,
0.10596007108688354,
0.09634587168693542,
0.11705268919467926,
0.17551329731941223,
0.20431260764598846
],
"sources": 8
},
{
"entropy_mean": 1.9408204555511475,
"mean_weights": [
0.08590779453516006,
0.09530991315841675,
0.09476494789123535,
0.1030702069401741,
0.08048167079687119,
0.12287819385528564,
0.22555866837501526,
0.19202858209609985
],
"sources": 8
},
{
"entropy_mean": 2.0024678707122803,
"mean_weights": [
0.10720442980527878,
0.09982497990131378,
0.10896851122379303,
0.1345594823360443,
0.09260416775941849,
0.1016254797577858,
0.1643495261669159,
0.19086343050003052
],
"sources": 8
},
{
"entropy_mean": 1.9389225244522095,
"mean_weights": [
0.08632777631282806,
0.09041796624660492,
0.09941554069519043,
0.10805264860391617,
0.07865813374519348,
0.11747924983501434,
0.2661469578742981,
0.1535017192363739
],
"sources": 8
},
{
"entropy_mean": 2.008256673812866,
"mean_weights": [
0.1016545370221138,
0.10481878370046616,
0.10823093354701996,
0.1270347535610199,
0.0919865146279335,
0.14368976652622223,
0.20522482693195343,
0.11735989153385162
],
"sources": 8
},
{
"entropy_mean": 2.094825267791748,
"mean_weights": [
0.08519881963729858,
0.09903685003519058,
0.08900875598192215,
0.09697237610816956,
0.071251779794693,
0.09201781451702118,
0.19253811240196228,
0.1091965064406395,
0.16477900743484497
],
"sources": 9
},
{
"entropy_mean": 2.140371322631836,
"mean_weights": [
0.10001853853464127,
0.10924500226974487,
0.10232416540384293,
0.11669869720935822,
0.10123330354690552,
0.1287832260131836,
0.14476822316646576,
0.09069216251373291,
0.10623672604560852
],
"sources": 9
},
{
"entropy_mean": 2.117004632949829,
"mean_weights": [
0.09184888750314713,
0.09620077908039093,
0.09620209038257599,
0.10222946852445602,
0.06924223154783249,
0.0821654349565506,
0.17978665232658386,
0.14873988926410675,
0.13358455896377563
],
"sources": 9
},
{
"entropy_mean": 2.1423683166503906,
"mean_weights": [
0.09596475213766098,
0.07681103050708771,
0.11225531250238419,
0.11592596024274826,
0.0810609757900238,
0.11778091639280319,
0.1649385243654251,
0.12366649508476257,
0.11159604042768478
],
"sources": 9
},
{
"entropy_mean": 2.093705415725708,
"mean_weights": [
0.08838655799627304,
0.09557235240936279,
0.09992480278015137,
0.10721968859434128,
0.07486072182655334,
0.09290309250354767,
0.21980023384094238,
0.12490785866975784,
0.09642469882965088
],
"sources": 9
},
{
"entropy_mean": 2.1627297401428223,
"mean_weights": [
0.0994885265827179,
0.09216944873332977,
0.11588040739297867,
0.12231601774692535,
0.09873615205287933,
0.13436099886894226,
0.13960200548171997,
0.1002899631857872,
0.09715649485588074
],
"sources": 9
},
{
"entropy_mean": 2.1300058364868164,
"mean_weights": [
0.089942067861557,
0.1065598577260971,
0.11013364791870117,
0.12377548217773438,
0.08726659417152405,
0.11411209404468536,
0.1931532621383667,
0.10132836550474167,
0.07372863590717316
],
"sources": 9
}
],
"layer_input_rms_by_sublayer": [
0.05571148172020912,
0.04887383058667183,
0.061161335557699203,
0.06501953303813934,
0.09595561027526855,
0.11691275984048843,
0.12880638241767883,
0.07614059001207352,
0.13726800680160522,
0.09499973803758621,
0.10176635533571243,
0.09968593716621399,
0.09925490617752075,
0.09547581523656845,
0.10055620223283768,
0.09975490719079971,
0.0990053340792656,
0.06202486902475357,
0.07309409230947495,
0.06455004960298538,
0.07328634709119797,
0.062386661767959595,
0.07434380799531937,
0.07036563754081726,
0.07861478626728058,
0.05747222527861595,
0.06236656382679939,
0.057312749326229095,
0.06600815057754517,
0.06149744614958763,
0.07462523877620697,
0.06584152579307556,
0.07498960196971893,
0.04952141270041466,
0.05233807489275932,
0.04880477488040924,
0.05626988410949707,
0.0513896569609642,
0.061378348618745804,
0.057161398231983185,
0.06856362521648407,
0.04507989436388016,
0.04801172763109207,
0.04335150495171547,
0.0518198125064373,
0.047289349138736725,
0.05105668306350708,
0.05114771053195,
0.054886870086193085,
0.042588189244270325,
0.04603559896349907,
0.044108107686042786,
0.04718422889709473,
0.05026337504386902,
0.05210597813129425,
0.05349528789520264,
0.05379685014486313,
0.04645879194140434,
0.05137190222740173,
0.05323651432991028,
0.05757243186235428,
0.05820684880018234,
0.06228902190923691,
0.058245208114385605
],
"loss_nats": 1.4232654571533203,
"loss_scale": 1.0,
"output_weights": {
"entropy_mean": 1.9258099794387817,
"mean_weights": [
0.11242504417896271,
0.0684729665517807,
0.07014584541320801,
0.059657223522663116,
0.05741160362958908,
0.05127572640776634,
0.12547241151332855,
0.2706507742404938,
0.18448840081691742
],
"sources": 9
},
"step": 2000,
"stream_state_rms_by_sublayer": [
0.05045261234045029,
0.10241243988275528,
0.16565440595149994,
0.1819411665201187,
0.23719695210456848,
0.24157075583934784,
0.24811406433582306,
0.26444777846336365,
0.1042611226439476,
0.10636615753173828,
0.12873972952365875,
0.13034582138061523,
0.1382850706577301,
0.14029134809970856,
0.14404986798763275,
0.15042904019355774,
0.040291935205459595,
0.04812335968017578,
0.06678888201713562,
0.07954440265893936,
0.09121427685022354,
0.10673297941684723,
0.12122578918933868,
0.14089107513427734,
0.05663999170064926,
0.06885477155447006,
0.09547740966081619,
0.10946234315633774,
0.12204843014478683,
0.13876035809516907,
0.14941293001174927,
0.1687854677438736,
0.03680795058608055,
0.05059739947319031,
0.055156365036964417,
0.07094386219978333,
0.07373107224702835,
0.09166282415390015,
0.09760652482509613,
0.12053482979536057,
0.035728659480810165,
0.048727843910455704,
0.05559300258755684,
0.07109228521585464,
0.08069392293691635,
0.09690473228693008,
0.10087756812572479,
0.11853098124265671,
0.025910304859280586,
0.05259029567241669,
0.060516636818647385,
0.08889466524124146,
0.09090615063905716,
0.12455403059720993,
0.1273101419210434,
0.1728687882423401,
0.013562481850385666,
0.06489895284175873,
0.07113923132419586,
0.14679722487926483,
0.14901472628116608,
0.23576539754867554,
0.23766301572322845,
0.35208994150161743
]
},
{
"activation_grad_rms_by_block": [
4.705962055595592e-05,
4.8540012357989326e-05,
4.8112680815393105e-05,
4.5881148253101856e-05,
6.218662019819021e-05,
6.19510465185158e-05,
5.5873108067316934e-05,
5.11586076754611e-05,
5.377902198233642e-05,
4.923442247672938e-05,
4.591418110067025e-05,
4.340316809248179e-05,
3.853498856187798e-05,
3.637185727711767e-05,
3.548145105014555e-05,
3.440591899561696e-05,
7.517015910707414e-05,
7.005458610365167e-05,
6.350460171233863e-05,
5.674128988175653e-05,
0.00010789759107865393,
9.314908675150946e-05,
7.818391168257222e-05,
6.73785325489007e-05,
5.735654485761188e-05,
4.582547626341693e-05,
3.993835343862884e-05,
3.59240839316044e-05,
2.40640074480325e-05,
2.1051846488262527e-05,
1.9940822312491946e-05,
1.993459591176361e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 5.259535555524053e-05,
"first_to_last_ratio": 1.59358297228639,
"imbalance_abs_log_ratio": 0.46598492272985975,
"last_quartile_mean": 3.300446633147658e-05,
"mean": 5.106260448428657e-05,
"normalized": [
0.921606350307445,
0.9505980521014451,
0.9422292752458163,
0.8985273805847644,
1.2178505351666271,
1.2132371065714032,
1.0942079557361923,
1.0018801076079868,
1.0531977858451338,
0.9641972432463806,
0.8991742893725557,
0.8499991046449304,
0.7546616344988103,
0.712299296999438,
0.6948617566317875,
0.6737987484795189,
1.4721176067352022,
1.3719352314903852,
1.2436616258358075,
1.111210257581308,
2.1130451955668876,
1.8242133884920444,
1.531138344238427,
1.319527925169485,
1.1232592899812257,
0.8974371112918603,
0.7821448561426008,
0.7035301918972675,
0.4712647874324093,
0.41227521981846393,
0.3905171409466258,
0.3903952043397641
],
"population_cv": 0.3859968358313239
},
"activation_output_rms_by_block": [
0.16142085194587708,
0.2773628532886505,
0.3768865168094635,
0.40868353843688965,
0.14338424801826477,
0.16827327013015747,
0.18227216601371765,
0.19825012981891632,
0.06985136866569519,
0.1086883395910263,
0.14860449731349945,
0.21192558109760284,
0.08672910183668137,
0.13778631389141083,
0.18114051222801208,
0.23228809237480164,
0.062035780400037766,
0.0909910649061203,
0.12021689862012863,
0.1606491208076477,
0.061457011848688126,
0.092374786734581,
0.12395651638507843,
0.15162576735019684,
0.06645520776510239,
0.10725662857294083,
0.14660172164440155,
0.20186874270439148,
0.07922796159982681,
0.17203646898269653,
0.2743971645832062,
0.41855114698410034
],
"activation_output_statistics": {
"first_quartile_mean": 0.23956669680774212,
"first_to_last_ratio": 1.306969485354029,
"imbalance_abs_log_ratio": 0.26771108728004156,
"last_quartile_mean": 0.18329938035458326,
"mean": 0.1694765428546816,
"normalized": [
0.9524672218755847,
1.6365855039093904,
2.223827029163714,
2.4114460417529116,
0.846041851002414,
0.9929000633110867,
1.0755008506988941,
1.1697791710850909,
0.41215950885657104,
0.6413178942658843,
0.8768440446706599,
1.250471466598887,
0.5117469378110197,
0.8130111198312341,
1.068823503104685,
1.370620904003087,
0.36604346156170187,
0.5368947429151949,
0.7093424057110318,
0.9479136056332998,
0.36262842522923483,
0.5450594234376621,
0.7314081010689808,
0.8946711137494054,
0.392120388141766,
0.6328700524939813,
0.8650266235965518,
1.1911308745392848,
0.46748629789882584,
1.0151048993854561,
1.6190863936756676,
2.4696700790208403
],
"population_cv": 0.5508327874312803
},
"bits_per_byte": 1.9147359946947653,
"branch_output_rms_by_sublayer": [
0.08130212873220444,
0.13576549291610718,
0.1432749330997467,
0.041234951466321945,
0.12958019971847534,
0.023600688204169273,
0.04409155994653702,
0.0554339736700058,
0.14100632071495056,
0.01666521653532982,
0.0481393076479435,
0.008341255597770214,
0.04208913818001747,
0.01377889234572649,
0.04236427694559097,
0.03695881366729736,
0.05848926678299904,
0.033706482499837875,
0.04504085332155228,
0.0386466309428215,
0.04397343844175339,
0.04499480128288269,
0.05665678158402443,
0.057638294994831085,
0.06478945910930634,
0.04709118604660034,
0.04678606986999512,
0.044345736503601074,
0.04817425087094307,
0.04982399195432663,
0.049439266324043274,
0.060589101165533066,
0.04376237466931343,
0.040811967104673386,
0.02850259095430374,
0.04198655113577843,
0.02922668866813183,
0.05064959079027176,
0.04021619260311127,
0.07217346876859665,
0.04409762844443321,
0.039372652769088745,
0.030135832726955414,
0.04702961444854736,
0.03803896903991699,
0.054020222276449203,
0.030100898817181587,
0.06186771020293236,
0.03295852243900299,
0.054687224328517914,
0.0306381918489933,
0.05866995081305504,
0.02206242084503174,
0.06297128647565842,
0.02074088715016842,
0.07938925921916962,
0.021799447014927864,
0.07573848962783813,
0.026608876883983612,
0.1105475053191185,
0.01902451552450657,
0.12047550827264786,
0.02519184537231922,
0.16982334852218628
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.00023209204329478853,
0.00014705811075659293,
0.0001443553349415038,
0.0001409677954299832,
0.0001547598070491459,
0.00010679063100428474,
0.00010173131038502872,
0.00012885752112210694,
0.00018269174364617834,
0.00015061935227176197,
0.00015101151039961197,
0.0001756105532961918,
0.00014068815971577485,
0.0001326364304799064,
0.00015121016067746042,
0.00016411832907359437,
0.0002779928432077,
0.00028472923376674154,
0.0003007215693300081,
0.0003372879732191828,
0.0005019822813973469,
0.0004655562424498461,
0.00043909856222474334,
0.00038874448014086744,
0.0003187708239967095,
0.00026126643200496464,
0.0002213996139509037,
0.00021927053392099255,
0.00013957865743180746,
0.00013181726813679266,
0.00012444014246083002,
0.00012714374135915476
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.00014457656924792933,
"first_to_last_ratio": 0.74925318033778,
"imbalance_abs_log_ratio": 0.28867832809930477,
"last_quartile_mean": 0.0001929609016577694,
"mean": 0.00021703122476695333,
"normalized": [
1.069394708268395,
0.6775896459806783,
0.6651362498484334,
0.6495277146472977,
0.7130762276963877,
0.49205192072687143,
0.46874043352179906,
0.5937280281234798,
0.8417763104933472,
0.6939985360792947,
0.6958054563889003,
0.8091487917683791,
0.6482392562030871,
0.6111398515228881,
0.6967207637510633,
0.7561968525488604,
1.2808886993390323,
1.311927507539442,
1.3856143034391522,
1.554098833388423,
2.3129495856477433,
2.145111805684907,
2.0232045536131698,
1.7911914774397033,
1.468778625467388,
1.2038195530873992,
1.0201279294656385,
1.0103179123485286,
0.6431270780584042,
0.6073654529588411,
0.573374373178113,
0.5858315617749513
],
"population_cv": 0.5013169471957857
},
"depth_weights": [
{
"entropy_mean": 0.0,
"mean_weights": [
1.0
],
"sources": 1
},
{
"entropy_mean": 0.22879935801029205,
"mean_weights": [
0.9326952695846558,
0.06730473041534424
],
"sources": 2
},
{
"entropy_mean": 0.5610548257827759,
"mean_weights": [
0.6759846210479736,
0.3240153193473816
],
"sources": 2
},
{
"entropy_mean": 0.43457120656967163,
"mean_weights": [
0.8295697569847107,
0.1704302579164505
],
"sources": 2
},
{
"entropy_mean": 0.6696063280105591,
"mean_weights": [
0.545992374420166,
0.45400765538215637
],
"sources": 2
},
{
"entropy_mean": 0.6690812110900879,
"mean_weights": [
0.5202739238739014,
0.47972607612609863
],
"sources": 2
},
{
"entropy_mean": 0.6865538358688354,
"mean_weights": [
0.5361902713775635,
0.4638097584247589
],
"sources": 2
},
{
"entropy_mean": 0.4152516722679138,
"mean_weights": [
0.8452726602554321,
0.15472736954689026
],
"sources": 2
},
{
"entropy_mean": 0.6853918433189392,
"mean_weights": [
0.5220736265182495,
0.4779263138771057
],
"sources": 2
},
{
"entropy_mean": 1.0265861749649048,
"mean_weights": [
0.3228098750114441,
0.23137259483337402,
0.4458175599575043
],
"sources": 3
},
{
"entropy_mean": 1.066486120223999,
"mean_weights": [
0.2909930944442749,
0.26484501361846924,
0.44416186213493347
],
"sources": 3
},
{
"entropy_mean": 0.8915231227874756,
"mean_weights": [
0.21873247623443604,
0.16816270351409912,
0.6131048202514648
],
"sources": 3
},
{
"entropy_mean": 1.0651111602783203,
"mean_weights": [
0.3639928698539734,
0.22285741567611694,
0.41314971446990967
],
"sources": 3
},
{
"entropy_mean": 1.0005524158477783,
"mean_weights": [
0.3166183531284332,
0.18387392163276672,
0.49950775504112244
],
"sources": 3
},
{
"entropy_mean": 1.0648009777069092,
"mean_weights": [
0.35599708557128906,
0.22703218460083008,
0.41697072982788086
],
"sources": 3
},
{
"entropy_mean": 1.0006463527679443,
"mean_weights": [
0.28650256991386414,
0.19727063179016113,
0.5162267684936523
],
"sources": 3
},
{
"entropy_mean": 1.072684407234192,
"mean_weights": [
0.41332390904426575,
0.2456190586090088,
0.34105703234672546
],
"sources": 3
},
{
"entropy_mean": 1.304518699645996,
"mean_weights": [
0.2721603810787201,
0.12663133442401886,
0.2481587529182434,
0.35304954648017883
],
"sources": 4
},
{
"entropy_mean": 1.3646502494812012,
"mean_weights": [
0.29890745878219604,
0.18950369954109192,
0.2219487726688385,
0.2896400988101959
],
"sources": 4
},
{
"entropy_mean": 1.331923484802246,
"mean_weights": [
0.31035515666007996,
0.14580613374710083,
0.22454026341438293,
0.31929847598075867
],
"sources": 4
},
{
"entropy_mean": 1.3639233112335205,
"mean_weights": [
0.2952677309513092,
0.19548878073692322,
0.21280673146247864,
0.29643672704696655
],
"sources": 4
},
{
"entropy_mean": 1.2764989137649536,
"mean_weights": [
0.3656603693962097,
0.14041098952293396,
0.17602922022342682,
0.3178994059562683
],
"sources": 4
},
{
"entropy_mean": 1.364103078842163,
"mean_weights": [
0.27470695972442627,
0.18989428877830505,
0.23374854028224945,
0.30165016651153564
],
"sources": 4
},
{
"entropy_mean": 1.3429455757141113,
"mean_weights": [
0.2830207943916321,
0.17677831649780273,
0.23265977203845978,
0.3075411319732666
],
"sources": 4
},
{
"entropy_mean": 1.3732160329818726,
"mean_weights": [
0.2642192244529724,
0.22012627124786377,
0.22637531161308289,
0.28927919268608093
],
"sources": 4
},
{
"entropy_mean": 1.5599325895309448,
"mean_weights": [
0.18314200639724731,
0.13039523363113403,
0.18218961358070374,
0.26679670810699463,
0.2374764084815979
],
"sources": 5
},
{
"entropy_mean": 1.5772080421447754,
"mean_weights": [
0.19838765263557434,
0.18937477469444275,
0.15325719118118286,
0.16343195736408234,
0.2955484390258789
],
"sources": 5
},
{
"entropy_mean": 1.5662866830825806,
"mean_weights": [
0.22973918914794922,
0.12454323470592499,
0.16623356938362122,
0.21617932617664337,
0.2633046507835388
],
"sources": 5
},
{
"entropy_mean": 1.5421115159988403,
"mean_weights": [
0.2021060585975647,
0.1677371859550476,
0.11891372501850128,
0.17452098429203033,
0.3367220163345337
],
"sources": 5
},
{
"entropy_mean": 1.564535140991211,
"mean_weights": [
0.22419840097427368,
0.12987476587295532,
0.1717071682214737,
0.21590615808963776,
0.25831347703933716
],
"sources": 5
},
{
"entropy_mean": 1.5267750024795532,
"mean_weights": [
0.215010404586792,
0.19819971919059753,
0.10917140543460846,
0.13804513216018677,
0.33957335352897644
],
"sources": 5
},
{
"entropy_mean": 1.556459903717041,
"mean_weights": [
0.2037975788116455,
0.12750469148159027,
0.1812792420387268,
0.22193393111228943,
0.2654845714569092
],
"sources": 5
},
{
"entropy_mean": 1.5851528644561768,
"mean_weights": [
0.21840542554855347,
0.1923592984676361,
0.1442098617553711,
0.1790001392364502,
0.26602524518966675
],
"sources": 5
},
{
"entropy_mean": 1.7158764600753784,
"mean_weights": [
0.13390135765075684,
0.10310791432857513,
0.14190614223480225,
0.16341811418533325,
0.15331929922103882,
0.3043471872806549
],
"sources": 6
},
{
"entropy_mean": 1.6410574913024902,
"mean_weights": [
0.14665581285953522,
0.12231245636940002,
0.08100371062755585,
0.11105543375015259,
0.16015371680259705,
0.3788188695907593
],
"sources": 6
},
{
"entropy_mean": 1.6537573337554932,
"mean_weights": [
0.12217442691326141,
0.09171081334352493,
0.1348741054534912,
0.16064059734344482,
0.1268058568239212,
0.363794207572937
],
"sources": 6
},
{
"entropy_mean": 1.6571812629699707,
"mean_weights": [
0.15173450112342834,
0.14253422617912292,
0.09320533275604248,
0.10436245799064636,
0.14149092137813568,
0.3666725754737854
],
"sources": 6
},
{
"entropy_mean": 1.6046829223632812,
"mean_weights": [
0.12253664433956146,
0.08343374729156494,
0.1302124410867691,
0.14439499378204346,
0.11938183009624481,
0.4000403583049774
],
"sources": 6
},
{
"entropy_mean": 1.7041404247283936,
"mean_weights": [
0.1659325659275055,
0.15505465865135193,
0.10219550132751465,
0.11347830295562744,
0.1465306580066681,
0.3168083429336548
],
"sources": 6
},
{
"entropy_mean": 1.5808616876602173,
"mean_weights": [
0.12713032960891724,
0.08526259660720825,
0.1294248104095459,
0.1489948332309723,
0.10688205063343048,
0.40230536460876465
],
"sources": 6
},
{
"entropy_mean": 1.7305450439453125,
"mean_weights": [
0.17891491949558258,
0.17673785984516144,
0.09075747430324554,
0.11627361923456192,
0.17774534225463867,
0.25957077741622925
],
"sources": 6
},
{
"entropy_mean": 1.7513642311096191,
"mean_weights": [
0.09819091856479645,
0.06520214676856995,
0.091708242893219,
0.11103156208992004,
0.08388154953718185,
0.22660189867019653,
0.3233836889266968
],
"sources": 7
},
{
"entropy_mean": 1.766061782836914,
"mean_weights": [
0.12103402614593506,
0.1159343495965004,
0.06956179440021515,
0.07680170238018036,
0.10650800168514252,
0.1487376093864441,
0.36142250895500183
],
"sources": 7
},
{
"entropy_mean": 1.6586570739746094,
"mean_weights": [
0.09605097770690918,
0.06012085825204849,
0.08562932163476944,
0.10102558881044388,
0.07161714881658554,
0.17364460229873657,
0.4119114875793457
],
"sources": 7
},
{
"entropy_mean": 1.7821578979492188,
"mean_weights": [
0.13399386405944824,
0.12986589968204498,
0.0802837461233139,
0.07629825174808502,
0.10134232044219971,
0.12680892646312714,
0.351406991481781
],
"sources": 7
},
{
"entropy_mean": 1.655133843421936,
"mean_weights": [
0.10968073457479477,
0.0657510757446289,
0.08727138489484787,
0.09466101229190826,
0.0703636109828949,
0.14186590909957886,
0.4304063022136688
],
"sources": 7
},
{
"entropy_mean": 1.8160101175308228,
"mean_weights": [
0.13769462704658508,
0.11569900065660477,
0.08787183463573456,
0.0929793268442154,
0.10649852454662323,
0.1315194070339203,
0.3277372717857361
],
"sources": 7
},
{
"entropy_mean": 1.6999847888946533,
"mean_weights": [
0.10220035165548325,
0.07546016573905945,
0.1028895378112793,
0.10639216750860214,
0.07665040343999863,
0.13976991176605225,
0.3966374397277832
],
"sources": 7
},
{
"entropy_mean": 1.8706753253936768,
"mean_weights": [
0.14694823324680328,
0.12422746419906616,
0.09348422288894653,
0.1027352511882782,
0.12104703485965729,
0.14027650654315948,
0.27128130197525024
],
"sources": 7
},
{
"entropy_mean": 1.7911043167114258,
"mean_weights": [
0.09078209102153778,
0.05932023376226425,
0.06744872033596039,
0.0717800185084343,
0.05460817739367485,
0.10924961417913437,
0.271850049495697,
0.27496111392974854
],
"sources": 8
},
{
"entropy_mean": 1.9753689765930176,
"mean_weights": [
0.10981250554323196,
0.10515635460615158,
0.07847881317138672,
0.0806870236992836,
0.10027817636728287,
0.1099884957075119,
0.1757785975933075,
0.23982004821300507
],
"sources": 8
},
{
"entropy_mean": 1.81705904006958,
"mean_weights": [
0.0980612188577652,
0.060419049113988876,
0.06808654963970184,
0.07581339031457901,
0.06074263155460358,
0.09064903855323792,
0.23993313312530518,
0.3062949776649475
],
"sources": 8
},
{
"entropy_mean": 1.9814136028289795,
"mean_weights": [
0.11045937240123749,
0.08757394552230835,
0.08714327216148376,
0.09700852632522583,
0.09435248374938965,
0.114869624376297,
0.1777447760105133,
0.23084799945354462
],
"sources": 8
},
{
"entropy_mean": 1.8373757600784302,
"mean_weights": [
0.08649781346321106,
0.07379771769046783,
0.0769633948802948,
0.0880836620926857,
0.066677525639534,
0.1187550276517868,
0.23661984503269196,
0.25260502099990845
],
"sources": 8
},
{
"entropy_mean": 1.968585729598999,
"mean_weights": [
0.10324696451425552,
0.0860632136464119,
0.09797552973031998,
0.1271316111087799,
0.08284065127372742,
0.10640716552734375,
0.17243653535842896,
0.22389833629131317
],
"sources": 8
},
{
"entropy_mean": 1.8529754877090454,
"mean_weights": [
0.08385773003101349,
0.07188580930233002,
0.08174896240234375,
0.09558320790529251,
0.06307300925254822,
0.108722984790802,
0.2897399365901947,
0.20538833737373352
],
"sources": 8
},
{
"entropy_mean": 1.9637342691421509,
"mean_weights": [
0.10303820669651031,
0.08717727661132812,
0.09220535308122635,
0.11336663365364075,
0.08865514397621155,
0.14618384838104248,
0.2349606156349182,
0.13441291451454163
],
"sources": 8
},
{
"entropy_mean": 1.962258219718933,
"mean_weights": [
0.07841141521930695,
0.07437640428543091,
0.07400830090045929,
0.07925958186388016,
0.05560165271162987,
0.08937598764896393,
0.1956467628479004,
0.1118645966053009,
0.24145527184009552
],
"sources": 9
},
{
"entropy_mean": 2.0993216037750244,
"mean_weights": [
0.1043272316455841,
0.09323035180568695,
0.08778752386569977,
0.10311444103717804,
0.09694567322731018,
0.13869863748550415,
0.1680138111114502,
0.0874539315700531,
0.12042839825153351
],
"sources": 9
},
{
"entropy_mean": 2.0489909648895264,
"mean_weights": [
0.0919051468372345,
0.07878251373767853,
0.07574650645256042,
0.09311886876821518,
0.05773407965898514,
0.07325181365013123,
0.17863741517066956,
0.18801960349082947,
0.1628040373325348
],
"sources": 9
},
{
"entropy_mean": 2.126072406768799,
"mean_weights": [
0.09415344893932343,
0.07414553314447403,
0.10936354845762253,
0.11765885353088379,
0.07923045754432678,
0.12059831619262695,
0.17183929681777954,
0.12057285010814667,
0.11243770271539688
],
"sources": 9
},
{
"entropy_mean": 2.028747081756592,
"mean_weights": [
0.08796507865190506,
0.0757753849029541,
0.0847562700510025,
0.09340266138315201,
0.06450396031141281,
0.08635804057121277,
0.24226921796798706,
0.14922890067100525,
0.11574049293994904
],
"sources": 9
},
{
"entropy_mean": 2.1297550201416016,
"mean_weights": [
0.10085826367139816,
0.08009681105613708,
0.0982624888420105,
0.11403533071279526,
0.09301646053791046,
0.12771156430244446,
0.17309343814849854,
0.10872717201709747,
0.10419845581054688
],
"sources": 9
},
{
"entropy_mean": 2.1051812171936035,
"mean_weights": [
0.09203394502401352,
0.08722756803035736,
0.10295312106609344,
0.11847955733537674,
0.07401065528392792,
0.11382973194122314,
0.21416787803173065,
0.11479790508747101,
0.08249962329864502
],
"sources": 9
}
],
"layer_input_rms_by_sublayer": [
0.059127259999513626,
0.055979277938604355,
0.0761958509683609,
0.07538121938705444,
0.13674014806747437,
0.18601374328136444,
0.18700072169303894,
0.08912894874811172,
0.2049511820077896,
0.12973569333553314,
0.14231130480766296,
0.13784728944301605,
0.13109618425369263,
0.1306019276380539,
0.13412337005138397,
0.13534265756607056,
0.13132649660110474,
0.08038423955440521,
0.09715579450130463,
0.0852823480963707,
0.09982985258102417,
0.08248664438724518,
0.10234102606773376,
0.09743349254131317,
0.11231652647256851,
0.07765212655067444,
0.08827291429042816,
0.07436380535364151,
0.08991213887929916,
0.08191535621881485,
0.1065397635102272,
0.09093168377876282,
0.10877490043640137,
0.06605636328458786,
0.07123783230781555,
0.06527329236268997,
0.07851143926382065,
0.0699063166975975,
0.08676032721996307,
0.08022409677505493,
0.10016339272260666,
0.05980068817734718,
0.06517573446035385,
0.05788872763514519,
0.07041674107313156,
0.0643942728638649,
0.07159053534269333,
0.07148924469947815,
0.07631570100784302,
0.056988880038261414,
0.06180137023329735,
0.05852367728948593,
0.06370948255062103,
0.06811124831438065,
0.07006713002920151,
0.07283915579319,
0.0730312243103981,
0.057786133140325546,
0.06770572811365128,
0.07090020179748535,
0.07336394488811493,
0.07763758301734924,
0.08089767396450043,
0.07564623653888702
],
"loss_nats": 1.3271938562393188,
"loss_scale": 1.0,
"output_weights": {
"entropy_mean": 1.8321013450622559,
"mean_weights": [
0.09867185354232788,
0.05898427963256836,
0.057430200278759,
0.05067210644483566,
0.04944504797458649,
0.047245003283023834,
0.1162109300494194,
0.30178403854370117,
0.219556525349617
],
"sources": 9
},
"step": 4000,
"stream_state_rms_by_sublayer": [
0.08130212873220444,
0.16142085194587708,
0.2561342120170593,
0.2773628532886505,
0.3685380816459656,
0.3768865168094635,
0.38441672921180725,
0.40868353843688965,
0.14100632071495056,
0.14338424801826477,
0.16686734557151794,
0.16827327013015747,
0.18023288249969482,
0.18227216601371765,
0.18965557217597961,
0.19825012981891632,
0.05848926678299904,
0.06985136866569519,
0.09072534739971161,
0.1086883395910263,
0.12624414265155792,
0.14860449731349945,
0.17554737627506256,
0.21192558109760284,
0.06478945910930634,
0.08672910183668137,
0.11512516438961029,
0.13778631389141083,
0.15369470417499542,
0.18114051222801208,
0.19735252857208252,
0.23228809237480164,
0.04376237466931343,
0.062035780400037766,
0.07002050429582596,
0.0909910649061203,
0.09636416286230087,
0.12021689862012863,
0.12901709973812103,
0.1606491208076477,
0.04409762844443321,
0.061457011848688126,
0.07137759029865265,
0.092374786734581,
0.10297419875860214,
0.12395651638507843,
0.1298016905784607,
0.15162576735019684,
0.03295852243900299,
0.06645520776510239,
0.07529281079769135,
0.10725662857294083,
0.11068660020828247,
0.14660172164440155,
0.1495535671710968,
0.20186874270439148,
0.021799447014927864,
0.07922796159982681,
0.08539005368947983,
0.17203646898269653,
0.17447474598884583,
0.2743971645832062,
0.27695441246032715,
0.41855114698410034
]
},
{
"activation_grad_rms_by_block": [
6.761295662727207e-05,
7.037314207991585e-05,
6.971564289415255e-05,
6.589700205950066e-05,
0.00010337811545468867,
0.00010314521205145866,
9.018617129186168e-05,
8.218877337640151e-05,
8.93865289981477e-05,
8.14282102510333e-05,
7.567049033241346e-05,
7.11992834112607e-05,
6.774777284590527e-05,
6.376176315825433e-05,
6.184441008372232e-05,
5.9780148149002343e-05,
0.00014381110668182373,
0.00013238933752290905,
0.0001167725640698336,
0.00010191192995989695,
0.0002003924601012841,
0.0001659920671954751,
0.0001361987233394757,
0.00011443908442743123,
0.00010709096386563033,
8.16169849713333e-05,
6.831773498561233e-05,
5.791887815576047e-05,
4.921185245621018e-05,
3.8056739867897704e-05,
3.282592660980299e-05,
3.210229624528438e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 8.156212697940646e-05,
"first_to_last_ratio": 1.3967870279562358,
"imbalance_abs_log_ratio": 0.33417461908716395,
"last_quartile_mean": 5.839267214469146e-05,
"mean": 8.757388354752038e-05,
"normalized": [
0.7720675832605176,
0.803585946279625,
0.7960780094481321,
0.752473219070419,
1.180467409539882,
1.1778079020041263,
1.029829529518624,
0.9385078067458692,
1.0206984705622164,
0.9298229900567075,
0.8640759923745102,
0.8130195958778703,
0.7736070401530517,
0.7280910766467854,
0.7061969571118035,
0.6826252956632183,
1.642168884788434,
1.5117445082936212,
1.3334176736203447,
1.1637251407789435,
2.2882673690329693,
1.8954517085610634,
1.5552436162725376,
1.306771834154562,
1.2228641636923565,
0.9319785952743014,
0.7801153976292595,
0.661371584878177,
0.561946672486053,
0.4345672285647456,
0.37483694087850516,
0.36657385678076787
],
"population_cv": 0.42684708327825066
},
"activation_output_rms_by_block": [
0.16424456238746643,
0.29588937759399414,
0.42337876558303833,
0.45154979825019836,
0.13891471922397614,
0.1555756777524948,
0.16914354264736176,
0.1874145120382309,
0.06211879849433899,
0.09842939674854279,
0.137006014585495,
0.20118924975395203,
0.08399304747581482,
0.12138351052999496,
0.1587214469909668,
0.2071460783481598,
0.0581100657582283,
0.08358953893184662,
0.10947012156248093,
0.1472771316766739,
0.05584336072206497,
0.08318652212619781,
0.11238116025924683,
0.1379220336675644,
0.05995604768395424,
0.09392217546701431,
0.1259489357471466,
0.17086055874824524,
0.06700707972049713,
0.13971327245235443,
0.2211977243423462,
0.32473331689834595
],
"activation_output_statistics": {
"first_quartile_mean": 0.2482638694345951,
"first_to_last_ratio": 1.6504997944655762,
"imbalance_abs_log_ratio": 0.5010781477825214,
"last_quartile_mean": 0.150417388882488,
"mean": 0.1577255482552573,
"normalized": [
1.0413313772202522,
1.8759762186094133,
2.6842751238870792,
2.86288304745295,
0.8807369558111258,
0.9863695480754758,
1.0723915340184837,
1.1882317995492147,
0.3938410687519575,
0.6240548714989927,
0.8686355260833801,
1.2755653854399962,
0.532526584341831,
0.7695868670150526,
1.006314124418815,
1.3133324349770026,
0.3684251942759779,
0.5299682889456079,
0.6940544684956076,
0.9337557124120812,
0.3540539965769534,
0.5274131112327584,
0.7125108234042923,
0.874443203356988,
0.3801289540418877,
0.5954785163594047,
0.7985322424957776,
1.083277634082024,
0.4248333923179985,
0.8857998846594436,
1.4024216545081702,
2.0588504556840044
],
"population_cv": 0.6067899489376947
},
"bits_per_byte": 1.8565786854442996,
"branch_output_rms_by_sublayer": [
0.08626964688301086,
0.13547764718532562,
0.16337208449840546,
0.039667773991823196,
0.15584443509578705,
0.026703152805566788,
0.04305505007505417,
0.05304057523608208,
0.1360139101743698,
0.016798952594399452,
0.03393339365720749,
0.00753376679494977,
0.03730965405702591,
0.01294450368732214,
0.037494905292987823,
0.03526800498366356,
0.05234000086784363,
0.030294200405478477,
0.04223497211933136,
0.034149833023548126,
0.041619230061769485,
0.04367009550333023,
0.05722156912088394,
0.05930330231785774,
0.062187470495700836,
0.04527241364121437,
0.0369485504925251,
0.04281846433877945,
0.04293901473283768,
0.04812675341963768,
0.045369699597358704,
0.06094835698604584,
0.04149825870990753,
0.038031257688999176,
0.02440357394516468,
0.040106989443302155,
0.026009807363152504,
0.04961257055401802,
0.03634416684508324,
0.07085621356964111,
0.038355328142642975,
0.03753058984875679,
0.02631118707358837,
0.04544757679104805,
0.033921729773283005,
0.05368982255458832,
0.02731483057141304,
0.06174425780773163,
0.027557097375392914,
0.05051252990961075,
0.024315863847732544,
0.054937634617090225,
0.02051021158695221,
0.05777662619948387,
0.018631720915436745,
0.07133159041404724,
0.019497457891702652,
0.0636468306183815,
0.021715763956308365,
0.09261580556631088,
0.017114246264100075,
0.1020733118057251,
0.020576251670718193,
0.12878532707691193
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.0003637111411518744,
0.0002328437345071155,
0.00026754486121072013,
0.00020961054010746675,
0.00023324217449373482,
0.00013226952769200274,
0.0001490970084679736,
0.0001891597343986376,
0.0002918746014559555,
0.00023307226647352922,
0.00024024223572494834,
0.00029360747277183056,
0.00025626387819923106,
0.00024114885933974834,
0.0002716298155375107,
0.0003086967264992385,
0.0005417594428612147,
0.0005350465304351142,
0.0005646380221663597,
0.0006164043223392351,
0.0009244274084097353,
0.0008484364345428017,
0.0007892136159088089,
0.0006791712867767833,
0.0005903268079560613,
0.00047055631494833115,
0.0003773406829883918,
0.0003589161190616393,
0.00026391753197443733,
0.00022775345188039026,
0.00019334088037647755,
0.0001778215115139631
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.00022218484025369068,
"first_to_last_ratio": 0.6682317907333766,
"imbalance_abs_log_ratio": 0.4031201735174996,
"last_quartile_mean": 0.00033249666258746146,
"mean": 0.00037728390444285195,
"normalized": [
0.9640250667172751,
0.6171578796900011,
0.7091340448403511,
0.5555777430182339,
0.6182139543911144,
0.3505835423520907,
0.3951851820664077,
0.5013723940277184,
0.7736206024664017,
0.6177636091253748,
0.6367677838780912,
0.7782136192780627,
0.6792335299266601,
0.6391708114234587,
0.7199613138509996,
0.81820804668331,
1.4359463430099129,
1.4181536082893218,
1.496586563903859,
1.633794378929312,
2.4502169255666204,
2.2488010343184843,
2.0918295390158974,
1.8001597173347186,
1.5646753042057733,
1.2472207518187604,
1.000150492891086,
0.9513157461399342,
0.6995197220622846,
0.6036659640085139,
0.5124546213069721,
0.471320163462998
],
"population_cv": 0.5587385684613161
},
"depth_weights": [
{
"entropy_mean": 0.0,
"mean_weights": [
1.0
],
"sources": 1
},
{
"entropy_mean": 0.21598131954669952,
"mean_weights": [
0.9387255907058716,
0.06127442046999931
],
"sources": 2
},
{
"entropy_mean": 0.5377621054649353,
"mean_weights": [
0.7082071900367737,
0.2917928099632263
],
"sources": 2
},
{
"entropy_mean": 0.43774232268333435,
"mean_weights": [
0.829714834690094,
0.1702851504087448
],
"sources": 2
},
{
"entropy_mean": 0.6749181151390076,
"mean_weights": [
0.5407299399375916,
0.45927006006240845
],
"sources": 2
},
{
"entropy_mean": 0.6663151383399963,
"mean_weights": [
0.5011438131332397,
0.49885618686676025
],
"sources": 2
},
{
"entropy_mean": 0.6865031719207764,
"mean_weights": [
0.5308656692504883,
0.4691343307495117
],
"sources": 2
},
{
"entropy_mean": 0.4133813679218292,
"mean_weights": [
0.8479139804840088,
0.15208598971366882
],
"sources": 2
},
{
"entropy_mean": 0.6850745677947998,
"mean_weights": [
0.5398513674736023,
0.4601486027240753
],
"sources": 2
},
{
"entropy_mean": 1.0074119567871094,
"mean_weights": [
0.32744842767715454,
0.20981493592262268,
0.4627366065979004
],
"sources": 3
},
{
"entropy_mean": 1.0489801168441772,
"mean_weights": [
0.28095993399620056,
0.24264614284038544,
0.4763939380645752
],
"sources": 3
},
{
"entropy_mean": 0.8527575731277466,
"mean_weights": [
0.20522087812423706,
0.1493145227432251,
0.6454645991325378
],
"sources": 3
},
{
"entropy_mean": 1.049942135810852,
"mean_weights": [
0.35788866877555847,
0.20253834128379822,
0.4395729899406433
],
"sources": 3
},
{
"entropy_mean": 0.9791408181190491,
"mean_weights": [
0.31465935707092285,
0.1638563871383667,
0.5214842557907104
],
"sources": 3
},
{
"entropy_mean": 1.0485806465148926,
"mean_weights": [
0.346283882856369,
0.20800183713436127,
0.4457142949104309
],
"sources": 3
},
{
"entropy_mean": 0.9801348447799683,
"mean_weights": [
0.2748759686946869,
0.1817527413368225,
0.5433712601661682
],
"sources": 3
},
{
"entropy_mean": 1.0688585042953491,
"mean_weights": [
0.4051227569580078,
0.23231396079063416,
0.36256328225135803
],
"sources": 3
},
{
"entropy_mean": 1.2915732860565186,
"mean_weights": [
0.26621368527412415,
0.1108347550034523,
0.2563389539718628,
0.36661261320114136
],
"sources": 4
},
{
"entropy_mean": 1.3568191528320312,
"mean_weights": [
0.294538676738739,
0.1710844486951828,
0.2252884805202484,
0.30908840894699097
],
"sources": 4
},
{
"entropy_mean": 1.320021152496338,
"mean_weights": [
0.304654598236084,
0.12911874055862427,
0.23493197560310364,
0.3312946557998657
],
"sources": 4
},
{
"entropy_mean": 1.3578683137893677,
"mean_weights": [
0.29558902978897095,
0.17812995612621307,
0.21904321014881134,
0.30723777413368225
],
"sources": 4
},
{
"entropy_mean": 1.258887767791748,
"mean_weights": [
0.35744374990463257,
0.12067858874797821,
0.17883820831775665,
0.34303945302963257
],
"sources": 4
},
{
"entropy_mean": 1.3539760112762451,
"mean_weights": [
0.2736307978630066,
0.1722734123468399,
0.2365075945854187,
0.3175881803035736
],
"sources": 4
},
{
"entropy_mean": 1.3203014135360718,
"mean_weights": [
0.28207293152809143,
0.15420743823051453,
0.22419029474258423,
0.3395293354988098
],
"sources": 4
},
{
"entropy_mean": 1.3734662532806396,
"mean_weights": [
0.2766023576259613,
0.21176086366176605,
0.22839435935020447,
0.283242404460907
],
"sources": 4
},
{
"entropy_mean": 1.5325813293457031,
"mean_weights": [
0.17451074719429016,
0.10643941164016724,
0.1691833734512329,
0.27394938468933105,
0.27591708302497864
],
"sources": 5
},
{
"entropy_mean": 1.560360074043274,
"mean_weights": [
0.19377154111862183,
0.17211592197418213,
0.14605137705802917,
0.16254930198192596,
0.3255118727684021
],
"sources": 5
},
{
"entropy_mean": 1.540909767150879,
"mean_weights": [
0.21921303868293762,
0.10205383598804474,
0.16381604969501495,
0.215368390083313,
0.2995486855506897
],
"sources": 5
},
{
"entropy_mean": 1.5142419338226318,
"mean_weights": [
0.2000826597213745,
0.15480534732341766,
0.10827909409999847,
0.16766759753227234,
0.3691653311252594
],
"sources": 5
},
{
"entropy_mean": 1.5333527326583862,
"mean_weights": [
0.21770668029785156,
0.10525082051753998,
0.15970060229301453,
0.217601016163826,
0.29974091053009033
],
"sources": 5
},
{
"entropy_mean": 1.512608289718628,
"mean_weights": [
0.21392285823822021,
0.1860550343990326,
0.10291537642478943,
0.1385909616947174,
0.35851576924324036
],
"sources": 5
},
{
"entropy_mean": 1.5262645483016968,
"mean_weights": [
0.20246261358261108,
0.10530520975589752,
0.16650918126106262,
0.21146778762340546,
0.3142552077770233
],
"sources": 5
},
{
"entropy_mean": 1.5794764757156372,
"mean_weights": [
0.2210189402103424,
0.18839940428733826,
0.14016607403755188,
0.17440536618232727,
0.2760102152824402
],
"sources": 5
},
{
"entropy_mean": 1.6602650880813599,
"mean_weights": [
0.12695768475532532,
0.08036886155605316,
0.12672865390777588,
0.15391480922698975,
0.1544383019208908,
0.3575916886329651
],
"sources": 6
},
{
"entropy_mean": 1.5925545692443848,
"mean_weights": [
0.14075040817260742,
0.11028306186199188,
0.07537610828876495,
0.10244421660900116,
0.1520608514547348,
0.41908538341522217
],
"sources": 6
},
{
"entropy_mean": 1.5918848514556885,
"mean_weights": [
0.11773697286844254,
0.07344591617584229,
0.11976177245378494,
0.14735516905784607,
0.1265447735786438,
0.4151553511619568
],
"sources": 6
},
{
"entropy_mean": 1.6208878755569458,
"mean_weights": [
0.14785674214363098,
0.12492769956588745,
0.08781817555427551,
0.10019385814666748,
0.14005999267101288,
0.3991435170173645
],
"sources": 6
},
{
"entropy_mean": 1.5470075607299805,
"mean_weights": [
0.12043031305074692,
0.06855648756027222,
0.11753211170434952,
0.13396470248699188,
0.1162077933549881,
0.44330859184265137
],
"sources": 6
},
{
"entropy_mean": 1.683492660522461,
"mean_weights": [
0.1658952832221985,
0.1435280442237854,
0.09461991488933563,
0.10784181207418442,
0.15157341957092285,
0.3365415334701538
],
"sources": 6
},
{
"entropy_mean": 1.5540567636489868,
"mean_weights": [
0.12987136840820312,
0.07178180664777756,
0.12132491171360016,
0.1441602110862732,
0.11020088195800781,
0.42266082763671875
],
"sources": 6
},
{
"entropy_mean": 1.7208337783813477,
"mean_weights": [
0.18048501014709473,
0.1685236394405365,
0.08726631104946136,
0.11160479485988617,
0.18141084909439087,
0.27070939540863037
],
"sources": 6
},
{
"entropy_mean": 1.6729483604431152,
"mean_weights": [
0.09229261428117752,
0.051420144736766815,
0.08079522103071213,
0.09550285339355469,
0.07590493559837341,
0.22372649610042572,
0.3803577423095703
],
"sources": 7
},
{
"entropy_mean": 1.710235595703125,
"mean_weights": [
0.11657393723726273,
0.10617929697036743,
0.061112720519304276,
0.06796726584434509,
0.10333786904811859,
0.14365722239017487,
0.4011717140674591
],
"sources": 7
},
{
"entropy_mean": 1.5640671253204346,
"mean_weights": [
0.0904226303100586,
0.04561381787061691,
0.07565715163946152,
0.08878682553768158,
0.06555216014385223,
0.16504976153373718,
0.4689176678657532
],
"sources": 7
},
{
"entropy_mean": 1.7556228637695312,
"mean_weights": [
0.13281463086605072,
0.12175846099853516,
0.07235272228717804,
0.0707377940416336,
0.10551950335502625,
0.12620802223682404,
0.3706088662147522
],
"sources": 7
},
{
"entropy_mean": 1.594892978668213,
"mean_weights": [
0.10721130669116974,
0.053925663232803345,
0.07673212885856628,
0.08865121006965637,
0.06888366490602493,
0.14094024896621704,
0.4636557698249817
],
"sources": 7
},
{
"entropy_mean": 1.7775511741638184,
"mean_weights": [
0.13637228310108185,
0.10478455573320389,
0.07710614800453186,
0.08548048138618469,
0.11000139266252518,
0.12931162118911743,
0.3569435477256775
],
"sources": 7
},
{
"entropy_mean": 1.654036045074463,
"mean_weights": [
0.10232305526733398,
0.061968062072992325,
0.08900615572929382,
0.1005433052778244,
0.07864195853471756,
0.14099234342575073,
0.4265251159667969
],
"sources": 7
},
{
"entropy_mean": 1.84956693649292,
"mean_weights": [
0.14851760864257812,
0.11630287766456604,
0.08512166142463684,
0.09215595573186874,
0.1263951063156128,
0.14375528693199158,
0.28775155544281006
],
"sources": 7
},
{
"entropy_mean": 1.7161309719085693,
"mean_weights": [
0.0837959572672844,
0.04434927552938461,
0.0567089319229126,
0.06495873630046844,
0.04885464161634445,
0.09997115284204483,
0.2760006785392761,
0.32536062598228455
],
"sources": 8
},
{
"entropy_mean": 1.9285151958465576,
"mean_weights": [
0.10564712435007095,
0.09173665940761566,
0.07135222852230072,
0.07045470178127289,
0.09785081446170807,
0.10425601899623871,
0.1782122403383255,
0.2804902195930481
],
"sources": 8
},
{
"entropy_mean": 1.7219383716583252,
"mean_weights": [
0.0923302173614502,
0.04646994173526764,
0.05750556290149689,
0.06549686193466187,
0.054404959082603455,
0.08427833020687103,
0.2333412766456604,
0.3661728501319885
],
"sources": 8
},
{
"entropy_mean": 1.9470824003219604,
"mean_weights": [
0.10804469138383865,
0.07850410044193268,
0.07298290729522705,
0.08511960506439209,
0.09968686103820801,
0.11502514779567719,
0.18013443052768707,
0.26050227880477905
],
"sources": 8
},
{
"entropy_mean": 1.7855088710784912,
"mean_weights": [
0.08493785560131073,
0.05972691997885704,
0.07017223536968231,
0.07963316142559052,
0.0645163506269455,
0.11586237698793411,
0.23710276186466217,
0.28804832696914673
],
"sources": 8
},
{
"entropy_mean": 1.973182201385498,
"mean_weights": [
0.1049819216132164,
0.0840599462389946,
0.09310521930456161,
0.11566175520420074,
0.0963074192404747,
0.1096896231174469,
0.16087493300437927,
0.23531918227672577
],
"sources": 8
},
{
"entropy_mean": 1.802741289138794,
"mean_weights": [
0.08190082013607025,
0.05632811039686203,
0.07102417945861816,
0.08421063423156738,
0.06341388821601868,
0.10201875865459442,
0.2913346588611603,
0.24976897239685059
],
"sources": 8
},
{
"entropy_mean": 1.945815086364746,
"mean_weights": [
0.1064736619591713,
0.07925166189670563,
0.07823627442121506,
0.09447507560253143,
0.09724453091621399,
0.1459730863571167,
0.2451319396495819,
0.153213769197464
],
"sources": 8
},
{
"entropy_mean": 1.8717057704925537,
"mean_weights": [
0.07336360216140747,
0.05720400810241699,
0.057025715708732605,
0.06655597686767578,
0.05032068490982056,
0.08329878002405167,
0.1901058554649353,
0.11727118492126465,
0.3048541843891144
],
"sources": 9
},
{
"entropy_mean": 2.09096622467041,
"mean_weights": [
0.10326676070690155,
0.08232800662517548,
0.07164021581411362,
0.08822821080684662,
0.10803373903036118,
0.13578416407108307,
0.17320571839809418,
0.09955950826406479,
0.13795366883277893
],
"sources": 9
},
{
"entropy_mean": 1.96881103515625,
"mean_weights": [
0.08843819797039032,
0.06052382290363312,
0.0642135739326477,
0.07744920998811722,
0.05161681026220322,
0.06631232053041458,
0.16417551040649414,
0.2115624099969864,
0.21570813655853271
],
"sources": 9
},
{
"entropy_mean": 2.124063014984131,
"mean_weights": [
0.09220977127552032,
0.06839151680469513,
0.10039852559566498,
0.10902463644742966,
0.0849665105342865,
0.12137697637081146,
0.17017139494419098,
0.12761826813220978,
0.1258423924446106
],
"sources": 9
},
{
"entropy_mean": 1.9633058309555054,
"mean_weights": [
0.0846441313624382,
0.05872901901602745,
0.07210009545087814,
0.07792744785547256,
0.05918677896261215,
0.07445863634347916,
0.23511825501918793,
0.18020221590995789,
0.15763342380523682
],
"sources": 9
},
{
"entropy_mean": 2.1238560676574707,
"mean_weights": [
0.10231245309114456,
0.07575537264347076,
0.08897507935762405,
0.0967075377702713,
0.09801244735717773,
0.12438583374023438,
0.1766587197780609,
0.12203006446361542,
0.11516247689723969
],
"sources": 9
},
{
"entropy_mean": 2.0835556983947754,
"mean_weights": [
0.09133295714855194,
0.07054752856492996,
0.09000501036643982,
0.10956106334924698,
0.07114207744598389,
0.11565886437892914,
0.22511571645736694,
0.13003571331501007,
0.09660108387470245
],
"sources": 9
}
],
"layer_input_rms_by_sublayer": [
0.062413111329078674,
0.05931045487523079,
0.07552354782819748,
0.07934419065713882,
0.14653873443603516,
0.214645653963089,
0.2105521410703659,
0.09514089673757553,
0.2186809629201889,
0.12821291387081146,
0.14272794127464294,
0.1321839988231659,
0.1287408471107483,
0.12568901479244232,
0.13253600895404816,
0.1335759311914444,
0.13371841609477997,
0.07788944244384766,
0.09585829079151154,
0.08352623134851456,
0.09883332997560501,
0.07934392243623734,
0.10020580142736435,
0.09343641996383667,
0.11237040907144547,
0.07063088566064835,
0.08538161963224411,
0.06870647519826889,
0.08663177490234375,
0.07622843235731125,
0.10358677804470062,
0.08666495233774185,
0.10874549299478531,
0.057347409427165985,
0.06467291712760925,
0.05777324363589287,
0.07278503477573395,
0.06351501494646072,
0.08346924930810928,
0.07469023764133453,
0.09857872873544693,
0.051319729536771774,
0.06093711405992508,
0.05102822557091713,
0.06834813207387924,
0.05902284011244774,
0.0680762529373169,
0.06656526774168015,
0.07369214296340942,
0.050236333161592484,
0.05633483827114105,
0.05184732377529144,
0.05850831791758537,
0.061248794198036194,
0.06479237228631973,
0.06608261168003082,
0.06705968081951141,
0.048912134021520615,
0.061747051775455475,
0.062179163098335266,
0.0654391348361969,
0.07131517678499222,
0.07205834239721298,
0.06802479177713394
],
"loss_nats": 1.2868822813034058,
"loss_scale": 1.0,
"output_weights": {
"entropy_mean": 1.679484486579895,
"mean_weights": [
0.08993907272815704,
0.043493133038282394,
0.04922422021627426,
0.037924446165561676,
0.03939434140920639,
0.03898489475250244,
0.09112586826086044,
0.32717907428741455,
0.2827349305152893
],
"sources": 9
},
"step": 8000,
"stream_state_rms_by_sublayer": [
0.08626964688301086,
0.16424456238746643,
0.27676042914390564,
0.29588937759399414,
0.41335058212280273,
0.42337876558303833,
0.43042075634002686,
0.45154979825019836,
0.1360139101743698,
0.13891471922397614,
0.15445689857006073,
0.1555756777524948,
0.1670466512441635,
0.16914354264736176,
0.17834730446338654,
0.1874145120382309,
0.05234000086784363,
0.06211879849433899,
0.08269349485635757,
0.09842939674854279,
0.11548293381929398,
0.137006014585495,
0.16410532593727112,
0.20118924975395203,
0.062187470495700836,
0.08399304747581482,
0.0988159254193306,
0.12138351052999496,
0.13199056684970856,
0.1587214469909668,
0.17199495434761047,
0.2071460783481598,
0.04149825870990753,
0.0581100657582283,
0.06434677541255951,
0.08358953893184662,
0.08728200942277908,
0.10947012156248093,
0.11647984385490417,
0.1472771316766739,
0.038355328142642975,
0.05584336072206497,
0.06353354454040527,
0.08318652212619781,
0.09196189045906067,
0.11238116025924683,
0.1169828325510025,
0.1379220336675644,
0.027557097375392914,
0.05995604768395424,
0.06579430401325226,
0.09392217546701431,
0.09665275365114212,
0.1259489357471466,
0.12799666821956635,
0.17086055874824524,
0.019497457891702652,
0.06700707972049713,
0.07176875323057175,
0.13971327245235443,
0.14131556451320648,
0.2211977243423462,
0.22250843048095703,
0.32473331689834595
]
}
],
"environment": {
"autocast": "cuda-bfloat16-forward-fp32-cross-entropy",
"compile": false,
"compute_capability": [
12,
0
],
"cublas_workspace_config": ":4096:8",
"cuda": "12.8",
"deterministic_algorithms": true,
"gpu": "NVIDIA GeForce RTX 5090",
"python": "3.10.14",
"torch": "2.11.0+cu128"
},
"evaluations": [
{
"bits_per_byte": 8.061194200882085,
"cross_entropy_nats": 5.587594032287598,
"step": 0
},
{
"bits_per_byte": 5.191908918496252,
"cross_entropy_nats": 3.598757028579712,
"step": 100
},
{
"bits_per_byte": 3.0881952483500545,
"cross_entropy_nats": 2.1405738294124603,
"step": 500
},
{
"bits_per_byte": 1.948972795980338,
"cross_entropy_nats": 1.3509249985218048,
"step": 2000
},
{
"bits_per_byte": 1.7872753095784035,
"cross_entropy_nats": 1.2388448417186737,
"step": 4000
},
{
"bits_per_byte": 1.7030966812870931,
"cross_entropy_nats": 1.1804966628551483,
"step": 8000
}
],
"gradient_gate": null,
"hashes": {
"final_mixer_parameters": "94814c85fe1adce99ef9f8a23b6b40ed1a12e214f38d376a95be44bd8a14d14a",
"final_model_state": "638568aede21890773b6932a19ec4e112f5ac0a4770ba3402fcd82980a9ecf76",
"final_optimizer_state": "83947fd743ec8e3e31ca7788fd201981846f0f1c7e9e38c88afcef95cfc6ec4e",
"final_public_parameters": "73e15de1ee917017dc95607378bdbf84b7b54f65be48f3497fdb5654f3f0479e",
"initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d",
"initial_public_parameter_elements": 18985152,
"initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670",
"initial_public_parameter_tensors": 227,
"initial_public_parameters": "3f93c11463780b386fff00c53b0424f687e1136b1b80f7b626f058adef96c764"
},
"manifest": {
"diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716",
"file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371",
"formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
"input_gate_tensor_hashes": {
"0": "3eff5c07fa0cfbd5ceed5922cbd09525ad514f801cc0a440c03fd2fa15df4c84",
"1": "e83e4001c5811b19c69a54c7ea7a802af996a34aa57072cde43a76bc8369e2d0",
"7999": "d04c4ca879f0209183a48ac54a0c4ffe3a0f62a78888479d4641f6c304426442"
},
"path": "experiments/k3/attnres_gradient/manifest.json",
"validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338"
},
"model": {
"attnres_aggregation_groups": 8,
"context": 256,
"d_ff": 768,
"d_head": 32,
"d_model": 192,
"heads": 6,
"layers": 32,
"parameters": {
"core": 18985152,
"embedding": 98304,
"mixer": 24960,
"total": 19010112
},
"sublayers": 64,
"sublayers_per_attnres_group": 8,
"transformer_blocks_per_attnres_group": 4,
"vocabulary": 256
},
"optimizer": {
"betas": [
0.9,
0.95
],
"epsilon": 1e-08,
"grad_clip": 1.0,
"min_lr": 3e-05,
"name": "AdamW",
"peak_lr": 0.0003,
"warmup_steps": 400,
"weight_decay_ndim_ge_2": 0.1
},
"protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
"run_kind": "formal",
"schema_version": 1,
"seed": 2026073003,
"steps": 8000,
"target_bytes_seen": 65536000,
"timing": {
"mean_ms": 109.96596761305909,
"measured_steps": 7980,
"median_ms": 109.86474549281411,
"p95_ms": 110.51533734425902,
"peak_allocated_bytes": 12815843328,
"peak_reserved_bytes": 13337886720,
"warmup_steps_excluded": 20
},
"training_history": [
{
"bits_per_byte": 8.082576459898554,
"learning_rate": 7.499999999999999e-07,
"loss_nats": 5.602415084838867,
"step": 1,
"unclipped_grad_norm": 19.083253860473633
},
{
"bits_per_byte": 7.337116283872504,
"learning_rate": 7.499999999999999e-06,
"loss_nats": 5.0857014656066895,
"step": 10,
"unclipped_grad_norm": 11.73559856414795
},
{
"bits_per_byte": 6.794594947933964,
"learning_rate": 1.4999999999999999e-05,
"loss_nats": 4.709654331207275,
"step": 20,
"unclipped_grad_norm": 3.3705036640167236
},
{
"bits_per_byte": 6.669727976170927,
"learning_rate": 2.2499999999999998e-05,
"loss_nats": 4.623103141784668,
"step": 30,
"unclipped_grad_norm": 3.037728786468506
},
{
"bits_per_byte": 6.488515360401613,
"learning_rate": 2.9999999999999997e-05,
"loss_nats": 4.497496128082275,
"step": 40,
"unclipped_grad_norm": 3.071707010269165
},
{
"bits_per_byte": 6.170372909737745,
"learning_rate": 3.75e-05,
"loss_nats": 4.276976585388184,
"step": 50,
"unclipped_grad_norm": 2.8251144886016846
},
{
"bits_per_byte": 5.9458358004242795,
"learning_rate": 4.4999999999999996e-05,
"loss_nats": 4.121339321136475,
"step": 60,
"unclipped_grad_norm": 2.6051926612854004
},
{
"bits_per_byte": 5.763860269615738,
"learning_rate": 5.2499999999999995e-05,
"loss_nats": 3.9952034950256348,
"step": 70,
"unclipped_grad_norm": 2.513765573501587
},
{
"bits_per_byte": 5.595572150058764,
"learning_rate": 5.9999999999999995e-05,
"loss_nats": 3.8785550594329834,
"step": 80,
"unclipped_grad_norm": 2.595860719680786
},
{
"bits_per_byte": 5.4051244099034195,
"learning_rate": 6.749999999999999e-05,
"loss_nats": 3.746546745300293,
"step": 90,
"unclipped_grad_norm": 2.317351818084717
},
{
"bits_per_byte": 5.1779332643218385,
"learning_rate": 7.5e-05,
"loss_nats": 3.5890698432922363,
"step": 100,
"unclipped_grad_norm": 2.214414358139038
},
{
"bits_per_byte": 4.995566300999934,
"learning_rate": 8.249999999999999e-05,
"loss_nats": 3.462662696838379,
"step": 110,
"unclipped_grad_norm": 2.0333311557769775
},
{
"bits_per_byte": 4.799066491360726,
"learning_rate": 8.999999999999999e-05,
"loss_nats": 3.3264594078063965,
"step": 120,
"unclipped_grad_norm": 1.8019887208938599
},
{
"bits_per_byte": 4.633962458601826,
"learning_rate": 9.75e-05,
"loss_nats": 3.2120180130004883,
"step": 130,
"unclipped_grad_norm": 1.6018544435501099
},
{
"bits_per_byte": 4.476944362155892,
"learning_rate": 0.00010499999999999999,
"loss_nats": 3.1031813621520996,
"step": 140,
"unclipped_grad_norm": 1.3790134191513062
},
{
"bits_per_byte": 4.275779695094682,
"learning_rate": 0.0001125,
"loss_nats": 2.963744640350342,
"step": 150,
"unclipped_grad_norm": 1.2061023712158203
},
{
"bits_per_byte": 4.128819144071206,
"learning_rate": 0.00011999999999999999,
"loss_nats": 2.861879348754883,
"step": 160,
"unclipped_grad_norm": 1.0463906526565552
},
{
"bits_per_byte": 4.021535678532382,
"learning_rate": 0.00012749999999999998,
"loss_nats": 2.7875161170959473,
"step": 170,
"unclipped_grad_norm": 0.9023508429527283
},
{
"bits_per_byte": 3.919193618518149,
"learning_rate": 0.00013499999999999997,
"loss_nats": 2.7165780067443848,
"step": 180,
"unclipped_grad_norm": 0.9033810496330261
},
{
"bits_per_byte": 3.8348498868820102,
"learning_rate": 0.0001425,
"loss_nats": 2.6581153869628906,
"step": 190,
"unclipped_grad_norm": 0.8499347567558289
},
{
"bits_per_byte": 3.7665400977858114,
"learning_rate": 0.00015,
"loss_nats": 2.610766649246216,
"step": 200,
"unclipped_grad_norm": 0.8223105072975159
},
{
"bits_per_byte": 3.6822345462981643,
"learning_rate": 0.0001575,
"loss_nats": 2.552330493927002,
"step": 210,
"unclipped_grad_norm": 0.8248683214187622
},
{
"bits_per_byte": 3.6564082555833983,
"learning_rate": 0.00016499999999999997,
"loss_nats": 2.5344290733337402,
"step": 220,
"unclipped_grad_norm": 1.0457000732421875
},
{
"bits_per_byte": 3.6121584953776043,
"learning_rate": 0.0001725,
"loss_nats": 2.5037574768066406,
"step": 230,
"unclipped_grad_norm": 1.2628414630889893
},
{
"bits_per_byte": 3.5855720413460244,
"learning_rate": 0.00017999999999999998,
"loss_nats": 2.4853291511535645,
"step": 240,
"unclipped_grad_norm": 0.9464337229728699
},
{
"bits_per_byte": 3.5024318443006237,
"learning_rate": 0.0001875,
"loss_nats": 2.4277007579803467,
"step": 250,
"unclipped_grad_norm": 0.8258991837501526
},
{
"bits_per_byte": 3.520142273721174,
"learning_rate": 0.000195,
"loss_nats": 2.439976692199707,
"step": 260,
"unclipped_grad_norm": 1.1166419982910156
},
{
"bits_per_byte": 3.497664141253405,
"learning_rate": 0.00020249999999999996,
"loss_nats": 2.42439603805542,
"step": 270,
"unclipped_grad_norm": 0.9347699880599976
},
{
"bits_per_byte": 3.454170760746106,
"learning_rate": 0.00020999999999999998,
"loss_nats": 2.3942487239837646,
"step": 280,
"unclipped_grad_norm": 1.2552868127822876
},
{
"bits_per_byte": 3.4220065653817433,
"learning_rate": 0.00021749999999999997,
"loss_nats": 2.3719542026519775,
"step": 290,
"unclipped_grad_norm": 0.8032089471817017
},
{
"bits_per_byte": 3.435558454304431,
"learning_rate": 0.000225,
"loss_nats": 2.38134765625,
"step": 300,
"unclipped_grad_norm": 1.544488787651062
},
{
"bits_per_byte": 3.330305071976392,
"learning_rate": 0.00023249999999999996,
"loss_nats": 2.308391571044922,
"step": 310,
"unclipped_grad_norm": 0.8757423162460327
},
{
"bits_per_byte": 3.390824390953808,
"learning_rate": 0.00023999999999999998,
"loss_nats": 2.3503403663635254,
"step": 320,
"unclipped_grad_norm": 0.8823365569114685
},
{
"bits_per_byte": 3.3979087003081485,
"learning_rate": 0.0002475,
"loss_nats": 2.355250835418701,
"step": 330,
"unclipped_grad_norm": 0.898759663105011
},
{
"bits_per_byte": 3.325636774901372,
"learning_rate": 0.00025499999999999996,
"loss_nats": 2.3051557540893555,
"step": 340,
"unclipped_grad_norm": 0.9536095261573792
},
{
"bits_per_byte": 3.3170438337336625,
"learning_rate": 0.0002625,
"loss_nats": 2.2991995811462402,
"step": 350,
"unclipped_grad_norm": 1.2654696702957153
},
{
"bits_per_byte": 3.313786482326321,
"learning_rate": 0.00026999999999999995,
"loss_nats": 2.2969417572021484,
"step": 360,
"unclipped_grad_norm": 1.3773932456970215
},
{
"bits_per_byte": 3.3236555347634327,
"learning_rate": 0.00027749999999999997,
"loss_nats": 2.3037824630737305,
"step": 370,
"unclipped_grad_norm": 1.2408338785171509
},
{
"bits_per_byte": 3.208911117864879,
"learning_rate": 0.000285,
"loss_nats": 2.224247694015503,
"step": 380,
"unclipped_grad_norm": 1.0960922241210938
},
{
"bits_per_byte": 3.2804789462990227,
"learning_rate": 0.0002925,
"loss_nats": 2.2738547325134277,
"step": 390,
"unclipped_grad_norm": 0.9866538643836975
},
{
"bits_per_byte": 3.27038631641579,
"learning_rate": 0.0003,
"loss_nats": 2.2668590545654297,
"step": 400,
"unclipped_grad_norm": 1.2531954050064087
},
{
"bits_per_byte": 3.2889521875417316,
"learning_rate": 0.0002999988466112388,
"loss_nats": 2.2797279357910156,
"step": 410,
"unclipped_grad_norm": 1.448500633239746
},
{
"bits_per_byte": 3.205672684549132,
"learning_rate": 0.00029999538646466355,
"loss_nats": 2.2220029830932617,
"step": 420,
"unclipped_grad_norm": 0.9654834866523743
},
{
"bits_per_byte": 3.218386673996816,
"learning_rate": 0.0002999896196193985,
"loss_nats": 2.2308156490325928,
"step": 430,
"unclipped_grad_norm": 0.974204421043396
},
{
"bits_per_byte": 3.1822768526563587,
"learning_rate": 0.00029998154617398326,
"loss_nats": 2.2057862281799316,
"step": 440,
"unclipped_grad_norm": 1.4095590114593506
},
{
"bits_per_byte": 3.237966210866943,
"learning_rate": 0.0002999711662663707,
"loss_nats": 2.244387149810791,
"step": 450,
"unclipped_grad_norm": 0.9198341369628906
},
{
"bits_per_byte": 3.1857660366770637,
"learning_rate": 0.0002999584800739247,
"loss_nats": 2.208204746246338,
"step": 460,
"unclipped_grad_norm": 1.4268778562545776
},
{
"bits_per_byte": 3.1576792059993566,
"learning_rate": 0.0002999434878134173,
"loss_nats": 2.1887364387512207,
"step": 470,
"unclipped_grad_norm": 1.2876354455947876
},
{
"bits_per_byte": 3.131057323541699,
"learning_rate": 0.000299926189741025,
"loss_nats": 2.170283555984497,
"step": 480,
"unclipped_grad_norm": 1.0718865394592285
},
{
"bits_per_byte": 3.1147361699748184,
"learning_rate": 0.000299906586152324,
"loss_nats": 2.158970594406128,
"step": 490,
"unclipped_grad_norm": 0.8680974841117859
},
{
"bits_per_byte": 3.059730958749366,
"learning_rate": 0.0002998846773822856,
"loss_nats": 2.1208438873291016,
"step": 500,
"unclipped_grad_norm": 1.296195387840271
},
{
"bits_per_byte": 3.12068642572937,
"learning_rate": 0.0002998604638052702,
"loss_nats": 2.163094997406006,
"step": 510,
"unclipped_grad_norm": 1.0469720363616943
},
{
"bits_per_byte": 3.1397613655018657,
"learning_rate": 0.0002998339458350211,
"loss_nats": 2.176316738128662,
"step": 520,
"unclipped_grad_norm": 1.2089059352874756
},
{
"bits_per_byte": 2.9844827015867583,
"learning_rate": 0.0002998051239246572,
"loss_nats": 2.06868577003479,
"step": 530,
"unclipped_grad_norm": 1.1353814601898193
},
{
"bits_per_byte": 3.0015299659055814,
"learning_rate": 0.00029977399856666537,
"loss_nats": 2.0805020332336426,
"step": 540,
"unclipped_grad_norm": 0.919367253780365
},
{
"bits_per_byte": 2.987599371185694,
"learning_rate": 0.0002997405702928922,
"loss_nats": 2.0708460807800293,
"step": 550,
"unclipped_grad_norm": 0.8667266368865967
},
{
"bits_per_byte": 2.995183462123442,
"learning_rate": 0.0002997048396745345,
"loss_nats": 2.0761029720306396,
"step": 560,
"unclipped_grad_norm": 0.9804732203483582
},
{
"bits_per_byte": 2.962294187903042,
"learning_rate": 0.0002996668073221302,
"loss_nats": 2.0533058643341064,
"step": 570,
"unclipped_grad_norm": 0.9414609670639038
},
{
"bits_per_byte": 2.956716790535561,
"learning_rate": 0.000299626473885547,
"loss_nats": 2.0494399070739746,
"step": 580,
"unclipped_grad_norm": 0.8727462291717529
},
{
"bits_per_byte": 2.9113505189603663,
"learning_rate": 0.00029958384005397226,
"loss_nats": 2.0179944038391113,
"step": 590,
"unclipped_grad_norm": 1.1642946004867554
},
{
"bits_per_byte": 2.8665478345320485,
"learning_rate": 0.0002995389065559004,
"loss_nats": 1.986939549446106,
"step": 600,
"unclipped_grad_norm": 0.9213871359825134
},
{
"bits_per_byte": 2.875505550902238,
"learning_rate": 0.000299491674159121,
"loss_nats": 1.9931485652923584,
"step": 610,
"unclipped_grad_norm": 0.7122263312339783
},
{
"bits_per_byte": 2.878308696128651,
"learning_rate": 0.00029944214367070535,
"loss_nats": 1.9950915575027466,
"step": 620,
"unclipped_grad_norm": 1.4622896909713745
},
{
"bits_per_byte": 2.790202156074285,
"learning_rate": 0.00029939031593699284,
"loss_nats": 1.934020757675171,
"step": 630,
"unclipped_grad_norm": 1.1472660303115845
},
{
"bits_per_byte": 2.7890029210498186,
"learning_rate": 0.00029933619184357654,
"loss_nats": 1.9331895112991333,
"step": 640,
"unclipped_grad_norm": 1.1472764015197754
},
{
"bits_per_byte": 2.7734051265124666,
"learning_rate": 0.00029927977231528774,
"loss_nats": 1.9223779439926147,
"step": 650,
"unclipped_grad_norm": 0.9182148575782776
},
{
"bits_per_byte": 2.795219749913207,
"learning_rate": 0.00029922105831618057,
"loss_nats": 1.937498688697815,
"step": 660,
"unclipped_grad_norm": 1.083983302116394
},
{
"bits_per_byte": 2.7702638633944576,
"learning_rate": 0.0002991600508495153,
"loss_nats": 1.9202005863189697,
"step": 670,
"unclipped_grad_norm": 1.1721765995025635
},
{
"bits_per_byte": 2.755497089026075,
"learning_rate": 0.00029909675095774114,
"loss_nats": 1.9099650382995605,
"step": 680,
"unclipped_grad_norm": 1.280714750289917
},
{
"bits_per_byte": 2.7128121989604823,
"learning_rate": 0.0002990311597224787,
"loss_nats": 1.8803781270980835,
"step": 690,
"unclipped_grad_norm": 1.0380759239196777
},
{
"bits_per_byte": 2.7091302223881604,
"learning_rate": 0.00029896327826450103,
"loss_nats": 1.8778259754180908,
"step": 700,
"unclipped_grad_norm": 1.1401211023330688
},
{
"bits_per_byte": 2.6870259802033667,
"learning_rate": 0.0002988931077437152,
"loss_nats": 1.862504482269287,
"step": 710,
"unclipped_grad_norm": 1.6450653076171875
},
{
"bits_per_byte": 2.688485424978589,
"learning_rate": 0.00029882064935914164,
"loss_nats": 1.863516092300415,
"step": 720,
"unclipped_grad_norm": 1.0721391439437866
},
{
"bits_per_byte": 2.65192277331841,
"learning_rate": 0.0002987459043488943,
"loss_nats": 1.8381727933883667,
"step": 730,
"unclipped_grad_norm": 1.0645538568496704
},
{
"bits_per_byte": 2.6143492076364594,
"learning_rate": 0.0002986688739901591,
"loss_nats": 1.8121287822723389,
"step": 740,
"unclipped_grad_norm": 1.1877458095550537
},
{
"bits_per_byte": 2.6277774410331767,
"learning_rate": 0.0002985895595991723,
"loss_nats": 1.8214365243911743,
"step": 750,
"unclipped_grad_norm": 1.1091431379318237
},
{
"bits_per_byte": 2.540277655771055,
"learning_rate": 0.00029850796253119824,
"loss_nats": 1.7607862949371338,
"step": 760,
"unclipped_grad_norm": 1.222191333770752
},
{
"bits_per_byte": 2.5790566478529025,
"learning_rate": 0.00029842408418050544,
"loss_nats": 1.787665843963623,
"step": 770,
"unclipped_grad_norm": 1.254854679107666
},
{
"bits_per_byte": 2.5138998166949715,
"learning_rate": 0.0002983379259803436,
"loss_nats": 1.7425025701522827,
"step": 780,
"unclipped_grad_norm": 0.9573470950126648
},
{
"bits_per_byte": 2.5258618979930834,
"learning_rate": 0.0002982494894029186,
"loss_nats": 1.7507940530776978,
"step": 790,
"unclipped_grad_norm": 1.108724594116211
},
{
"bits_per_byte": 2.430956195819507,
"learning_rate": 0.0002981587759593675,
"loss_nats": 1.6850104331970215,
"step": 800,
"unclipped_grad_norm": 0.9832104444503784
},
{
"bits_per_byte": 2.468076415233783,
"learning_rate": 0.00029806578719973283,
"loss_nats": 1.7107402086257935,
"step": 810,
"unclipped_grad_norm": 0.9492170214653015
},
{
"bits_per_byte": 2.4602348662676934,
"learning_rate": 0.00029797052471293596,
"loss_nats": 1.7053048610687256,
"step": 820,
"unclipped_grad_norm": 1.192505121231079
},
{
"bits_per_byte": 2.4583616312344954,
"learning_rate": 0.0002978729901267499,
"loss_nats": 1.7040064334869385,
"step": 830,
"unclipped_grad_norm": 0.9113606214523315
},
{
"bits_per_byte": 2.4751598646748696,
"learning_rate": 0.0002977731851077716,
"loss_nats": 1.7156500816345215,
"step": 840,
"unclipped_grad_norm": 1.3671480417251587
},
{
"bits_per_byte": 2.386899400147981,
"learning_rate": 0.0002976711113613935,
"loss_nats": 1.6544725894927979,
"step": 850,
"unclipped_grad_norm": 1.0324512720108032
},
{
"bits_per_byte": 2.472668523994471,
"learning_rate": 0.0002975667706317743,
"loss_nats": 1.7139232158660889,
"step": 860,
"unclipped_grad_norm": 1.503003478050232
},
{
"bits_per_byte": 2.4348980381772827,
"learning_rate": 0.00029746016470180915,
"loss_nats": 1.6877427101135254,
"step": 870,
"unclipped_grad_norm": 1.1144746541976929
},
{
"bits_per_byte": 2.3709108610278693,
"learning_rate": 0.00029735129539309926,
"loss_nats": 1.64339017868042,
"step": 880,
"unclipped_grad_norm": 1.0134795904159546
},
{
"bits_per_byte": 2.3377405671559597,
"learning_rate": 0.0002972401645659207,
"loss_nats": 1.6203982830047607,
"step": 890,
"unclipped_grad_norm": 1.0625330209732056
},
{
"bits_per_byte": 2.389408283058768,
"learning_rate": 0.00029712677411919283,
"loss_nats": 1.6562116146087646,
"step": 900,
"unclipped_grad_norm": 0.9719302654266357
},
{
"bits_per_byte": 2.3782030974071033,
"learning_rate": 0.0002970111259904454,
"loss_nats": 1.6484447717666626,
"step": 910,
"unclipped_grad_norm": 1.0672563314437866
},
{
"bits_per_byte": 2.3530695528100125,
"learning_rate": 0.0002968932221557859,
"loss_nats": 1.6310235261917114,
"step": 920,
"unclipped_grad_norm": 1.1197935342788696
},
{
"bits_per_byte": 2.297453115243394,
"learning_rate": 0.0002967730646298655,
"loss_nats": 1.5924731492996216,
"step": 930,
"unclipped_grad_norm": 1.0027272701263428
},
{
"bits_per_byte": 2.344061961471073,
"learning_rate": 0.0002966506554658449,
"loss_nats": 1.6247799396514893,
"step": 940,
"unclipped_grad_norm": 1.0651428699493408
},
{
"bits_per_byte": 2.367935217202641,
"learning_rate": 0.00029652599675535876,
"loss_nats": 1.6413276195526123,
"step": 950,
"unclipped_grad_norm": 0.9822106957435608
},
{
"bits_per_byte": 2.309206925551311,
"learning_rate": 0.0002963990906284808,
"loss_nats": 1.6006202697753906,
"step": 960,
"unclipped_grad_norm": 0.8860561847686768
},
{
"bits_per_byte": 2.2419830349126597,
"learning_rate": 0.0002962699392536863,
"loss_nats": 1.5540242195129395,
"step": 970,
"unclipped_grad_norm": 1.1816905736923218
},
{
"bits_per_byte": 2.3851068249780383,
"learning_rate": 0.0002961385448378161,
"loss_nats": 1.65323007106781,
"step": 980,
"unclipped_grad_norm": 0.9257189035415649
},
{
"bits_per_byte": 2.232579711494085,
"learning_rate": 0.00029600490962603817,
"loss_nats": 1.547506332397461,
"step": 990,
"unclipped_grad_norm": 0.9634203910827637
},
{
"bits_per_byte": 2.2247954327507324,
"learning_rate": 0.00029586903590180956,
"loss_nats": 1.5421106815338135,
"step": 1000,
"unclipped_grad_norm": 0.809658944606781
},
{
"bits_per_byte": 2.2354428786656486,
"learning_rate": 0.00029573092598683735,
"loss_nats": 1.5494909286499023,
"step": 1010,
"unclipped_grad_norm": 1.3530596494674683
},
{
"bits_per_byte": 2.2067197121797495,
"learning_rate": 0.00029559058224103895,
"loss_nats": 1.5295815467834473,
"step": 1020,
"unclipped_grad_norm": 1.0303069353103638
},
{
"bits_per_byte": 2.1954008459958647,
"learning_rate": 0.0002954480070625017,
"loss_nats": 1.5217359066009521,
"step": 1030,
"unclipped_grad_norm": 0.9071640968322754
},
{
"bits_per_byte": 2.291673982226451,
"learning_rate": 0.00029530320288744205,
"loss_nats": 1.5884673595428467,
"step": 1040,
"unclipped_grad_norm": 1.3239798545837402
},
{
"bits_per_byte": 2.196653911590051,
"learning_rate": 0.0002951561721901638,
"loss_nats": 1.5226044654846191,
"step": 1050,
"unclipped_grad_norm": 0.9085557460784912
},
{
"bits_per_byte": 2.225028985190604,
"learning_rate": 0.00029500691748301594,
"loss_nats": 1.5422725677490234,
"step": 1060,
"unclipped_grad_norm": 0.9488101005554199
},
{
"bits_per_byte": 2.275690258620563,
"learning_rate": 0.00029485544131634963,
"loss_nats": 1.5773882865905762,
"step": 1070,
"unclipped_grad_norm": 1.047879695892334
},
{
"bits_per_byte": 2.235951603346901,
"learning_rate": 0.0002947017462784745,
"loss_nats": 1.5498435497283936,
"step": 1080,
"unclipped_grad_norm": 0.8919407725334167
},
{
"bits_per_byte": 2.1733449309359636,
"learning_rate": 0.00029454583499561476,
"loss_nats": 1.5064479112625122,
"step": 1090,
"unclipped_grad_norm": 1.0036922693252563
},
{
"bits_per_byte": 2.169043472855234,
"learning_rate": 0.00029438771013186413,
"loss_nats": 1.5034663677215576,
"step": 1100,
"unclipped_grad_norm": 0.9904995560646057
},
{
"bits_per_byte": 2.124962083664635,
"learning_rate": 0.0002942273743891401,
"loss_nats": 1.4729114770889282,
"step": 1110,
"unclipped_grad_norm": 0.9176430702209473
},
{
"bits_per_byte": 2.176794214981668,
"learning_rate": 0.00029406483050713824,
"loss_nats": 1.5088387727737427,
"step": 1120,
"unclipped_grad_norm": 0.9229135513305664
},
{
"bits_per_byte": 2.210271497885368,
"learning_rate": 0.00029390008126328504,
"loss_nats": 1.53204345703125,
"step": 1130,
"unclipped_grad_norm": 0.871993899345398
},
{
"bits_per_byte": 2.123456375556331,
"learning_rate": 0.00029373312947269045,
"loss_nats": 1.4718677997589111,
"step": 1140,
"unclipped_grad_norm": 0.8673478960990906
},
{
"bits_per_byte": 2.1911007637763418,
"learning_rate": 0.00029356397798809995,
"loss_nats": 1.518755316734314,
"step": 1150,
"unclipped_grad_norm": 0.8959671258926392
},
{
"bits_per_byte": 2.0953000638876755,
"learning_rate": 0.0002933926296998457,
"loss_nats": 1.4523513317108154,
"step": 1160,
"unclipped_grad_norm": 0.9113589525222778
},
{
"bits_per_byte": 2.1570418355474237,
"learning_rate": 0.0002932190875357972,
"loss_nats": 1.495147466659546,
"step": 1170,
"unclipped_grad_norm": 0.8980047702789307
},
{
"bits_per_byte": 2.1555593450969863,
"learning_rate": 0.00029304335446131126,
"loss_nats": 1.4941198825836182,
"step": 1180,
"unclipped_grad_norm": 0.948503315448761
},
{
"bits_per_byte": 2.1334348089593913,
"learning_rate": 0.00029286543347918116,
"loss_nats": 1.4787843227386475,
"step": 1190,
"unclipped_grad_norm": 1.1094794273376465
},
{
"bits_per_byte": 2.1317098229712372,
"learning_rate": 0.0002926853276295856,
"loss_nats": 1.4775886535644531,
"step": 1200,
"unclipped_grad_norm": 0.8304017782211304
},
{
"bits_per_byte": 2.1385674591917407,
"learning_rate": 0.0002925030399900368,
"loss_nats": 1.482342004776001,
"step": 1210,
"unclipped_grad_norm": 0.8620252013206482
},
{
"bits_per_byte": 2.0980762078379027,
"learning_rate": 0.00029231857367532726,
"loss_nats": 1.4542756080627441,
"step": 1220,
"unclipped_grad_norm": 0.88591068983078
},
{
"bits_per_byte": 2.108927281211441,
"learning_rate": 0.0002921319318374776,
"loss_nats": 1.4617969989776611,
"step": 1230,
"unclipped_grad_norm": 1.0479243993759155
},
{
"bits_per_byte": 2.081558994049039,
"learning_rate": 0.00029194311766568174,
"loss_nats": 1.442826747894287,
"step": 1240,
"unclipped_grad_norm": 0.9365729689598083
},
{
"bits_per_byte": 2.0172010223043655,
"learning_rate": 0.000291752134386253,
"loss_nats": 1.3982172012329102,
"step": 1250,
"unclipped_grad_norm": 0.8966301083564758
},
{
"bits_per_byte": 2.0690183631134245,
"learning_rate": 0.00029155898526256856,
"loss_nats": 1.4341342449188232,
"step": 1260,
"unclipped_grad_norm": 0.8974369168281555
},
{
"bits_per_byte": 2.101840220134686,
"learning_rate": 0.0002913636735950141,
"loss_nats": 1.4568846225738525,
"step": 1270,
"unclipped_grad_norm": 0.9220007061958313
},
{
"bits_per_byte": 2.1256708241688407,
"learning_rate": 0.0002911662027209272,
"loss_nats": 1.473402738571167,
"step": 1280,
"unclipped_grad_norm": 0.9313441514968872
},
{
"bits_per_byte": 2.076867651298803,
"learning_rate": 0.00029096657601454013,
"loss_nats": 1.439574956893921,
"step": 1290,
"unclipped_grad_norm": 0.8451681137084961
},
{
"bits_per_byte": 2.032476865319302,
"learning_rate": 0.00029076479688692266,
"loss_nats": 1.4088056087493896,
"step": 1300,
"unclipped_grad_norm": 0.9314314126968384
},
{
"bits_per_byte": 2.058903719450881,
"learning_rate": 0.00029056086878592323,
"loss_nats": 1.4271233081817627,
"step": 1310,
"unclipped_grad_norm": 0.8796194195747375
},
{
"bits_per_byte": 2.0275063947267156,
"learning_rate": 0.00029035479519611045,
"loss_nats": 1.4053603410720825,
"step": 1320,
"unclipped_grad_norm": 0.9140937924385071
},
{
"bits_per_byte": 2.037489299678698,
"learning_rate": 0.0002901465796387134,
"loss_nats": 1.4122799634933472,
"step": 1330,
"unclipped_grad_norm": 0.821657657623291
},
{
"bits_per_byte": 2.0199990080512524,
"learning_rate": 0.00028993622567156154,
"loss_nats": 1.4001566171646118,
"step": 1340,
"unclipped_grad_norm": 0.8321346044540405
},
{
"bits_per_byte": 2.048903616233812,
"learning_rate": 0.0002897237368890237,
"loss_nats": 1.420191764831543,
"step": 1350,
"unclipped_grad_norm": 0.9421875476837158
},
{
"bits_per_byte": 2.00944598061166,
"learning_rate": 0.0002895091169219469,
"loss_nats": 1.3928418159484863,
"step": 1360,
"unclipped_grad_norm": 0.8682821393013
},
{
"bits_per_byte": 2.0435301902902543,
"learning_rate": 0.00028929236943759425,
"loss_nats": 1.4164671897888184,
"step": 1370,
"unclipped_grad_norm": 0.8776147961616516
},
{
"bits_per_byte": 2.0135448431296727,
"learning_rate": 0.00028907349813958225,
"loss_nats": 1.39568293094635,
"step": 1380,
"unclipped_grad_norm": 0.87440025806427
},
{
"bits_per_byte": 2.0533221944997795,
"learning_rate": 0.0002888525067678175,
"loss_nats": 1.4232544898986816,
"step": 1390,
"unclipped_grad_norm": 0.7801422476768494
},
{
"bits_per_byte": 2.1564656936670334,
"learning_rate": 0.00028862939909843273,
"loss_nats": 1.4947481155395508,
"step": 1400,
"unclipped_grad_norm": 0.8561526536941528
},
{
"bits_per_byte": 2.0373921294809607,
"learning_rate": 0.0002884041789437225,
"loss_nats": 1.412212610244751,
"step": 1410,
"unclipped_grad_norm": 0.8376067876815796
},
{
"bits_per_byte": 2.0325960492963495,
"learning_rate": 0.0002881768501520778,
"loss_nats": 1.4088882207870483,
"step": 1420,
"unclipped_grad_norm": 0.751315712928772
},
{
"bits_per_byte": 1.9860154082234123,
"learning_rate": 0.0002879474166079205,
"loss_nats": 1.376600980758667,
"step": 1430,
"unclipped_grad_norm": 0.9057046175003052
},
{
"bits_per_byte": 2.015529006972677,
"learning_rate": 0.00028771588223163694,
"loss_nats": 1.3970582485198975,
"step": 1440,
"unclipped_grad_norm": 0.8342951536178589
},
{
"bits_per_byte": 1.9818307263626185,
"learning_rate": 0.00028748225097951054,
"loss_nats": 1.3737003803253174,
"step": 1450,
"unclipped_grad_norm": 0.8142686486244202
},
{
"bits_per_byte": 2.1060192265680016,
"learning_rate": 0.000287246526843655,
"loss_nats": 1.459781289100647,
"step": 1460,
"unclipped_grad_norm": 0.8357279896736145
},
{
"bits_per_byte": 2.079546797033944,
"learning_rate": 0.00028700871385194533,
"loss_nats": 1.441431999206543,
"step": 1470,
"unclipped_grad_norm": 0.7720605731010437
},
{
"bits_per_byte": 2.025708316111945,
"learning_rate": 0.00028676881606794945,
"loss_nats": 1.404114007949829,
"step": 1480,
"unclipped_grad_norm": 0.8328851461410522
},
{
"bits_per_byte": 2.0422327531721454,
"learning_rate": 0.00028652683759085854,
"loss_nats": 1.4155678749084473,
"step": 1490,
"unclipped_grad_norm": 0.9698525071144104
},
{
"bits_per_byte": 1.9217496191796013,
"learning_rate": 0.0002862827825554171,
"loss_nats": 1.3320553302764893,
"step": 1500,
"unclipped_grad_norm": 0.7396577000617981
},
{
"bits_per_byte": 1.9436662282923034,
"learning_rate": 0.00028603665513185245,
"loss_nats": 1.347246766090393,
"step": 1510,
"unclipped_grad_norm": 0.809975266456604
},
{
"bits_per_byte": 1.9993110429963143,
"learning_rate": 0.000285788459525803,
"loss_nats": 1.3858168125152588,
"step": 1520,
"unclipped_grad_norm": 0.8165730834007263
},
{
"bits_per_byte": 2.03089187320895,
"learning_rate": 0.00028553819997824707,
"loss_nats": 1.4077069759368896,
"step": 1530,
"unclipped_grad_norm": 0.7620540261268616
},
{
"bits_per_byte": 2.002577509484152,
"learning_rate": 0.00028528588076542964,
"loss_nats": 1.3880809545516968,
"step": 1540,
"unclipped_grad_norm": 0.8928612470626831
},
{
"bits_per_byte": 2.0629018001354886,
"learning_rate": 0.00028503150619878996,
"loss_nats": 1.4298945665359497,
"step": 1550,
"unclipped_grad_norm": 0.7963082194328308
},
{
"bits_per_byte": 1.9429003895480113,
"learning_rate": 0.0002847750806248876,
"loss_nats": 1.3467159271240234,
"step": 1560,
"unclipped_grad_norm": 0.7710158228874207
},
{
"bits_per_byte": 1.9502758655302628,
"learning_rate": 0.000284516608425328,
"loss_nats": 1.3518282175064087,
"step": 1570,
"unclipped_grad_norm": 0.8057935833930969
},
{
"bits_per_byte": 1.9693647358974786,
"learning_rate": 0.000284256094016688,
"loss_nats": 1.3650596141815186,
"step": 1580,
"unclipped_grad_norm": 0.8588574528694153
},
{
"bits_per_byte": 1.972771368245769,
"learning_rate": 0.0002839935418504399,
"loss_nats": 1.3674209117889404,
"step": 1590,
"unclipped_grad_norm": 0.8608299493789673
},
{
"bits_per_byte": 1.9610373359427014,
"learning_rate": 0.000283728956412876,
"loss_nats": 1.3592875003814697,
"step": 1600,
"unclipped_grad_norm": 0.8587453365325928
},
{
"bits_per_byte": 1.9257736692444893,
"learning_rate": 0.00028346234222503136,
"loss_nats": 1.3348445892333984,
"step": 1610,
"unclipped_grad_norm": 0.8028180003166199
},
{
"bits_per_byte": 1.929578785394829,
"learning_rate": 0.00028319370384260675,
"loss_nats": 1.3374820947647095,
"step": 1620,
"unclipped_grad_norm": 0.7991352677345276
},
{
"bits_per_byte": 1.9716548568763677,
"learning_rate": 0.00028292304585589105,
"loss_nats": 1.3666470050811768,
"step": 1630,
"unclipped_grad_norm": 0.7573540806770325
},
{
"bits_per_byte": 1.9236465878186182,
"learning_rate": 0.0002826503728896824,
"loss_nats": 1.3333702087402344,
"step": 1640,
"unclipped_grad_norm": 0.8524133563041687
},
{
"bits_per_byte": 1.9335809936630575,
"learning_rate": 0.0002823756896032095,
"loss_nats": 1.3402562141418457,
"step": 1650,
"unclipped_grad_norm": 0.7908568978309631
},
{
"bits_per_byte": 1.9524230689262851,
"learning_rate": 0.0002820990006900518,
"loss_nats": 1.3533165454864502,
"step": 1660,
"unclipped_grad_norm": 0.8385109901428223
},
{
"bits_per_byte": 1.8864807930018634,
"learning_rate": 0.00028182031087805947,
"loss_nats": 1.3076088428497314,
"step": 1670,
"unclipped_grad_norm": 0.8104477524757385
},
{
"bits_per_byte": 1.918672677573015,
"learning_rate": 0.0002815396249292725,
"loss_nats": 1.3299225568771362,
"step": 1680,
"unclipped_grad_norm": 0.7970330119132996
},
{
"bits_per_byte": 1.9517956762159374,
"learning_rate": 0.0002812569476398391,
"loss_nats": 1.352881669998169,
"step": 1690,
"unclipped_grad_norm": 0.7619603276252747
},
{
"bits_per_byte": 1.8990867773448057,
"learning_rate": 0.00028097228383993446,
"loss_nats": 1.3163466453552246,
"step": 1700,
"unclipped_grad_norm": 0.8769286274909973
},
{
"bits_per_byte": 1.938959579086141,
"learning_rate": 0.0002806856383936774,
"loss_nats": 1.3439843654632568,
"step": 1710,
"unclipped_grad_norm": 0.7246651649475098
},
{
"bits_per_byte": 1.847509524316351,
"learning_rate": 0.00028039701619904774,
"loss_nats": 1.2805960178375244,
"step": 1720,
"unclipped_grad_norm": 0.7415109276771545
},
{
"bits_per_byte": 1.944612132872049,
"learning_rate": 0.0002801064221878024,
"loss_nats": 1.3479024171829224,
"step": 1730,
"unclipped_grad_norm": 0.8057509660720825
},
{
"bits_per_byte": 1.956270664791388,
"learning_rate": 0.00027981386132539134,
"loss_nats": 1.3559834957122803,
"step": 1740,
"unclipped_grad_norm": 0.8188740611076355
},
{
"bits_per_byte": 1.908994697861012,
"learning_rate": 0.0002795193386108724,
"loss_nats": 1.3232142925262451,
"step": 1750,
"unclipped_grad_norm": 0.7595267295837402
},
{
"bits_per_byte": 1.9438418225788343,
"learning_rate": 0.00027922285907682625,
"loss_nats": 1.3473684787750244,
"step": 1760,
"unclipped_grad_norm": 0.7832275629043579
},
{
"bits_per_byte": 1.9277638524802738,
"learning_rate": 0.00027892442778927005,
"loss_nats": 1.33622407913208,
"step": 1770,
"unclipped_grad_norm": 0.7276267409324646
},
{
"bits_per_byte": 1.987644083927083,
"learning_rate": 0.00027862404984757116,
"loss_nats": 1.377729892730713,
"step": 1780,
"unclipped_grad_norm": 0.7699368000030518
},
{
"bits_per_byte": 1.8704368754681737,
"learning_rate": 0.0002783217303843597,
"loss_nats": 1.2964880466461182,
"step": 1790,
"unclipped_grad_norm": 0.7781511545181274
},
{
"bits_per_byte": 2.006532250540742,
"learning_rate": 0.00027801747456544134,
"loss_nats": 1.390822172164917,
"step": 1800,
"unclipped_grad_norm": 0.8113785982131958
},
{
"bits_per_byte": 1.8832052114335305,
"learning_rate": 0.00027771128758970843,
"loss_nats": 1.3053383827209473,
"step": 1810,
"unclipped_grad_norm": 0.8088275790214539
},
{
"bits_per_byte": 1.9263209180395349,
"learning_rate": 0.00027740317468905165,
"loss_nats": 1.335223913192749,
"step": 1820,
"unclipped_grad_norm": 0.7881855964660645
},
{
"bits_per_byte": 1.968589438107389,
"learning_rate": 0.0002770931411282704,
"loss_nats": 1.3645222187042236,
"step": 1830,
"unclipped_grad_norm": 0.7948311567306519
},
{
"bits_per_byte": 1.9398172565659937,
"learning_rate": 0.0002767811922049827,
"loss_nats": 1.3445788621902466,
"step": 1840,
"unclipped_grad_norm": 0.8343047499656677
},
{
"bits_per_byte": 1.8747907882574166,
"learning_rate": 0.00027646733324953503,
"loss_nats": 1.2995059490203857,
"step": 1850,
"unclipped_grad_norm": 0.7942258715629578
},
{
"bits_per_byte": 1.9135037390013332,
"learning_rate": 0.00027615156962491097,
"loss_nats": 1.3263397216796875,
"step": 1860,
"unclipped_grad_norm": 0.769598126411438
},
{
"bits_per_byte": 1.987024774401326,
"learning_rate": 0.0002758339067266396,
"loss_nats": 1.3773006200790405,
"step": 1870,
"unclipped_grad_norm": 0.7647593021392822
},
{
"bits_per_byte": 1.8769319722606586,
"learning_rate": 0.0002755143499827035,
"loss_nats": 1.300990104675293,
"step": 1880,
"unclipped_grad_norm": 0.8446611166000366
},
{
"bits_per_byte": 1.8953207012562119,
"learning_rate": 0.0002751929048534456,
"loss_nats": 1.3137362003326416,
"step": 1890,
"unclipped_grad_norm": 0.7048609852790833
},
{
"bits_per_byte": 1.915848206497895,
"learning_rate": 0.0002748695768314763,
"loss_nats": 1.3279647827148438,
"step": 1900,
"unclipped_grad_norm": 0.7416327595710754
},
{
"bits_per_byte": 1.9362319342434569,
"learning_rate": 0.00027454437144157947,
"loss_nats": 1.3420937061309814,
"step": 1910,
"unclipped_grad_norm": 0.8084686398506165
},
{
"bits_per_byte": 1.8860243510464736,
"learning_rate": 0.00027421729424061787,
"loss_nats": 1.3072924613952637,
"step": 1920,
"unclipped_grad_norm": 0.8338462710380554
},
{
"bits_per_byte": 1.930916638435891,
"learning_rate": 0.0002738883508174385,
"loss_nats": 1.338409423828125,
"step": 1930,
"unclipped_grad_norm": 0.6878981590270996
},
{
"bits_per_byte": 1.9257588787365152,
"learning_rate": 0.0002735575467927768,
"loss_nats": 1.334834337234497,
"step": 1940,
"unclipped_grad_norm": 0.6981199383735657
},
{
"bits_per_byte": 1.8699291826828268,
"learning_rate": 0.0002732248878191609,
"loss_nats": 1.2961361408233643,
"step": 1950,
"unclipped_grad_norm": 0.76193767786026
},
{
"bits_per_byte": 1.859464038377825,
"learning_rate": 0.0002728903795808148,
"loss_nats": 1.2888822555541992,
"step": 1960,
"unclipped_grad_norm": 0.7367894053459167
},
{
"bits_per_byte": 1.8691335909399354,
"learning_rate": 0.0002725540277935614,
"loss_nats": 1.2955846786499023,
"step": 1970,
"unclipped_grad_norm": 0.8052698373794556
},
{
"bits_per_byte": 1.8850715671606937,
"learning_rate": 0.00027221583820472456,
"loss_nats": 1.3066320419311523,
"step": 1980,
"unclipped_grad_norm": 0.7624203562736511
},
{
"bits_per_byte": 1.8715028239382214,
"learning_rate": 0.0002718758165930314,
"loss_nats": 1.297226905822754,
"step": 1990,
"unclipped_grad_norm": 0.779162585735321
},
{
"bits_per_byte": 1.9325518494802945,
"learning_rate": 0.0002715339687685131,
"loss_nats": 1.3395428657531738,
"step": 2000,
"unclipped_grad_norm": 0.7498615384101868
},
{
"bits_per_byte": 1.9335421255839624,
"learning_rate": 0.00027119030057240576,
"loss_nats": 1.3402292728424072,
"step": 2010,
"unclipped_grad_norm": 0.7343811392784119
},
{
"bits_per_byte": 1.838237939608337,
"learning_rate": 0.0002708448178770506,
"loss_nats": 1.2741694450378418,
"step": 2020,
"unclipped_grad_norm": 0.7684847116470337
},
{
"bits_per_byte": 1.8631659649376469,
"learning_rate": 0.00027049752658579376,
"loss_nats": 1.2914482355117798,
"step": 2030,
"unclipped_grad_norm": 0.6878870129585266
},
{
"bits_per_byte": 1.8841837927169398,
"learning_rate": 0.00027014843263288514,
"loss_nats": 1.3060166835784912,
"step": 2040,
"unclipped_grad_norm": 0.853155255317688
},
{
"bits_per_byte": 1.8339112000779305,
"learning_rate": 0.00026979754198337724,
"loss_nats": 1.2711703777313232,
"step": 2050,
"unclipped_grad_norm": 0.8042898774147034
},
{
"bits_per_byte": 1.8136427007084468,
"learning_rate": 0.00026944486063302305,
"loss_nats": 1.2571213245391846,
"step": 2060,
"unclipped_grad_norm": 0.7331950664520264
},
{
"bits_per_byte": 1.9306452498128293,
"learning_rate": 0.0002690903946081738,
"loss_nats": 1.3382213115692139,
"step": 2070,
"unclipped_grad_norm": 0.7544640898704529
},
{
"bits_per_byte": 1.9159075405124426,
"learning_rate": 0.00026873414996567596,
"loss_nats": 1.3280059099197388,
"step": 2080,
"unclipped_grad_norm": 0.7521310448646545
},
{
"bits_per_byte": 1.8394872215842044,
"learning_rate": 0.00026837613279276736,
"loss_nats": 1.2750353813171387,
"step": 2090,
"unclipped_grad_norm": 0.7430872321128845
},
{
"bits_per_byte": 1.7890550290451945,
"learning_rate": 0.0002680163492069739,
"loss_nats": 1.2400784492492676,
"step": 2100,
"unclipped_grad_norm": 0.7162277698516846
},
{
"bits_per_byte": 1.8155914361253733,
"learning_rate": 0.00026765480535600414,
"loss_nats": 1.2584720849990845,
"step": 2110,
"unclipped_grad_norm": 0.7056337594985962
},
{
"bits_per_byte": 1.834290249840432,
"learning_rate": 0.0002672915074176452,
"loss_nats": 1.2714331150054932,
"step": 2120,
"unclipped_grad_norm": 0.8666519522666931
},
{
"bits_per_byte": 1.8865812308699674,
"learning_rate": 0.00026692646159965624,
"loss_nats": 1.307678461074829,
"step": 2130,
"unclipped_grad_norm": 0.762921154499054
},
{
"bits_per_byte": 1.9030090136803828,
"learning_rate": 0.0002665596741396633,
"loss_nats": 1.3190653324127197,
"step": 2140,
"unclipped_grad_norm": 0.7508745789527893
},
{
"bits_per_byte": 1.8090877402003454,
"learning_rate": 0.000266191151305052,
"loss_nats": 1.2539640665054321,
"step": 2150,
"unclipped_grad_norm": 0.7472740411758423
},
{
"bits_per_byte": 1.898260228724759,
"learning_rate": 0.00026582089939286075,
"loss_nats": 1.3157737255096436,
"step": 2160,
"unclipped_grad_norm": 0.7288866639137268
},
{
"bits_per_byte": 1.8098697453138184,
"learning_rate": 0.0002654489247296732,
"loss_nats": 1.2545061111450195,
"step": 2170,
"unclipped_grad_norm": 0.7270416617393494
},
{
"bits_per_byte": 1.896411071262682,
"learning_rate": 0.00026507523367151007,
"loss_nats": 1.3144919872283936,
"step": 2180,
"unclipped_grad_norm": 0.7397056221961975
},
{
"bits_per_byte": 1.8517294626205718,
"learning_rate": 0.00026469983260372037,
"loss_nats": 1.283521056175232,
"step": 2190,
"unclipped_grad_norm": 0.7186560034751892
},
{
"bits_per_byte": 1.8299633383273748,
"learning_rate": 0.0002643227279408727,
"loss_nats": 1.268433928489685,
"step": 2200,
"unclipped_grad_norm": 0.6916610598564148
},
{
"bits_per_byte": 1.8274437924922347,
"learning_rate": 0.0002639439261266453,
"loss_nats": 1.2666875123977661,
"step": 2210,
"unclipped_grad_norm": 0.7494430541992188
},
{
"bits_per_byte": 1.8262679471082857,
"learning_rate": 0.000263563433633716,
"loss_nats": 1.2658724784851074,
"step": 2220,
"unclipped_grad_norm": 0.782909095287323
},
{
"bits_per_byte": 1.782249847533204,
"learning_rate": 0.00026318125696365156,
"loss_nats": 1.2353614568710327,
"step": 2230,
"unclipped_grad_norm": 0.7370295524597168
},
{
"bits_per_byte": 1.8783687153259665,
"learning_rate": 0.000262797402646797,
"loss_nats": 1.3019859790802002,
"step": 2240,
"unclipped_grad_norm": 0.7077252268791199
},
{
"bits_per_byte": 1.844123529886164,
"learning_rate": 0.00026241187724216334,
"loss_nats": 1.2782490253448486,
"step": 2250,
"unclipped_grad_norm": 0.7420696020126343
},
{
"bits_per_byte": 1.897104505310961,
"learning_rate": 0.0002620246873373162,
"loss_nats": 1.3149726390838623,
"step": 2260,
"unclipped_grad_norm": 0.7444695234298706
},
{
"bits_per_byte": 1.7929542197055561,
"learning_rate": 0.00026163583954826264,
"loss_nats": 1.242781162261963,
"step": 2270,
"unclipped_grad_norm": 0.7164454460144043
},
{
"bits_per_byte": 1.8643507534194408,
"learning_rate": 0.0002612453405193386,
"loss_nats": 1.2922694683074951,
"step": 2280,
"unclipped_grad_norm": 0.7152382135391235
},
{
"bits_per_byte": 1.8035947863144381,
"learning_rate": 0.000260853196923095,
"loss_nats": 1.2501566410064697,
"step": 2290,
"unclipped_grad_norm": 0.7178454399108887
},
{
"bits_per_byte": 1.7935200426268947,
"learning_rate": 0.0002604594154601839,
"loss_nats": 1.243173360824585,
"step": 2300,
"unclipped_grad_norm": 0.7068437337875366
},
{
"bits_per_byte": 1.8020768674379228,
"learning_rate": 0.0002600640028592441,
"loss_nats": 1.2491044998168945,
"step": 2310,
"unclipped_grad_norm": 0.7017168998718262
},
{
"bits_per_byte": 1.8564474626816911,
"learning_rate": 0.0002596669658767859,
"loss_nats": 1.2867913246154785,
"step": 2320,
"unclipped_grad_norm": 0.6947462558746338
},
{
"bits_per_byte": 1.7936961528613782,
"learning_rate": 0.000259268311297076,
"loss_nats": 1.243295431137085,
"step": 2330,
"unclipped_grad_norm": 0.6949398517608643
},
{
"bits_per_byte": 1.8129625093242843,
"learning_rate": 0.00025886804593202114,
"loss_nats": 1.2566498517990112,
"step": 2340,
"unclipped_grad_norm": 0.7047421336174011
},
{
"bits_per_byte": 1.871945851246844,
"learning_rate": 0.00025846617662105193,
"loss_nats": 1.2975339889526367,
"step": 2350,
"unclipped_grad_norm": 0.7291849851608276
},
{
"bits_per_byte": 1.7987307729827362,
"learning_rate": 0.00025806271023100637,
"loss_nats": 1.2467851638793945,
"step": 2360,
"unclipped_grad_norm": 0.7353813052177429
},
{
"bits_per_byte": 1.7311453748119714,
"learning_rate": 0.00025765765365601165,
"loss_nats": 1.1999385356903076,
"step": 2370,
"unclipped_grad_norm": 0.6746760010719299
},
{
"bits_per_byte": 1.834468595849377,
"learning_rate": 0.00025725101381736707,
"loss_nats": 1.2715567350387573,
"step": 2380,
"unclipped_grad_norm": 0.7166123390197754
},
{
"bits_per_byte": 1.8260691351638882,
"learning_rate": 0.00025684279766342567,
"loss_nats": 1.2657346725463867,
"step": 2390,
"unclipped_grad_norm": 0.6613916158676147
},
{
"bits_per_byte": 1.8641103216735346,
"learning_rate": 0.00025643301216947504,
"loss_nats": 1.2921028137207031,
"step": 2400,
"unclipped_grad_norm": 0.824955403804779
},
{
"bits_per_byte": 1.793840446305452,
"learning_rate": 0.00025602166433761866,
"loss_nats": 1.243395447731018,
"step": 2410,
"unclipped_grad_norm": 0.7929511070251465
},
{
"bits_per_byte": 1.8233349549804712,
"learning_rate": 0.0002556087611966561,
"loss_nats": 1.2638394832611084,
"step": 2420,
"unclipped_grad_norm": 0.725745677947998
},
{
"bits_per_byte": 1.825773496987055,
"learning_rate": 0.00025519430980196277,
"loss_nats": 1.265529751777649,
"step": 2430,
"unclipped_grad_norm": 0.7284960746765137
},
{
"bits_per_byte": 1.7725216488871491,
"learning_rate": 0.0002547783172353695,
"loss_nats": 1.2286183834075928,
"step": 2440,
"unclipped_grad_norm": 0.7458904981613159
},
{
"bits_per_byte": 1.9001444706476118,
"learning_rate": 0.0002543607906050415,
"loss_nats": 1.317079782485962,
"step": 2450,
"unclipped_grad_norm": 0.8321223258972168
},
{
"bits_per_byte": 1.7654523020234334,
"learning_rate": 0.0002539417370453568,
"loss_nats": 1.223718285560608,
"step": 2460,
"unclipped_grad_norm": 0.7338651418685913
},
{
"bits_per_byte": 1.7862824967306352,
"learning_rate": 0.00025352116371678464,
"loss_nats": 1.2381566762924194,
"step": 2470,
"unclipped_grad_norm": 0.7279908061027527
},
{
"bits_per_byte": 1.8666427662074894,
"learning_rate": 0.0002530990778057627,
"loss_nats": 1.2938581705093384,
"step": 2480,
"unclipped_grad_norm": 0.7701565027236938
},
{
"bits_per_byte": 1.850673661126925,
"learning_rate": 0.00025267548652457475,
"loss_nats": 1.2827892303466797,
"step": 2490,
"unclipped_grad_norm": 0.6776601672172546
},
{
"bits_per_byte": 1.8000481600883451,
"learning_rate": 0.00025225039711122696,
"loss_nats": 1.2476983070373535,
"step": 2500,
"unclipped_grad_norm": 0.7444605231285095
},
{
"bits_per_byte": 1.7603095048146828,
"learning_rate": 0.00025182381682932464,
"loss_nats": 1.220153570175171,
"step": 2510,
"unclipped_grad_norm": 0.7512273788452148
},
{
"bits_per_byte": 1.7805241736144466,
"learning_rate": 0.0002513957529679477,
"loss_nats": 1.2341653108596802,
"step": 2520,
"unclipped_grad_norm": 0.6968159079551697
},
{
"bits_per_byte": 1.803689032807111,
"learning_rate": 0.00025096621284152656,
"loss_nats": 1.2502219676971436,
"step": 2530,
"unclipped_grad_norm": 0.7150991559028625
},
{
"bits_per_byte": 1.8049077018711248,
"learning_rate": 0.0002505352037897168,
"loss_nats": 1.2510666847229004,
"step": 2540,
"unclipped_grad_norm": 0.7343016862869263
},
{
"bits_per_byte": 1.742795479581418,
"learning_rate": 0.0002501027331772739,
"loss_nats": 1.2080137729644775,
"step": 2550,
"unclipped_grad_norm": 0.7049802541732788
},
{
"bits_per_byte": 1.7179068382791558,
"learning_rate": 0.00024966880839392746,
"loss_nats": 1.1907622814178467,
"step": 2560,
"unclipped_grad_norm": 0.7055768370628357
},
{
"bits_per_byte": 1.7731971967397384,
"learning_rate": 0.0002492334368542548,
"loss_nats": 1.2290866374969482,
"step": 2570,
"unclipped_grad_norm": 0.6996883749961853
},
{
"bits_per_byte": 1.7458676056737803,
"learning_rate": 0.00024879662599755416,
"loss_nats": 1.2101432085037231,
"step": 2580,
"unclipped_grad_norm": 0.6933735609054565
},
{
"bits_per_byte": 1.7458146350173145,
"learning_rate": 0.00024835838328771803,
"loss_nats": 1.2101064920425415,
"step": 2590,
"unclipped_grad_norm": 0.7094681859016418
},
{
"bits_per_byte": 1.7637799427264436,
"learning_rate": 0.0002479187162131051,
"loss_nats": 1.2225590944290161,
"step": 2600,
"unclipped_grad_norm": 0.7199745178222656
},
{
"bits_per_byte": 1.786311561798631,
"learning_rate": 0.00024747763228641264,
"loss_nats": 1.2381768226623535,
"step": 2610,
"unclipped_grad_norm": 0.7421150207519531
},
{
"bits_per_byte": 1.7252271078304804,
"learning_rate": 0.00024703513904454795,
"loss_nats": 1.1958363056182861,
"step": 2620,
"unclipped_grad_norm": 0.7062536478042603
},
{
"bits_per_byte": 1.7243496523457782,
"learning_rate": 0.00024659124404849956,
"loss_nats": 1.195228099822998,
"step": 2630,
"unclipped_grad_norm": 0.6937817931175232
},
{
"bits_per_byte": 1.7124126805146924,
"learning_rate": 0.00024614595488320833,
"loss_nats": 1.1869540214538574,
"step": 2640,
"unclipped_grad_norm": 0.7252171039581299
},
{
"bits_per_byte": 1.781446344588373,
"learning_rate": 0.00024569927915743745,
"loss_nats": 1.2348045110702515,
"step": 2650,
"unclipped_grad_norm": 0.6726356744766235
},
{
"bits_per_byte": 1.6915262474806587,
"learning_rate": 0.0002452512245036427,
"loss_nats": 1.1724766492843628,
"step": 2660,
"unclipped_grad_norm": 0.6618354320526123
},
{
"bits_per_byte": 1.7563157236963467,
"learning_rate": 0.00024480179857784177,
"loss_nats": 1.2173852920532227,
"step": 2670,
"unclipped_grad_norm": 0.7113842368125916
},
{
"bits_per_byte": 1.858153874543552,
"learning_rate": 0.00024435100905948387,
"loss_nats": 1.2879741191864014,
"step": 2680,
"unclipped_grad_norm": 0.7770810127258301
},
{
"bits_per_byte": 1.784997614346039,
"learning_rate": 0.00024389886365131793,
"loss_nats": 1.2372660636901855,
"step": 2690,
"unclipped_grad_norm": 0.7249491810798645
},
{
"bits_per_byte": 1.7001362149308035,
"learning_rate": 0.00024344537007926163,
"loss_nats": 1.1784446239471436,
"step": 2700,
"unclipped_grad_norm": 0.6978740096092224
},
{
"bits_per_byte": 1.7671420315681605,
"learning_rate": 0.00024299053609226878,
"loss_nats": 1.2248895168304443,
"step": 2710,
"unclipped_grad_norm": 0.6759880185127258
},
{
"bits_per_byte": 1.8404900524213854,
"learning_rate": 0.00024253436946219733,
"loss_nats": 1.2757304906845093,
"step": 2720,
"unclipped_grad_norm": 0.7600791454315186
},
{
"bits_per_byte": 1.708491820040322,
"learning_rate": 0.0002420768779836764,
"loss_nats": 1.1842362880706787,
"step": 2730,
"unclipped_grad_norm": 0.6715761423110962
},
{
"bits_per_byte": 1.7849604660934528,
"learning_rate": 0.00024161806947397305,
"loss_nats": 1.2372403144836426,
"step": 2740,
"unclipped_grad_norm": 0.7194302082061768
},
{
"bits_per_byte": 1.7485555225241152,
"learning_rate": 0.00024115795177285886,
"loss_nats": 1.2120063304901123,
"step": 2750,
"unclipped_grad_norm": 0.6991336941719055
},
{
"bits_per_byte": 1.8629487508496072,
"learning_rate": 0.0002406965327424758,
"loss_nats": 1.2912976741790771,
"step": 2760,
"unclipped_grad_norm": 0.8377771973609924
},
{
"bits_per_byte": 1.7856594035865592,
"learning_rate": 0.00024023382026720197,
"loss_nats": 1.237724781036377,
"step": 2770,
"unclipped_grad_norm": 0.7150954604148865
},
{
"bits_per_byte": 1.7901013714930438,
"learning_rate": 0.00023976982225351698,
"loss_nats": 1.2408037185668945,
"step": 2780,
"unclipped_grad_norm": 0.6968044638633728
},
{
"bits_per_byte": 1.752669863480706,
"learning_rate": 0.0002393045466298666,
"loss_nats": 1.2148581743240356,
"step": 2790,
"unclipped_grad_norm": 0.7223197817802429
},
{
"bits_per_byte": 1.7567675221201633,
"learning_rate": 0.00023883800134652763,
"loss_nats": 1.2176984548568726,
"step": 2800,
"unclipped_grad_norm": 0.7307369709014893
},
{
"bits_per_byte": 1.7124591158304254,
"learning_rate": 0.0002383701943754715,
"loss_nats": 1.1869862079620361,
"step": 2810,
"unclipped_grad_norm": 0.7025272250175476
},
{
"bits_per_byte": 1.7803284573577647,
"learning_rate": 0.00023790113371022884,
"loss_nats": 1.2340296506881714,
"step": 2820,
"unclipped_grad_norm": 0.7275472283363342
},
{
"bits_per_byte": 1.7225665362216327,
"learning_rate": 0.0002374308273657521,
"loss_nats": 1.1939921379089355,
"step": 2830,
"unclipped_grad_norm": 0.6614095568656921
},
{
"bits_per_byte": 1.7602923065495966,
"learning_rate": 0.00023695928337827911,
"loss_nats": 1.2201416492462158,
"step": 2840,
"unclipped_grad_norm": 0.7144313454627991
},
{
"bits_per_byte": 1.7021394688480533,
"learning_rate": 0.00023648650980519564,
"loss_nats": 1.179833173751831,
"step": 2850,
"unclipped_grad_norm": 0.7220690846443176
},
{
"bits_per_byte": 1.744293620453084,
"learning_rate": 0.00023601251472489757,
"loss_nats": 1.2090522050857544,
"step": 2860,
"unclipped_grad_norm": 0.7174125909805298
},
{
"bits_per_byte": 1.7433115995166573,
"learning_rate": 0.00023553730623665305,
"loss_nats": 1.2083715200424194,
"step": 2870,
"unclipped_grad_norm": 0.730305552482605
},
{
"bits_per_byte": 1.7421880368585705,
"learning_rate": 0.00023506089246046403,
"loss_nats": 1.2075927257537842,
"step": 2880,
"unclipped_grad_norm": 0.7075303792953491
},
{
"bits_per_byte": 1.7085926018737276,
"learning_rate": 0.00023458328153692736,
"loss_nats": 1.1843061447143555,
"step": 2890,
"unclipped_grad_norm": 0.6744402050971985
},
{
"bits_per_byte": 1.7378320602775172,
"learning_rate": 0.0002341044816270961,
"loss_nats": 1.204573392868042,
"step": 2900,
"unclipped_grad_norm": 0.6889722347259521
},
{
"bits_per_byte": 1.7031551983840492,
"learning_rate": 0.00023362450091233953,
"loss_nats": 1.180537223815918,
"step": 2910,
"unclipped_grad_norm": 0.7552688717842102
},
{
"bits_per_byte": 1.7508614659068837,
"learning_rate": 0.00023314334759420388,
"loss_nats": 1.2136046886444092,
"step": 2920,
"unclipped_grad_norm": 0.6863148808479309
},
{
"bits_per_byte": 1.701510700276499,
"learning_rate": 0.0002326610298942717,
"loss_nats": 1.1793973445892334,
"step": 2930,
"unclipped_grad_norm": 0.6827563047409058
},
{
"bits_per_byte": 1.7106488264474435,
"learning_rate": 0.00023217755605402175,
"loss_nats": 1.1857314109802246,
"step": 2940,
"unclipped_grad_norm": 0.6807736158370972
},
{
"bits_per_byte": 1.7703209588867088,
"learning_rate": 0.00023169293433468802,
"loss_nats": 1.227092981338501,
"step": 2950,
"unclipped_grad_norm": 0.6866815686225891
},
{
"bits_per_byte": 1.7651767858167513,
"learning_rate": 0.00023120717301711857,
"loss_nats": 1.2235273122787476,
"step": 2960,
"unclipped_grad_norm": 0.741482138633728
},
{
"bits_per_byte": 1.7465264712092357,
"learning_rate": 0.00023072028040163408,
"loss_nats": 1.2105998992919922,
"step": 2970,
"unclipped_grad_norm": 0.6754798293113708
},
{
"bits_per_byte": 1.7663952828981142,
"learning_rate": 0.00023023226480788593,
"loss_nats": 1.2243719100952148,
"step": 2980,
"unclipped_grad_norm": 0.688591480255127
},
{
"bits_per_byte": 1.7069632382394535,
"learning_rate": 0.00022974313457471416,
"loss_nats": 1.1831767559051514,
"step": 2990,
"unclipped_grad_norm": 0.7298949956893921
},
{
"bits_per_byte": 1.79113911480835,
"learning_rate": 0.0002292528980600049,
"loss_nats": 1.241523027420044,
"step": 3000,
"unclipped_grad_norm": 0.713433027267456
},
{
"bits_per_byte": 1.6766162115641075,
"learning_rate": 0.00022876156364054758,
"loss_nats": 1.1621417999267578,
"step": 3010,
"unclipped_grad_norm": 0.7228698134422302
},
{
"bits_per_byte": 1.7365501015979858,
"learning_rate": 0.0002282691397118918,
"loss_nats": 1.2036848068237305,
"step": 3020,
"unclipped_grad_norm": 0.6969108581542969
},
{
"bits_per_byte": 1.6833617150962485,
"learning_rate": 0.00022777563468820376,
"loss_nats": 1.1668174266815186,
"step": 3030,
"unclipped_grad_norm": 0.669261634349823
},
{
"bits_per_byte": 1.6671776317021056,
"learning_rate": 0.00022728105700212282,
"loss_nats": 1.1555994749069214,
"step": 3040,
"unclipped_grad_norm": 0.6839757561683655
},
{
"bits_per_byte": 1.8241205717296125,
"learning_rate": 0.00022678541510461698,
"loss_nats": 1.2643840312957764,
"step": 3050,
"unclipped_grad_norm": 0.7963703870773315
},
{
"bits_per_byte": 1.7604708245411922,
"learning_rate": 0.00022628871746483876,
"loss_nats": 1.2202653884887695,
"step": 3060,
"unclipped_grad_norm": 0.7280165553092957
},
{
"bits_per_byte": 1.6594571585222233,
"learning_rate": 0.0002257909725699804,
"loss_nats": 1.1502480506896973,
"step": 3070,
"unclipped_grad_norm": 0.7014188766479492
},
{
"bits_per_byte": 1.7706030104341237,
"learning_rate": 0.00022529218892512886,
"loss_nats": 1.2272884845733643,
"step": 3080,
"unclipped_grad_norm": 0.7385008335113525
},
{
"bits_per_byte": 1.7493225651469633,
"learning_rate": 0.00022479237505312034,
"loss_nats": 1.2125380039215088,
"step": 3090,
"unclipped_grad_norm": 0.6976860165596008
},
{
"bits_per_byte": 1.706454857523503,
"learning_rate": 0.00022429153949439496,
"loss_nats": 1.1828243732452393,
"step": 3100,
"unclipped_grad_norm": 0.7350029349327087
},
{
"bits_per_byte": 1.8454660264587992,
"learning_rate": 0.00022378969080685053,
"loss_nats": 1.279179573059082,
"step": 3110,
"unclipped_grad_norm": 0.8262335658073425
},
{
"bits_per_byte": 1.7237298268720687,
"learning_rate": 0.0002232868375656965,
"loss_nats": 1.194798469543457,
"step": 3120,
"unclipped_grad_norm": 0.7770306468009949
},
{
"bits_per_byte": 1.6658043502349662,
"learning_rate": 0.00022278298836330733,
"loss_nats": 1.1546475887298584,
"step": 3130,
"unclipped_grad_norm": 0.6939164996147156
},
{
"bits_per_byte": 1.7147224075157799,
"learning_rate": 0.00022227815180907572,
"loss_nats": 1.1885550022125244,
"step": 3140,
"unclipped_grad_norm": 0.744745135307312
},
{
"bits_per_byte": 1.7012960659282221,
"learning_rate": 0.00022177233652926546,
"loss_nats": 1.179248571395874,
"step": 3150,
"unclipped_grad_norm": 0.6927894353866577
},
{
"bits_per_byte": 1.700822425707746,
"learning_rate": 0.00022126555116686416,
"loss_nats": 1.1789202690124512,
"step": 3160,
"unclipped_grad_norm": 0.6962236166000366
},
{
"bits_per_byte": 1.7730090477196945,
"learning_rate": 0.00022075780438143532,
"loss_nats": 1.2289562225341797,
"step": 3170,
"unclipped_grad_norm": 0.7581587433815002
},
{
"bits_per_byte": 1.7445914944043783,
"learning_rate": 0.00022024910484897065,
"loss_nats": 1.2092586755752563,
"step": 3180,
"unclipped_grad_norm": 0.7753618359565735
},
{
"bits_per_byte": 1.7320080397886992,
"learning_rate": 0.00021973946126174158,
"loss_nats": 1.2005364894866943,
"step": 3190,
"unclipped_grad_norm": 0.7117310166358948
},
{
"bits_per_byte": 1.7215322325593436,
"learning_rate": 0.00021922888232815086,
"loss_nats": 1.1932752132415771,
"step": 3200,
"unclipped_grad_norm": 0.712298572063446
},
{
"bits_per_byte": 1.6960516269728116,
"learning_rate": 0.0002187173767725837,
"loss_nats": 1.1756134033203125,
"step": 3210,
"unclipped_grad_norm": 0.7590813636779785
},
{
"bits_per_byte": 1.6593746068498092,
"learning_rate": 0.0002182049533352588,
"loss_nats": 1.150190830230713,
"step": 3220,
"unclipped_grad_norm": 0.6987801194190979
},
{
"bits_per_byte": 1.7324070395387008,
"learning_rate": 0.00021769162077207884,
"loss_nats": 1.2008130550384521,
"step": 3230,
"unclipped_grad_norm": 0.6938228607177734
},
{
"bits_per_byte": 1.7474627447605329,
"learning_rate": 0.00021717738785448102,
"loss_nats": 1.2112488746643066,
"step": 3240,
"unclipped_grad_norm": 0.78226238489151
},
{
"bits_per_byte": 1.7110863503112386,
"learning_rate": 0.00021666226336928708,
"loss_nats": 1.1860346794128418,
"step": 3250,
"unclipped_grad_norm": 0.6999595165252686
},
{
"bits_per_byte": 1.7142417160066183,
"learning_rate": 0.0002161462561185532,
"loss_nats": 1.18822181224823,
"step": 3260,
"unclipped_grad_norm": 0.700433075428009
},
{
"bits_per_byte": 1.748484321706658,
"learning_rate": 0.00021562937491941957,
"loss_nats": 1.2119569778442383,
"step": 3270,
"unclipped_grad_norm": 0.7274234294891357
},
{
"bits_per_byte": 1.738793787261142,
"learning_rate": 0.00021511162860395975,
"loss_nats": 1.20524001121521,
"step": 3280,
"unclipped_grad_norm": 0.7183425426483154
},
{
"bits_per_byte": 1.6962500949519075,
"learning_rate": 0.0002145930260190298,
"loss_nats": 1.175750970840454,
"step": 3290,
"unclipped_grad_norm": 0.7457373738288879
},
{
"bits_per_byte": 1.7210296992535226,
"learning_rate": 0.00021407357602611698,
"loss_nats": 1.1929268836975098,
"step": 3300,
"unclipped_grad_norm": 0.7834483981132507
},
{
"bits_per_byte": 1.6835460804979734,
"learning_rate": 0.0002135532875011885,
"loss_nats": 1.166945219039917,
"step": 3310,
"unclipped_grad_norm": 0.6523665189743042
},
{
"bits_per_byte": 1.6871995079502522,
"learning_rate": 0.0002130321693345397,
"loss_nats": 1.1694775819778442,
"step": 3320,
"unclipped_grad_norm": 0.7348048686981201
},
{
"bits_per_byte": 1.7008401399207849,
"learning_rate": 0.00021251023043064224,
"loss_nats": 1.178932547569275,
"step": 3330,
"unclipped_grad_norm": 0.7228167653083801
},
{
"bits_per_byte": 1.643402750046831,
"learning_rate": 0.00021198747970799188,
"loss_nats": 1.1391199827194214,
"step": 3340,
"unclipped_grad_norm": 0.7094708681106567
},
{
"bits_per_byte": 1.686364876145615,
"learning_rate": 0.00021146392609895618,
"loss_nats": 1.1688990592956543,
"step": 3350,
"unclipped_grad_norm": 0.7186631560325623
},
{
"bits_per_byte": 1.7065993229502279,
"learning_rate": 0.00021093957854962178,
"loss_nats": 1.182924509048462,
"step": 3360,
"unclipped_grad_norm": 0.6950210332870483
},
{
"bits_per_byte": 1.6548356407282379,
"learning_rate": 0.00021041444601964157,
"loss_nats": 1.1470446586608887,
"step": 3370,
"unclipped_grad_norm": 0.6926314830780029
},
{
"bits_per_byte": 1.6926787032240904,
"learning_rate": 0.00020988853748208164,
"loss_nats": 1.1732754707336426,
"step": 3380,
"unclipped_grad_norm": 0.7107762694358826
},
{
"bits_per_byte": 1.7003921251152874,
"learning_rate": 0.00020936186192326782,
"loss_nats": 1.1786220073699951,
"step": 3390,
"unclipped_grad_norm": 0.700124979019165
},
{
"bits_per_byte": 1.6889117672222422,
"learning_rate": 0.00020883442834263225,
"loss_nats": 1.1706644296646118,
"step": 3400,
"unclipped_grad_norm": 0.7049819827079773
},
{
"bits_per_byte": 1.6802605239358903,
"learning_rate": 0.0002083062457525596,
"loss_nats": 1.1646678447723389,
"step": 3410,
"unclipped_grad_norm": 0.6922183036804199
},
{
"bits_per_byte": 1.664921907253389,
"learning_rate": 0.00020777732317823306,
"loss_nats": 1.1540359258651733,
"step": 3420,
"unclipped_grad_norm": 0.7144856452941895
},
{
"bits_per_byte": 1.698013089105902,
"learning_rate": 0.00020724766965748005,
"loss_nats": 1.1769729852676392,
"step": 3430,
"unclipped_grad_norm": 0.6953245401382446
},
{
"bits_per_byte": 1.7072490734051875,
"learning_rate": 0.00020671729424061788,
"loss_nats": 1.1833748817443848,
"step": 3440,
"unclipped_grad_norm": 0.727461040019989
},
{
"bits_per_byte": 1.7747906159999822,
"learning_rate": 0.00020618620599029906,
"loss_nats": 1.2301911115646362,
"step": 3450,
"unclipped_grad_norm": 0.7409155368804932
},
{
"bits_per_byte": 1.7588486841782542,
"learning_rate": 0.00020565441398135649,
"loss_nats": 1.2191410064697266,
"step": 3460,
"unclipped_grad_norm": 0.748551070690155
},
{
"bits_per_byte": 1.7339454243506687,
"learning_rate": 0.00020512192730064827,
"loss_nats": 1.2018793821334839,
"step": 3470,
"unclipped_grad_norm": 0.7151429653167725
},
{
"bits_per_byte": 1.7067943512763062,
"learning_rate": 0.00020458875504690273,
"loss_nats": 1.1830596923828125,
"step": 3480,
"unclipped_grad_norm": 0.7189637422561646
},
{
"bits_per_byte": 1.6200879219826143,
"learning_rate": 0.00020405490633056245,
"loss_nats": 1.1229593753814697,
"step": 3490,
"unclipped_grad_norm": 0.6848708391189575
},
{
"bits_per_byte": 1.6373195517031627,
"learning_rate": 0.00020352039027362917,
"loss_nats": 1.1349034309387207,
"step": 3500,
"unclipped_grad_norm": 0.6704034209251404
},
{
"bits_per_byte": 1.6849599498707166,
"learning_rate": 0.0002029852160095075,
"loss_nats": 1.167925238609314,
"step": 3510,
"unclipped_grad_norm": 0.7942876219749451
},
{
"bits_per_byte": 1.6514716600773613,
"learning_rate": 0.000202449392682849,
"loss_nats": 1.1447129249572754,
"step": 3520,
"unclipped_grad_norm": 0.7149106860160828
},
{
"bits_per_byte": 1.6705435041621417,
"learning_rate": 0.0002019129294493959,
"loss_nats": 1.1579325199127197,
"step": 3530,
"unclipped_grad_norm": 0.7720932364463806
},
{
"bits_per_byte": 1.6828739722984016,
"learning_rate": 0.00020137583547582478,
"loss_nats": 1.1664793491363525,
"step": 3540,
"unclipped_grad_norm": 0.6750161647796631
},
{
"bits_per_byte": 1.7101762181228726,
"learning_rate": 0.00020083811993958973,
"loss_nats": 1.185403823852539,
"step": 3550,
"unclipped_grad_norm": 0.6993052363395691
},
{
"bits_per_byte": 1.6591575647444203,
"learning_rate": 0.00020029979202876563,
"loss_nats": 1.1500403881072998,
"step": 3560,
"unclipped_grad_norm": 0.6886851787567139
},
{
"bits_per_byte": 1.7182958630354075,
"learning_rate": 0.00019976086094189118,
"loss_nats": 1.1910319328308105,
"step": 3570,
"unclipped_grad_norm": 0.749891459941864
},
{
"bits_per_byte": 1.7607525321233055,
"learning_rate": 0.00019922133588781175,
"loss_nats": 1.2204606533050537,
"step": 3580,
"unclipped_grad_norm": 0.8161401152610779
},
{
"bits_per_byte": 1.690903842267186,
"learning_rate": 0.0001986812260855219,
"loss_nats": 1.1720452308654785,
"step": 3590,
"unclipped_grad_norm": 0.6925192475318909
},
{
"bits_per_byte": 1.6772946831217612,
"learning_rate": 0.00019814054076400787,
"loss_nats": 1.1626120805740356,
"step": 3600,
"unclipped_grad_norm": 0.7061766982078552
},
{
"bits_per_byte": 1.6673109182565242,
"learning_rate": 0.00019759928916209004,
"loss_nats": 1.1556918621063232,
"step": 3610,
"unclipped_grad_norm": 0.7162109017372131
},
{
"bits_per_byte": 1.6598764522250269,
"learning_rate": 0.00019705748052826493,
"loss_nats": 1.150538682937622,
"step": 3620,
"unclipped_grad_norm": 0.6953511238098145
},
{
"bits_per_byte": 1.6688818077895051,
"learning_rate": 0.00019651512412054723,
"loss_nats": 1.15678071975708,
"step": 3630,
"unclipped_grad_norm": 0.7265485525131226
},
{
"bits_per_byte": 1.6378186453559664,
"learning_rate": 0.00019597222920631144,
"loss_nats": 1.135249376296997,
"step": 3640,
"unclipped_grad_norm": 0.68751460313797
},
{
"bits_per_byte": 1.6996694540163617,
"learning_rate": 0.0001954288050621338,
"loss_nats": 1.1781210899353027,
"step": 3650,
"unclipped_grad_norm": 0.7145417928695679
},
{
"bits_per_byte": 1.6073453834148872,
"learning_rate": 0.00019488486097363363,
"loss_nats": 1.1141269207000732,
"step": 3660,
"unclipped_grad_norm": 0.71860671043396
},
{
"bits_per_byte": 1.62879970316207,
"learning_rate": 0.00019434040623531452,
"loss_nats": 1.1289979219436646,
"step": 3670,
"unclipped_grad_norm": 0.6769548654556274
},
{
"bits_per_byte": 1.702227523965295,
"learning_rate": 0.0001937954501504058,
"loss_nats": 1.179894208908081,
"step": 3680,
"unclipped_grad_norm": 0.7466121315956116
},
{
"bits_per_byte": 1.653425382991163,
"learning_rate": 0.00019325000203070333,
"loss_nats": 1.1460671424865723,
"step": 3690,
"unclipped_grad_norm": 0.6637522578239441
},
{
"bits_per_byte": 1.5989233930021354,
"learning_rate": 0.0001927040711964106,
"loss_nats": 1.1082892417907715,
"step": 3700,
"unclipped_grad_norm": 0.7335078120231628
},
{
"bits_per_byte": 1.6701862961962997,
"learning_rate": 0.00019215766697597923,
"loss_nats": 1.1576849222183228,
"step": 3710,
"unclipped_grad_norm": 0.7247957587242126
},
{
"bits_per_byte": 1.6538059805975223,
"learning_rate": 0.00019161079870594978,
"loss_nats": 1.1463309526443481,
"step": 3720,
"unclipped_grad_norm": 0.7534144520759583
},
{
"bits_per_byte": 1.7119505631318241,
"learning_rate": 0.0001910634757307922,
"loss_nats": 1.1866337060928345,
"step": 3730,
"unclipped_grad_norm": 0.6951875686645508
},
{
"bits_per_byte": 1.6437530787066386,
"learning_rate": 0.0001905157074027459,
"loss_nats": 1.1393628120422363,
"step": 3740,
"unclipped_grad_norm": 0.7089563608169556
},
{
"bits_per_byte": 1.6267918057132467,
"learning_rate": 0.00018996750308166037,
"loss_nats": 1.1276061534881592,
"step": 3750,
"unclipped_grad_norm": 0.6922760009765625
},
{
"bits_per_byte": 1.6201740852906965,
"learning_rate": 0.00018941887213483482,
"loss_nats": 1.1230190992355347,
"step": 3760,
"unclipped_grad_norm": 0.6971311569213867
},
{
"bits_per_byte": 1.6730486034546093,
"learning_rate": 0.00018886982393685845,
"loss_nats": 1.1596689224243164,
"step": 3770,
"unclipped_grad_norm": 0.6925700902938843
},
{
"bits_per_byte": 1.6258194158052683,
"learning_rate": 0.00018832036786945008,
"loss_nats": 1.126932144165039,
"step": 3780,
"unclipped_grad_norm": 0.71840900182724
},
{
"bits_per_byte": 1.6611288298886098,
"learning_rate": 0.0001877705133212979,
"loss_nats": 1.1514067649841309,
"step": 3790,
"unclipped_grad_norm": 0.6731990575790405
},
{
"bits_per_byte": 1.622545554063444,
"learning_rate": 0.00018722026968789906,
"loss_nats": 1.1246628761291504,
"step": 3800,
"unclipped_grad_norm": 0.7068047523498535
},
{
"bits_per_byte": 1.613386446009094,
"learning_rate": 0.00018666964637139907,
"loss_nats": 1.118314266204834,
"step": 3810,
"unclipped_grad_norm": 0.679490327835083
},
{
"bits_per_byte": 1.723293162921528,
"learning_rate": 0.00018611865278043115,
"loss_nats": 1.1944957971572876,
"step": 3820,
"unclipped_grad_norm": 0.7167435884475708
},
{
"bits_per_byte": 1.6410942269242994,
"learning_rate": 0.00018556729832995555,
"loss_nats": 1.1375198364257812,
"step": 3830,
"unclipped_grad_norm": 0.749890148639679
},
{
"bits_per_byte": 1.6271811744348,
"learning_rate": 0.00018501559244109864,
"loss_nats": 1.1278760433197021,
"step": 3840,
"unclipped_grad_norm": 0.6869876980781555
},
{
"bits_per_byte": 1.6520392028252087,
"learning_rate": 0.00018446354454099178,
"loss_nats": 1.145106315612793,
"step": 3850,
"unclipped_grad_norm": 0.6890377998352051
},
{
"bits_per_byte": 1.6079827511189846,
"learning_rate": 0.00018391116406261048,
"loss_nats": 1.1145687103271484,
"step": 3860,
"unclipped_grad_norm": 0.717820942401886
},
{
"bits_per_byte": 1.6751354409401786,
"learning_rate": 0.00018335846044461305,
"loss_nats": 1.1611154079437256,
"step": 3870,
"unclipped_grad_norm": 0.7368612289428711
},
{
"bits_per_byte": 1.6338718155013159,
"learning_rate": 0.0001828054431311793,
"loss_nats": 1.1325136423110962,
"step": 3880,
"unclipped_grad_norm": 0.7661339640617371
},
{
"bits_per_byte": 1.656585564200767,
"learning_rate": 0.00018225212157184934,
"loss_nats": 1.1482576131820679,
"step": 3890,
"unclipped_grad_norm": 0.7343658804893494
},
{
"bits_per_byte": 1.6647341021986468,
"learning_rate": 0.00018169850522136192,
"loss_nats": 1.1539057493209839,
"step": 3900,
"unclipped_grad_norm": 0.7533658146858215
},
{
"bits_per_byte": 1.6762937440937398,
"learning_rate": 0.000181144603539493,
"loss_nats": 1.16191828250885,
"step": 3910,
"unclipped_grad_norm": 0.7147406935691833
},
{
"bits_per_byte": 1.6322013480134854,
"learning_rate": 0.00018059042599089412,
"loss_nats": 1.1313557624816895,
"step": 3920,
"unclipped_grad_norm": 0.6887187361717224
},
{
"bits_per_byte": 1.6221845624792828,
"learning_rate": 0.00018003598204493048,
"loss_nats": 1.1244126558303833,
"step": 3930,
"unclipped_grad_norm": 0.7910217642784119
},
{
"bits_per_byte": 1.63472175376188,
"learning_rate": 0.00017948128117551936,
"loss_nats": 1.1331027746200562,
"step": 3940,
"unclipped_grad_norm": 0.730076789855957
},
{
"bits_per_byte": 1.6972579132859635,
"learning_rate": 0.00017892633286096816,
"loss_nats": 1.1764495372772217,
"step": 3950,
"unclipped_grad_norm": 0.7732967138290405
},
{
"bits_per_byte": 1.7135035664691152,
"learning_rate": 0.00017837114658381247,
"loss_nats": 1.187710165977478,
"step": 3960,
"unclipped_grad_norm": 0.7405394911766052
},
{
"bits_per_byte": 1.685746770498414,
"learning_rate": 0.00017781573183065386,
"loss_nats": 1.1684706211090088,
"step": 3970,
"unclipped_grad_norm": 0.7160924673080444
},
{
"bits_per_byte": 1.6665311489175123,
"learning_rate": 0.0001772600980919981,
"loss_nats": 1.1551513671875,
"step": 3980,
"unclipped_grad_norm": 0.7374541759490967
},
{
"bits_per_byte": 1.7040839046987084,
"learning_rate": 0.0001767042548620927,
"loss_nats": 1.1811809539794922,
"step": 3990,
"unclipped_grad_norm": 0.7841057181358337
},
{
"bits_per_byte": 1.693502844087025,
"learning_rate": 0.00017614821163876486,
"loss_nats": 1.17384672164917,
"step": 4000,
"unclipped_grad_norm": 0.7609017491340637
},
{
"bits_per_byte": 1.699465998540391,
"learning_rate": 0.00017559197792325906,
"loss_nats": 1.1779800653457642,
"step": 4010,
"unclipped_grad_norm": 0.7629905343055725
},
{
"bits_per_byte": 1.5838037102341824,
"learning_rate": 0.0001750355632200748,
"loss_nats": 1.097809076309204,
"step": 4020,
"unclipped_grad_norm": 0.6868616342544556
},
{
"bits_per_byte": 1.617556165379263,
"learning_rate": 0.00017447897703680418,
"loss_nats": 1.1212044954299927,
"step": 4030,
"unclipped_grad_norm": 0.749467670917511
},
{
"bits_per_byte": 1.6189757101794846,
"learning_rate": 0.0001739222288839694,
"loss_nats": 1.1221884489059448,
"step": 4040,
"unclipped_grad_norm": 0.7323976755142212
},
{
"bits_per_byte": 1.6111234982890412,
"learning_rate": 0.00017336532827486026,
"loss_nats": 1.1167457103729248,
"step": 4050,
"unclipped_grad_norm": 0.7304336428642273
},
{
"bits_per_byte": 1.6428754512392856,
"learning_rate": 0.00017280828472537163,
"loss_nats": 1.1387544870376587,
"step": 4060,
"unclipped_grad_norm": 0.745829164981842
},
{
"bits_per_byte": 1.6097484969953932,
"learning_rate": 0.0001722511077538408,
"loss_nats": 1.1157926321029663,
"step": 4070,
"unclipped_grad_norm": 0.7523763179779053
},
{
"bits_per_byte": 1.6445053308215125,
"learning_rate": 0.00017169380688088497,
"loss_nats": 1.1398842334747314,
"step": 4080,
"unclipped_grad_norm": 0.7511109113693237
},
{
"bits_per_byte": 1.6034616711931031,
"learning_rate": 0.0001711363916292384,
"loss_nats": 1.1114349365234375,
"step": 4090,
"unclipped_grad_norm": 0.7438745498657227
},
{
"bits_per_byte": 1.561935084281071,
"learning_rate": 0.00017057887152358974,
"loss_nats": 1.082650899887085,
"step": 4100,
"unclipped_grad_norm": 0.6872318387031555
},
{
"bits_per_byte": 1.6907129415247284,
"learning_rate": 0.0001700212560904194,
"loss_nats": 1.1719129085540771,
"step": 4110,
"unclipped_grad_norm": 0.7602059841156006
},
{
"bits_per_byte": 1.6125045189754696,
"learning_rate": 0.00016946355485783656,
"loss_nats": 1.1177029609680176,
"step": 4120,
"unclipped_grad_norm": 0.7136654257774353
},
{
"bits_per_byte": 1.5891117827704122,
"learning_rate": 0.0001689057773554166,
"loss_nats": 1.1014883518218994,
"step": 4130,
"unclipped_grad_norm": 0.7062129378318787
},
{
"bits_per_byte": 1.6146775197691212,
"learning_rate": 0.00016834793311403806,
"loss_nats": 1.1192091703414917,
"step": 4140,
"unclipped_grad_norm": 0.7150430679321289
},
{
"bits_per_byte": 1.6618133208390438,
"learning_rate": 0.00016779003166571981,
"loss_nats": 1.151881217956543,
"step": 4150,
"unclipped_grad_norm": 0.7448095679283142
},
{
"bits_per_byte": 1.6228898633304711,
"learning_rate": 0.00016723208254345838,
"loss_nats": 1.124901533126831,
"step": 4160,
"unclipped_grad_norm": 0.7362097501754761
},
{
"bits_per_byte": 1.5905947891688021,
"learning_rate": 0.00016667409528106481,
"loss_nats": 1.1025162935256958,
"step": 4170,
"unclipped_grad_norm": 0.7651577591896057
},
{
"bits_per_byte": 1.5608787668394717,
"learning_rate": 0.00016611607941300184,
"loss_nats": 1.081918716430664,
"step": 4180,
"unclipped_grad_norm": 0.7135135531425476
},
{
"bits_per_byte": 1.713072749928704,
"learning_rate": 0.0001655580444742211,
"loss_nats": 1.1874115467071533,
"step": 4190,
"unclipped_grad_norm": 0.7463747262954712
},
{
"bits_per_byte": 1.5972510337051453,
"learning_rate": 0.00016499999999999997,
"loss_nats": 1.1071300506591797,
"step": 4200,
"unclipped_grad_norm": 0.7313898801803589
},
{
"bits_per_byte": 1.6169509584308768,
"learning_rate": 0.0001644419555257789,
"loss_nats": 1.1207849979400635,
"step": 4210,
"unclipped_grad_norm": 0.7539675831794739
},
{
"bits_per_byte": 1.6536405332873922,
"learning_rate": 0.0001638839205869981,
"loss_nats": 1.1462162733078003,
"step": 4220,
"unclipped_grad_norm": 0.7351350784301758
},
{
"bits_per_byte": 1.6357479742395782,
"learning_rate": 0.0001633259047189352,
"loss_nats": 1.1338140964508057,
"step": 4230,
"unclipped_grad_norm": 0.7141795754432678
},
{
"bits_per_byte": 1.6650933739562992,
"learning_rate": 0.0001627679174565416,
"loss_nats": 1.1541547775268555,
"step": 4240,
"unclipped_grad_norm": 0.7625938653945923
},
{
"bits_per_byte": 1.6824343846427963,
"learning_rate": 0.00016220996833428016,
"loss_nats": 1.1661746501922607,
"step": 4250,
"unclipped_grad_norm": 0.702212393283844
},
{
"bits_per_byte": 1.6672134040934852,
"learning_rate": 0.00016165206688596191,
"loss_nats": 1.155624270439148,
"step": 4260,
"unclipped_grad_norm": 0.775193989276886
},
{
"bits_per_byte": 1.643573356836487,
"learning_rate": 0.00016109422264458335,
"loss_nats": 1.1392382383346558,
"step": 4270,
"unclipped_grad_norm": 0.7224892973899841
},
{
"bits_per_byte": 1.6848497089915135,
"learning_rate": 0.00016053644514216336,
"loss_nats": 1.167848825454712,
"step": 4280,
"unclipped_grad_norm": 0.792568564414978
},
{
"bits_per_byte": 1.6287462165576516,
"learning_rate": 0.00015997874390958058,
"loss_nats": 1.1289608478546143,
"step": 4290,
"unclipped_grad_norm": 0.7352984547615051
},
{
"bits_per_byte": 1.5679441581022175,
"learning_rate": 0.00015942112847641018,
"loss_nats": 1.0868160724639893,
"step": 4300,
"unclipped_grad_norm": 0.7459802031517029
},
{
"bits_per_byte": 1.7134996108681453,
"learning_rate": 0.00015886360837076156,
"loss_nats": 1.1877074241638184,
"step": 4310,
"unclipped_grad_norm": 0.8436940312385559
},
{
"bits_per_byte": 1.585466610485375,
"learning_rate": 0.00015830619311911495,
"loss_nats": 1.0989617109298706,
"step": 4320,
"unclipped_grad_norm": 0.7326236367225647
},
{
"bits_per_byte": 1.5870087789156617,
"learning_rate": 0.00015774889224615915,
"loss_nats": 1.1000306606292725,
"step": 4330,
"unclipped_grad_norm": 0.7154185175895691
},
{
"bits_per_byte": 1.5845594020020737,
"learning_rate": 0.00015719171527462834,
"loss_nats": 1.0983328819274902,
"step": 4340,
"unclipped_grad_norm": 0.7299889326095581
},
{
"bits_per_byte": 1.5087386145947432,
"learning_rate": 0.0001566346717251397,
"loss_nats": 1.0457779169082642,
"step": 4350,
"unclipped_grad_norm": 0.7116599082946777
},
{
"bits_per_byte": 1.6642632137005844,
"learning_rate": 0.00015607777111603056,
"loss_nats": 1.1535793542861938,
"step": 4360,
"unclipped_grad_norm": 0.7841640114784241
},
{
"bits_per_byte": 1.6118272512963718,
"learning_rate": 0.00015552102296319576,
"loss_nats": 1.1172335147857666,
"step": 4370,
"unclipped_grad_norm": 0.7682546973228455
},
{
"bits_per_byte": 1.5975416843851036,
"learning_rate": 0.00015496443677992516,
"loss_nats": 1.1073315143585205,
"step": 4380,
"unclipped_grad_norm": 0.7542603611946106
},
{
"bits_per_byte": 1.6190350441940322,
"learning_rate": 0.0001544080220767409,
"loss_nats": 1.1222295761108398,
"step": 4390,
"unclipped_grad_norm": 0.7354364991188049
},
{
"bits_per_byte": 1.5791293937663822,
"learning_rate": 0.00015385178836123511,
"loss_nats": 1.0945690870285034,
"step": 4400,
"unclipped_grad_norm": 0.7447035908699036
},
{
"bits_per_byte": 1.5777399459300616,
"learning_rate": 0.00015329574513790726,
"loss_nats": 1.0936059951782227,
"step": 4410,
"unclipped_grad_norm": 0.7433028221130371
},
{
"bits_per_byte": 1.5774630538621723,
"learning_rate": 0.00015273990190800186,
"loss_nats": 1.093414068222046,
"step": 4420,
"unclipped_grad_norm": 0.7514091730117798
},
{
"bits_per_byte": 1.5840658117940973,
"learning_rate": 0.0001521842681693461,
"loss_nats": 1.0979907512664795,
"step": 4430,
"unclipped_grad_norm": 0.748140811920166
},
{
"bits_per_byte": 1.5992336497042918,
"learning_rate": 0.00015162885341618752,
"loss_nats": 1.108504295349121,
"step": 4440,
"unclipped_grad_norm": 0.7693572044372559
},
{
"bits_per_byte": 1.5825853851354703,
"learning_rate": 0.00015107366713903178,
"loss_nats": 1.0969645977020264,
"step": 4450,
"unclipped_grad_norm": 0.7571823596954346
},
{
"bits_per_byte": 1.5385392523882981,
"learning_rate": 0.0001505187188244806,
"loss_nats": 1.0664341449737549,
"step": 4460,
"unclipped_grad_norm": 0.7259396314620972
},
{
"bits_per_byte": 1.6428487939284018,
"learning_rate": 0.00014996401795506952,
"loss_nats": 1.1387360095977783,
"step": 4470,
"unclipped_grad_norm": 0.7504101991653442
},
{
"bits_per_byte": 1.5687149843433847,
"learning_rate": 0.00014940957400910585,
"loss_nats": 1.0873503684997559,
"step": 4480,
"unclipped_grad_norm": 0.7544133067131042
},
{
"bits_per_byte": 1.5307804270772734,
"learning_rate": 0.00014885539646050699,
"loss_nats": 1.061056137084961,
"step": 4490,
"unclipped_grad_norm": 0.7395938634872437
},
{
"bits_per_byte": 1.6123227333135077,
"learning_rate": 0.00014830149477863805,
"loss_nats": 1.1175769567489624,
"step": 4500,
"unclipped_grad_norm": 0.7624281644821167
},
{
"bits_per_byte": 1.5895855949735391,
"learning_rate": 0.00014774787842815066,
"loss_nats": 1.1018167734146118,
"step": 4510,
"unclipped_grad_norm": 0.7442159652709961
},
{
"bits_per_byte": 1.5720557473363947,
"learning_rate": 0.00014719455686882066,
"loss_nats": 1.0896660089492798,
"step": 4520,
"unclipped_grad_norm": 0.7299280166625977
},
{
"bits_per_byte": 1.5930625682260326,
"learning_rate": 0.00014664153955538695,
"loss_nats": 1.10422682762146,
"step": 4530,
"unclipped_grad_norm": 0.7508688569068909
},
{
"bits_per_byte": 1.551802554422838,
"learning_rate": 0.00014608883593738946,
"loss_nats": 1.0756275653839111,
"step": 4540,
"unclipped_grad_norm": 0.7326505780220032
},
{
"bits_per_byte": 1.6245931795046165,
"learning_rate": 0.00014553645545900817,
"loss_nats": 1.126082181930542,
"step": 4550,
"unclipped_grad_norm": 0.7591264247894287
},
{
"bits_per_byte": 1.6222865481912445,
"learning_rate": 0.0001449844075589013,
"loss_nats": 1.124483346939087,
"step": 4560,
"unclipped_grad_norm": 0.7490679621696472
},
{
"bits_per_byte": 1.5636793323261218,
"learning_rate": 0.0001444327016700444,
"loss_nats": 1.083859920501709,
"step": 4570,
"unclipped_grad_norm": 0.7205959558486938
},
{
"bits_per_byte": 1.5242402708302625,
"learning_rate": 0.0001438813472195688,
"loss_nats": 1.0565228462219238,
"step": 4580,
"unclipped_grad_norm": 0.7278034687042236
},
{
"bits_per_byte": 1.649223158899981,
"learning_rate": 0.00014333035362860093,
"loss_nats": 1.1431543827056885,
"step": 4590,
"unclipped_grad_norm": 0.80039381980896
},
{
"bits_per_byte": 1.621592598195013,
"learning_rate": 0.00014277973031210088,
"loss_nats": 1.1240023374557495,
"step": 4600,
"unclipped_grad_norm": 0.7594562768936157
},
{
"bits_per_byte": 1.591440943811047,
"learning_rate": 0.00014222948667870207,
"loss_nats": 1.1031028032302856,
"step": 4610,
"unclipped_grad_norm": 0.7573962211608887
},
{
"bits_per_byte": 1.5204752266375738,
"learning_rate": 0.00014167963213054991,
"loss_nats": 1.0539131164550781,
"step": 4620,
"unclipped_grad_norm": 0.7093440890312195
},
{
"bits_per_byte": 1.6007937043302685,
"learning_rate": 0.00014113017606314152,
"loss_nats": 1.1095856428146362,
"step": 4630,
"unclipped_grad_norm": 0.7716385126113892
},
{
"bits_per_byte": 1.566827990698118,
"learning_rate": 0.00014058112786516518,
"loss_nats": 1.0860424041748047,
"step": 4640,
"unclipped_grad_norm": 0.7477832436561584
},
{
"bits_per_byte": 1.5540792607549598,
"learning_rate": 0.0001400324969183396,
"loss_nats": 1.0772056579589844,
"step": 4650,
"unclipped_grad_norm": 0.7563350200653076
},
{
"bits_per_byte": 1.5711925664117143,
"learning_rate": 0.00013948429259725406,
"loss_nats": 1.0890676975250244,
"step": 4660,
"unclipped_grad_norm": 0.7477205991744995
},
{
"bits_per_byte": 1.6065126434194095,
"learning_rate": 0.00013893652426920776,
"loss_nats": 1.1135497093200684,
"step": 4670,
"unclipped_grad_norm": 0.7434800267219543
},
{
"bits_per_byte": 1.5613855997115644,
"learning_rate": 0.00013838920129405016,
"loss_nats": 1.0822700262069702,
"step": 4680,
"unclipped_grad_norm": 0.7484313249588013
},
{
"bits_per_byte": 1.533432227570927,
"learning_rate": 0.0001378423330240207,
"loss_nats": 1.0628942251205444,
"step": 4690,
"unclipped_grad_norm": 0.7767173647880554
},
{
"bits_per_byte": 1.6430371149310967,
"learning_rate": 0.00013729592880358936,
"loss_nats": 1.1388665437698364,
"step": 4700,
"unclipped_grad_norm": 0.7674224972724915
},
{
"bits_per_byte": 1.552380416129737,
"learning_rate": 0.00013674999796929659,
"loss_nats": 1.0760281085968018,
"step": 4710,
"unclipped_grad_norm": 0.7836162447929382
},
{
"bits_per_byte": 1.5203768525612804,
"learning_rate": 0.00013620454984959417,
"loss_nats": 1.053844928741455,
"step": 4720,
"unclipped_grad_norm": 0.7613595128059387
},
{
"bits_per_byte": 1.6524877335786587,
"learning_rate": 0.00013565959376468542,
"loss_nats": 1.1454172134399414,
"step": 4730,
"unclipped_grad_norm": 0.7924392223358154
},
{
"bits_per_byte": 1.587415861850254,
"learning_rate": 0.00013511513902636634,
"loss_nats": 1.1003128290176392,
"step": 4740,
"unclipped_grad_norm": 0.7647171020507812
},
{
"bits_per_byte": 1.6396726183322676,
"learning_rate": 0.0001345711949378662,
"loss_nats": 1.1365344524383545,
"step": 4750,
"unclipped_grad_norm": 0.8181672096252441
},
{
"bits_per_byte": 1.5091890371573529,
"learning_rate": 0.00013402777079368853,
"loss_nats": 1.0460901260375977,
"step": 4760,
"unclipped_grad_norm": 0.7407304048538208
},
{
"bits_per_byte": 1.6098291568586478,
"learning_rate": 0.0001334848758794528,
"loss_nats": 1.1158485412597656,
"step": 4770,
"unclipped_grad_norm": 0.7601819038391113
},
{
"bits_per_byte": 1.5582154435082103,
"learning_rate": 0.00013294251947173504,
"loss_nats": 1.0800726413726807,
"step": 4780,
"unclipped_grad_norm": 0.7307054996490479
},
{
"bits_per_byte": 1.5944504682184955,
"learning_rate": 0.00013240071083790996,
"loss_nats": 1.1051888465881348,
"step": 4790,
"unclipped_grad_norm": 0.7617009282112122
},
{
"bits_per_byte": 1.5823225956449518,
"learning_rate": 0.0001318594592359921,
"loss_nats": 1.0967824459075928,
"step": 4800,
"unclipped_grad_norm": 0.7428550124168396
},
{
"bits_per_byte": 1.542078311377753,
"learning_rate": 0.0001313187739144781,
"loss_nats": 1.0688872337341309,
"step": 4810,
"unclipped_grad_norm": 0.773262083530426
},
{
"bits_per_byte": 1.6108488419956133,
"learning_rate": 0.0001307786641121882,
"loss_nats": 1.1165553331375122,
"step": 4820,
"unclipped_grad_norm": 0.8227523565292358
},
{
"bits_per_byte": 1.5982464692883394,
"learning_rate": 0.00013023913905810876,
"loss_nats": 1.1078200340270996,
"step": 4830,
"unclipped_grad_norm": 0.783271312713623
},
{
"bits_per_byte": 1.6134836162068316,
"learning_rate": 0.00012970020797123432,
"loss_nats": 1.1183816194534302,
"step": 4840,
"unclipped_grad_norm": 0.7595719695091248
},
{
"bits_per_byte": 1.5430993723759254,
"learning_rate": 0.00012916188006041027,
"loss_nats": 1.0695949792861938,
"step": 4850,
"unclipped_grad_norm": 0.8344528079032898
},
{
"bits_per_byte": 1.551674427347945,
"learning_rate": 0.00012862416452417516,
"loss_nats": 1.0755387544631958,
"step": 4860,
"unclipped_grad_norm": 0.7924710512161255
},
{
"bits_per_byte": 1.5551434893984988,
"learning_rate": 0.00012808707055060405,
"loss_nats": 1.0779433250427246,
"step": 4870,
"unclipped_grad_norm": 0.7731344103813171
},
{
"bits_per_byte": 1.7162552888829201,
"learning_rate": 0.000127550607317151,
"loss_nats": 1.1896175146102905,
"step": 4880,
"unclipped_grad_norm": 0.8454892039299011
},
{
"bits_per_byte": 1.5015098398136777,
"learning_rate": 0.00012701478399049248,
"loss_nats": 1.0407673120498657,
"step": 4890,
"unclipped_grad_norm": 0.7279435396194458
},
{
"bits_per_byte": 1.6017640304464364,
"learning_rate": 0.0001264796097263708,
"loss_nats": 1.1102582216262817,
"step": 4900,
"unclipped_grad_norm": 0.7495299577713013
},
{
"bits_per_byte": 1.5795956387328711,
"learning_rate": 0.0001259450936694375,
"loss_nats": 1.0948922634124756,
"step": 4910,
"unclipped_grad_norm": 0.8354763388633728
},
{
"bits_per_byte": 1.5130457481029513,
"learning_rate": 0.00012541124495309727,
"loss_nats": 1.048763394355774,
"step": 4920,
"unclipped_grad_norm": 0.7128972411155701
},
{
"bits_per_byte": 1.5263470583033318,
"learning_rate": 0.00012487807269935167,
"loss_nats": 1.057983160018921,
"step": 4930,
"unclipped_grad_norm": 0.7301722168922424
},
{
"bits_per_byte": 1.56106777577277,
"learning_rate": 0.00012434558601864349,
"loss_nats": 1.0820497274398804,
"step": 4940,
"unclipped_grad_norm": 0.7844029664993286
},
{
"bits_per_byte": 1.5809642766684375,
"learning_rate": 0.0001238137940097009,
"loss_nats": 1.0958409309387207,
"step": 4950,
"unclipped_grad_norm": 0.7815173864364624
},
{
"bits_per_byte": 1.6004612618661507,
"learning_rate": 0.0001232827057593821,
"loss_nats": 1.1093552112579346,
"step": 4960,
"unclipped_grad_norm": 0.8055645227432251
},
{
"bits_per_byte": 1.5618184800437862,
"learning_rate": 0.00012275233034251992,
"loss_nats": 1.0825700759887695,
"step": 4970,
"unclipped_grad_norm": 0.7773133516311646
},
{
"bits_per_byte": 1.606830467358204,
"learning_rate": 0.0001222226768217669,
"loss_nats": 1.1137700080871582,
"step": 4980,
"unclipped_grad_norm": 0.7920810580253601
},
{
"bits_per_byte": 1.567882244347907,
"learning_rate": 0.00012169375424744034,
"loss_nats": 1.086773157119751,
"step": 4990,
"unclipped_grad_norm": 0.7929123640060425
},
{
"bits_per_byte": 1.5708692390280923,
"learning_rate": 0.00012116557165736772,
"loss_nats": 1.088843584060669,
"step": 5000,
"unclipped_grad_norm": 0.762309730052948
},
{
"bits_per_byte": 1.6157285057485438,
"learning_rate": 0.00012063813807673214,
"loss_nats": 1.1199376583099365,
"step": 5010,
"unclipped_grad_norm": 0.7734562158584595
},
{
"bits_per_byte": 1.5685849654593325,
"learning_rate": 0.00012011146251791832,
"loss_nats": 1.0872602462768555,
"step": 5020,
"unclipped_grad_norm": 0.8221466541290283
},
{
"bits_per_byte": 1.5125002191344146,
"learning_rate": 0.0001195855539803584,
"loss_nats": 1.0483852624893188,
"step": 5030,
"unclipped_grad_norm": 0.7515868544578552
},
{
"bits_per_byte": 1.5645020973278494,
"learning_rate": 0.00011906042145037817,
"loss_nats": 1.08443021774292,
"step": 5040,
"unclipped_grad_norm": 0.7403401732444763
},
{
"bits_per_byte": 1.5465288784167808,
"learning_rate": 0.0001185360739010438,
"loss_nats": 1.071972131729126,
"step": 5050,
"unclipped_grad_norm": 0.8082988262176514
},
{
"bits_per_byte": 1.6139199361920706,
"learning_rate": 0.00011801252029200808,
"loss_nats": 1.1186840534210205,
"step": 5060,
"unclipped_grad_norm": 0.8066977262496948
},
{
"bits_per_byte": 1.5533101543403012,
"learning_rate": 0.00011748976956935777,
"loss_nats": 1.0766725540161133,
"step": 5070,
"unclipped_grad_norm": 0.7855141758918762
},
{
"bits_per_byte": 1.6163216739113697,
"learning_rate": 0.00011696783066546024,
"loss_nats": 1.1203488111495972,
"step": 5080,
"unclipped_grad_norm": 0.776370644569397
},
{
"bits_per_byte": 1.5614495772576855,
"learning_rate": 0.00011644671249881147,
"loss_nats": 1.082314372062683,
"step": 5090,
"unclipped_grad_norm": 0.776827871799469
},
{
"bits_per_byte": 1.5419546558517827,
"learning_rate": 0.00011592642397388293,
"loss_nats": 1.0688015222549438,
"step": 5100,
"unclipped_grad_norm": 0.7785685062408447
},
{
"bits_per_byte": 1.5800214677764075,
"learning_rate": 0.00011540697398097015,
"loss_nats": 1.0951874256134033,
"step": 5110,
"unclipped_grad_norm": 0.8104537725448608
},
{
"bits_per_byte": 1.487382496941203,
"learning_rate": 0.00011488837139604016,
"loss_nats": 1.0309749841690063,
"step": 5120,
"unclipped_grad_norm": 0.7389267086982727
},
{
"bits_per_byte": 1.6243671943013827,
"learning_rate": 0.00011437062508058039,
"loss_nats": 1.1259255409240723,
"step": 5130,
"unclipped_grad_norm": 0.8053697943687439
},
{
"bits_per_byte": 1.5876562935961602,
"learning_rate": 0.00011385374388144673,
"loss_nats": 1.1004794836044312,
"step": 5140,
"unclipped_grad_norm": 0.8099423050880432
},
{
"bits_per_byte": 1.5188835272038386,
"learning_rate": 0.00011333773663071286,
"loss_nats": 1.0528098344802856,
"step": 5150,
"unclipped_grad_norm": 0.7463088035583496
},
{
"bits_per_byte": 1.5574305146896725,
"learning_rate": 0.00011282261214551897,
"loss_nats": 1.079528570175171,
"step": 5160,
"unclipped_grad_norm": 0.7795574069023132
},
{
"bits_per_byte": 1.604379370618107,
"learning_rate": 0.00011230837922792112,
"loss_nats": 1.1120710372924805,
"step": 5170,
"unclipped_grad_norm": 0.8112488985061646
},
{
"bits_per_byte": 1.570943535533265,
"learning_rate": 0.0001117950466647412,
"loss_nats": 1.0888950824737549,
"step": 5180,
"unclipped_grad_norm": 0.7960402965545654
},
{
"bits_per_byte": 1.4686855750353036,
"learning_rate": 0.00011128262322741627,
"loss_nats": 1.0180152654647827,
"step": 5190,
"unclipped_grad_norm": 0.7735135555267334
},
{
"bits_per_byte": 1.5401951013508053,
"learning_rate": 0.00011077111767184913,
"loss_nats": 1.0675818920135498,
"step": 5200,
"unclipped_grad_norm": 0.7935121655464172
},
{
"bits_per_byte": 1.522642552003747,
"learning_rate": 0.00011026053873825837,
"loss_nats": 1.055415391921997,
"step": 5210,
"unclipped_grad_norm": 0.7616920471191406
},
{
"bits_per_byte": 1.5296819738862126,
"learning_rate": 0.0001097508951510293,
"loss_nats": 1.0602947473526,
"step": 5220,
"unclipped_grad_norm": 0.799717128276825
},
{
"bits_per_byte": 1.60920967535024,
"learning_rate": 0.0001092421956185646,
"loss_nats": 1.1154191493988037,
"step": 5230,
"unclipped_grad_norm": 0.8263183832168579
},
{
"bits_per_byte": 1.5759482306733728,
"learning_rate": 0.00010873444883313581,
"loss_nats": 1.0923640727996826,
"step": 5240,
"unclipped_grad_norm": 0.9089767336845398
},
{
"bits_per_byte": 1.5751560785834986,
"learning_rate": 0.00010822766347073447,
"loss_nats": 1.0918149948120117,
"step": 5250,
"unclipped_grad_norm": 0.8223497271537781
},
{
"bits_per_byte": 1.518346597367845,
"learning_rate": 0.00010772184819092424,
"loss_nats": 1.052437663078308,
"step": 5260,
"unclipped_grad_norm": 0.7989961504936218
},
{
"bits_per_byte": 1.5652323356634128,
"learning_rate": 0.0001072170116366926,
"loss_nats": 1.0849363803863525,
"step": 5270,
"unclipped_grad_norm": 0.7858845591545105
},
{
"bits_per_byte": 1.4824085866955998,
"learning_rate": 0.00010671316243430345,
"loss_nats": 1.0275273323059082,
"step": 5280,
"unclipped_grad_norm": 0.7963522672653198
},
{
"bits_per_byte": 1.5109090356486319,
"learning_rate": 0.00010621030919314944,
"loss_nats": 1.047282338142395,
"step": 5290,
"unclipped_grad_norm": 0.7786324620246887
},
{
"bits_per_byte": 1.4973765807654917,
"learning_rate": 0.000105708460505605,
"loss_nats": 1.0379023551940918,
"step": 5300,
"unclipped_grad_norm": 0.8200122714042664
},
{
"bits_per_byte": 1.4979238295605373,
"learning_rate": 0.00010520762494687964,
"loss_nats": 1.0382816791534424,
"step": 5310,
"unclipped_grad_norm": 0.798004686832428
},
{
"bits_per_byte": 1.4963071926424267,
"learning_rate": 0.00010470781107487111,
"loss_nats": 1.037161111831665,
"step": 5320,
"unclipped_grad_norm": 0.7881578207015991
},
{
"bits_per_byte": 1.5549873291515153,
"learning_rate": 0.00010420902743001955,
"loss_nats": 1.0778350830078125,
"step": 5330,
"unclipped_grad_norm": 0.8092938661575317
},
{
"bits_per_byte": 1.6144941862633015,
"learning_rate": 0.00010371128253516117,
"loss_nats": 1.1190820932388306,
"step": 5340,
"unclipped_grad_norm": 0.8349001407623291
},
{
"bits_per_byte": 1.4860422361430292,
"learning_rate": 0.000103214584895383,
"loss_nats": 1.030045986175537,
"step": 5350,
"unclipped_grad_norm": 0.7843849062919617
},
{
"bits_per_byte": 1.5082066722556244,
"learning_rate": 0.00010271894299787715,
"loss_nats": 1.0454092025756836,
"step": 5360,
"unclipped_grad_norm": 0.7918040752410889
},
{
"bits_per_byte": 1.577472340925319,
"learning_rate": 0.00010222436531179622,
"loss_nats": 1.0934205055236816,
"step": 5370,
"unclipped_grad_norm": 0.8359649181365967
},
{
"bits_per_byte": 1.5013394050066726,
"learning_rate": 0.0001017308602881082,
"loss_nats": 1.040649175643921,
"step": 5380,
"unclipped_grad_norm": 0.7734270095825195
},
{
"bits_per_byte": 1.5016145772480531,
"learning_rate": 0.00010123843635945242,
"loss_nats": 1.0408399105072021,
"step": 5390,
"unclipped_grad_norm": 0.7885358333587646
},
{
"bits_per_byte": 1.5491196250693782,
"learning_rate": 0.00010074710193999505,
"loss_nats": 1.073767900466919,
"step": 5400,
"unclipped_grad_norm": 0.8202832341194153
},
{
"bits_per_byte": 1.4666546319112648,
"learning_rate": 0.0001002568654252858,
"loss_nats": 1.0166075229644775,
"step": 5410,
"unclipped_grad_norm": 0.8083077073097229
},
{
"bits_per_byte": 1.6009971598062391,
"learning_rate": 9.976773519211404e-05,
"loss_nats": 1.1097266674041748,
"step": 5420,
"unclipped_grad_norm": 0.847724974155426
},
{
"bits_per_byte": 1.5289176829857782,
"learning_rate": 9.92797195983659e-05,
"loss_nats": 1.0597649812698364,
"step": 5430,
"unclipped_grad_norm": 0.8225540518760681
},
{
"bits_per_byte": 1.530357349756151,
"learning_rate": 9.879282698288142e-05,
"loss_nats": 1.060762882232666,
"step": 5440,
"unclipped_grad_norm": 0.8045809864997864
},
{
"bits_per_byte": 1.5424515137301253,
"learning_rate": 9.830706566531197e-05,
"loss_nats": 1.069145917892456,
"step": 5450,
"unclipped_grad_norm": 0.8051854968070984
},
{
"bits_per_byte": 1.5648894022575925,
"learning_rate": 9.782244394597826e-05,
"loss_nats": 1.0846986770629883,
"step": 5460,
"unclipped_grad_norm": 0.7835660576820374
},
{
"bits_per_byte": 1.5450825043230245,
"learning_rate": 9.733897010572829e-05,
"loss_nats": 1.070969581604004,
"step": 5470,
"unclipped_grad_norm": 0.8599129915237427
},
{
"bits_per_byte": 1.4780804713039863,
"learning_rate": 9.685665240579612e-05,
"loss_nats": 1.0245273113250732,
"step": 5480,
"unclipped_grad_norm": 0.8000594973564148
},
{
"bits_per_byte": 1.5424891779306642,
"learning_rate": 9.637549908766041e-05,
"loss_nats": 1.0691720247268677,
"step": 5490,
"unclipped_grad_norm": 0.8055532574653625
},
{
"bits_per_byte": 1.5569413960306187,
"learning_rate": 9.589551837290389e-05,
"loss_nats": 1.0791895389556885,
"step": 5500,
"unclipped_grad_norm": 0.8300883769989014
},
{
"bits_per_byte": 1.5474785666148452,
"learning_rate": 9.541671846307257e-05,
"loss_nats": 1.0726304054260254,
"step": 5510,
"unclipped_grad_norm": 0.826352059841156
},
{
"bits_per_byte": 1.5350409532870977,
"learning_rate": 9.493910753953597e-05,
"loss_nats": 1.0640093088150024,
"step": 5520,
"unclipped_grad_norm": 0.853265643119812
},
{
"bits_per_byte": 1.5635150888945477,
"learning_rate": 9.446269376334689e-05,
"loss_nats": 1.083746075630188,
"step": 5530,
"unclipped_grad_norm": 0.8060055375099182
},
{
"bits_per_byte": 1.5843347926600468,
"learning_rate": 9.39874852751024e-05,
"loss_nats": 1.098177194595337,
"step": 5540,
"unclipped_grad_norm": 0.8706573843955994
},
{
"bits_per_byte": 1.5165710484803372,
"learning_rate": 9.351349019480433e-05,
"loss_nats": 1.0512069463729858,
"step": 5550,
"unclipped_grad_norm": 0.7865509986877441
},
{
"bits_per_byte": 1.5106121935932426,
"learning_rate": 9.304071662172086e-05,
"loss_nats": 1.0470765829086304,
"step": 5560,
"unclipped_grad_norm": 0.8064424991607666
},
{
"bits_per_byte": 1.5654490338035,
"learning_rate": 9.256917263424787e-05,
"loss_nats": 1.0850865840911865,
"step": 5570,
"unclipped_grad_norm": 0.8376733064651489
},
{
"bits_per_byte": 1.4567845475782482,
"learning_rate": 9.209886628977114e-05,
"loss_nats": 1.0097661018371582,
"step": 5580,
"unclipped_grad_norm": 0.7972759008407593
},
{
"bits_per_byte": 1.5257076268074239,
"learning_rate": 9.162980562452843e-05,
"loss_nats": 1.057539939880371,
"step": 5590,
"unclipped_grad_norm": 0.8159511089324951
},
{
"bits_per_byte": 1.5137768463517691,
"learning_rate": 9.116199865347238e-05,
"loss_nats": 1.0492701530456543,
"step": 5600,
"unclipped_grad_norm": 0.7912799715995789
},
{
"bits_per_byte": 1.5312812405565859,
"learning_rate": 9.069545337013333e-05,
"loss_nats": 1.0614032745361328,
"step": 5610,
"unclipped_grad_norm": 0.7786760330200195
},
{
"bits_per_byte": 1.4626550033827992,
"learning_rate": 9.0230177746483e-05,
"loss_nats": 1.0138351917266846,
"step": 5620,
"unclipped_grad_norm": 0.8055174350738525
},
{
"bits_per_byte": 1.4690646247978054,
"learning_rate": 8.976617973279797e-05,
"loss_nats": 1.0182780027389526,
"step": 5630,
"unclipped_grad_norm": 0.7851108312606812
},
{
"bits_per_byte": 1.4663743001903584,
"learning_rate": 8.930346725752419e-05,
"loss_nats": 1.0164132118225098,
"step": 5640,
"unclipped_grad_norm": 0.8267287015914917
},
{
"bits_per_byte": 1.5738650048234715,
"learning_rate": 8.88420482271411e-05,
"loss_nats": 1.090920090675354,
"step": 5650,
"unclipped_grad_norm": 0.8359448313713074
},
{
"bits_per_byte": 1.5300369460775933,
"learning_rate": 8.838193052602691e-05,
"loss_nats": 1.060540795326233,
"step": 5660,
"unclipped_grad_norm": 0.8635727167129517
},
{
"bits_per_byte": 1.5230305448640935,
"learning_rate": 8.792312201632352e-05,
"loss_nats": 1.0556843280792236,
"step": 5670,
"unclipped_grad_norm": 0.8095638751983643
},
{
"bits_per_byte": 1.524155483383387,
"learning_rate": 8.746563053780261e-05,
"loss_nats": 1.0564640760421753,
"step": 5680,
"unclipped_grad_norm": 0.8353701233863831
},
{
"bits_per_byte": 1.566590998605229,
"learning_rate": 8.700946390773114e-05,
"loss_nats": 1.0858781337738037,
"step": 5690,
"unclipped_grad_norm": 0.8297926187515259
},
{
"bits_per_byte": 1.499949785187701,
"learning_rate": 8.655462992073833e-05,
"loss_nats": 1.0396859645843506,
"step": 5700,
"unclipped_grad_norm": 0.8335801959037781
},
{
"bits_per_byte": 1.5306388853556132,
"learning_rate": 8.610113634868199e-05,
"loss_nats": 1.0609580278396606,
"step": 5710,
"unclipped_grad_norm": 0.8218200206756592
},
{
"bits_per_byte": 1.5663932185567369,
"learning_rate": 8.564899094051613e-05,
"loss_nats": 1.0857410430908203,
"step": 5720,
"unclipped_grad_norm": 0.8909743428230286
},
{
"bits_per_byte": 1.4614645394735268,
"learning_rate": 8.519820142215816e-05,
"loss_nats": 1.013010025024414,
"step": 5730,
"unclipped_grad_norm": 0.7809724807739258
},
{
"bits_per_byte": 1.5314872757723195,
"learning_rate": 8.474877549635728e-05,
"loss_nats": 1.0615460872650146,
"step": 5740,
"unclipped_grad_norm": 0.8163876533508301
},
{
"bits_per_byte": 1.499649159513993,
"learning_rate": 8.430072084256247e-05,
"loss_nats": 1.0394775867462158,
"step": 5750,
"unclipped_grad_norm": 0.8667106628417969
},
{
"bits_per_byte": 1.4999243317553734,
"learning_rate": 8.385404511679161e-05,
"loss_nats": 1.039668321609497,
"step": 5760,
"unclipped_grad_norm": 0.8516601324081421
},
{
"bits_per_byte": 1.450708744488567,
"learning_rate": 8.340875595150034e-05,
"loss_nats": 1.0055546760559082,
"step": 5770,
"unclipped_grad_norm": 0.7943668365478516
},
{
"bits_per_byte": 1.5466515020468459,
"learning_rate": 8.296486095545203e-05,
"loss_nats": 1.0720571279525757,
"step": 5780,
"unclipped_grad_norm": 0.8325088024139404
},
{
"bits_per_byte": 1.494320105094358,
"learning_rate": 8.252236771358729e-05,
"loss_nats": 1.0357837677001953,
"step": 5790,
"unclipped_grad_norm": 0.8366720080375671
},
{
"bits_per_byte": 1.4906410522271007,
"learning_rate": 8.208128378689484e-05,
"loss_nats": 1.033233642578125,
"step": 5800,
"unclipped_grad_norm": 0.8333538770675659
},
{
"bits_per_byte": 1.5321304908865463,
"learning_rate": 8.16416167122819e-05,
"loss_nats": 1.0619919300079346,
"step": 5810,
"unclipped_grad_norm": 0.8446563482284546
},
{
"bits_per_byte": 1.51830377368778,
"learning_rate": 8.120337400244578e-05,
"loss_nats": 1.05240797996521,
"step": 5820,
"unclipped_grad_norm": 0.8496891856193542
},
{
"bits_per_byte": 1.4932847695361637,
"learning_rate": 8.076656314574521e-05,
"loss_nats": 1.0350661277770996,
"step": 5830,
"unclipped_grad_norm": 0.8205961585044861
},
{
"bits_per_byte": 1.456290785387621,
"learning_rate": 8.03311916060725e-05,
"loss_nats": 1.009423851966858,
"step": 5840,
"unclipped_grad_norm": 0.8371089100837708
},
{
"bits_per_byte": 1.454681199758196,
"learning_rate": 7.989726682272606e-05,
"loss_nats": 1.0083081722259521,
"step": 5850,
"unclipped_grad_norm": 0.8174164295196533
},
{
"bits_per_byte": 1.4966561454410274,
"learning_rate": 7.946479621028318e-05,
"loss_nats": 1.0374029874801636,
"step": 5860,
"unclipped_grad_norm": 0.9058852791786194
},
{
"bits_per_byte": 1.5418609253070623,
"learning_rate": 7.903378715847343e-05,
"loss_nats": 1.0687365531921387,
"step": 5870,
"unclipped_grad_norm": 0.8718268275260925
},
{
"bits_per_byte": 1.5420466665699941,
"learning_rate": 7.860424703205224e-05,
"loss_nats": 1.0688652992248535,
"step": 5880,
"unclipped_grad_norm": 0.8315351009368896
},
{
"bits_per_byte": 1.525186175410008,
"learning_rate": 7.817618317067534e-05,
"loss_nats": 1.0571784973144531,
"step": 5890,
"unclipped_grad_norm": 0.8407886028289795
},
{
"bits_per_byte": 1.4989159974733648,
"learning_rate": 7.774960288877295e-05,
"loss_nats": 1.0389693975448608,
"step": 5900,
"unclipped_grad_norm": 0.8770707249641418
},
{
"bits_per_byte": 1.4937374278732347,
"learning_rate": 7.732451347542522e-05,
"loss_nats": 1.0353798866271973,
"step": 5910,
"unclipped_grad_norm": 0.8901946544647217
},
{
"bits_per_byte": 1.4996551789067731,
"learning_rate": 7.690092219423722e-05,
"loss_nats": 1.03948175907135,
"step": 5920,
"unclipped_grad_norm": 0.85027015209198
},
{
"bits_per_byte": 1.5573538104273876,
"learning_rate": 7.647883628321536e-05,
"loss_nats": 1.0794754028320312,
"step": 5930,
"unclipped_grad_norm": 0.8851437568664551
},
{
"bits_per_byte": 1.5404470559343193,
"learning_rate": 7.605826295464316e-05,
"loss_nats": 1.0677565336227417,
"step": 5940,
"unclipped_grad_norm": 0.8759228587150574
},
{
"bits_per_byte": 1.5066732749405318,
"learning_rate": 7.563920939495851e-05,
"loss_nats": 1.0443463325500488,
"step": 5950,
"unclipped_grad_norm": 0.8818607926368713
},
{
"bits_per_byte": 1.5044743047665998,
"learning_rate": 7.522168276463047e-05,
"loss_nats": 1.0428221225738525,
"step": 5960,
"unclipped_grad_norm": 0.816109299659729
},
{
"bits_per_byte": 1.5173669841885302,
"learning_rate": 7.480569019803719e-05,
"loss_nats": 1.0517586469650269,
"step": 5970,
"unclipped_grad_norm": 0.8590478301048279
},
{
"bits_per_byte": 1.4670959393933787,
"learning_rate": 7.439123880334388e-05,
"loss_nats": 1.0169134140014648,
"step": 5980,
"unclipped_grad_norm": 0.8544116616249084
},
{
"bits_per_byte": 1.4408131207318973,
"learning_rate": 7.39783356623813e-05,
"loss_nats": 0.9986955523490906,
"step": 5990,
"unclipped_grad_norm": 0.8525132536888123
},
{
"bits_per_byte": 1.5172739415744134,
"learning_rate": 7.356698783052497e-05,
"loss_nats": 1.0516941547393799,
"step": 6000,
"unclipped_grad_norm": 0.860197126865387
},
{
"bits_per_byte": 1.499894750739425,
"learning_rate": 7.315720233657431e-05,
"loss_nats": 1.0396478176116943,
"step": 6010,
"unclipped_grad_norm": 0.8407278060913086
},
{
"bits_per_byte": 1.4776142263374972,
"learning_rate": 7.274898618263286e-05,
"loss_nats": 1.024204134941101,
"step": 6020,
"unclipped_grad_norm": 0.8550556302070618
},
{
"bits_per_byte": 1.5367169242197558,
"learning_rate": 7.234234634398831e-05,
"loss_nats": 1.0651710033416748,
"step": 6030,
"unclipped_grad_norm": 0.8657232522964478
},
{
"bits_per_byte": 1.4931389282482321,
"learning_rate": 7.193728976899362e-05,
"loss_nats": 1.0349650382995605,
"step": 6040,
"unclipped_grad_norm": 0.8972119092941284
},
{
"bits_per_byte": 1.498401597364634,
"learning_rate": 7.153382337894797e-05,
"loss_nats": 1.0386128425598145,
"step": 6050,
"unclipped_grad_norm": 0.8974336981773376
},
{
"bits_per_byte": 1.5272425719663745,
"learning_rate": 7.113195406797887e-05,
"loss_nats": 1.0586038827896118,
"step": 6060,
"unclipped_grad_norm": 0.8948551416397095
},
{
"bits_per_byte": 1.5059438965182226,
"learning_rate": 7.073168870292397e-05,
"loss_nats": 1.043840765953064,
"step": 6070,
"unclipped_grad_norm": 0.843971312046051
},
{
"bits_per_byte": 1.4791374766761891,
"learning_rate": 7.033303412321404e-05,
"loss_nats": 1.0252599716186523,
"step": 6080,
"unclipped_grad_norm": 0.8345775008201599
},
{
"bits_per_byte": 1.4884991802932552,
"learning_rate": 6.993599714075586e-05,
"loss_nats": 1.0317490100860596,
"step": 6090,
"unclipped_grad_norm": 0.8372970223426819
},
{
"bits_per_byte": 1.486896301987214,
"learning_rate": 6.954058453981609e-05,
"loss_nats": 1.0306379795074463,
"step": 6100,
"unclipped_grad_norm": 0.827723503112793
},
{
"bits_per_byte": 1.4668594632484424,
"learning_rate": 6.914680307690504e-05,
"loss_nats": 1.0167495012283325,
"step": 6110,
"unclipped_grad_norm": 0.871588945388794
},
{
"bits_per_byte": 1.4677823221529722,
"learning_rate": 6.875465948066139e-05,
"loss_nats": 1.017389178276062,
"step": 6120,
"unclipped_grad_norm": 0.8457695245742798
},
{
"bits_per_byte": 1.5061057321926845,
"learning_rate": 6.836416045173733e-05,
"loss_nats": 1.0439529418945312,
"step": 6130,
"unclipped_grad_norm": 0.9012824296951294
},
{
"bits_per_byte": 1.5178127632195666,
"learning_rate": 6.797531266268377e-05,
"loss_nats": 1.0520676374435425,
"step": 6140,
"unclipped_grad_norm": 0.8450613617897034
},
{
"bits_per_byte": 1.4675592606548031,
"learning_rate": 6.758812275783663e-05,
"loss_nats": 1.0172345638275146,
"step": 6150,
"unclipped_grad_norm": 0.885280191898346
},
{
"bits_per_byte": 1.5554258849112155,
"learning_rate": 6.720259735320298e-05,
"loss_nats": 1.078139066696167,
"step": 6160,
"unclipped_grad_norm": 0.9099593758583069
},
{
"bits_per_byte": 1.421598961003528,
"learning_rate": 6.681874303634842e-05,
"loss_nats": 0.985377311706543,
"step": 6170,
"unclipped_grad_norm": 0.7732160091400146
},
{
"bits_per_byte": 1.5028951600663776,
"learning_rate": 6.643656636628402e-05,
"loss_nats": 1.0417275428771973,
"step": 6180,
"unclipped_grad_norm": 0.869156002998352
},
{
"bits_per_byte": 1.4956368042693635,
"learning_rate": 6.605607387335466e-05,
"loss_nats": 1.036696434020996,
"step": 6190,
"unclipped_grad_norm": 0.8908746838569641
},
{
"bits_per_byte": 1.4533810109176732,
"learning_rate": 6.567727205912724e-05,
"loss_nats": 1.0074069499969482,
"step": 6200,
"unclipped_grad_norm": 0.8600793480873108
},
{
"bits_per_byte": 1.475978499345141,
"learning_rate": 6.530016739627959e-05,
"loss_nats": 1.0230703353881836,
"step": 6210,
"unclipped_grad_norm": 0.8739057779312134
},
{
"bits_per_byte": 1.4582183669384914,
"learning_rate": 6.492476632848993e-05,
"loss_nats": 1.010759949684143,
"step": 6220,
"unclipped_grad_norm": 0.8542900681495667
},
{
"bits_per_byte": 1.4401650041121208,
"learning_rate": 6.455107527032678e-05,
"loss_nats": 0.9982463121414185,
"step": 6230,
"unclipped_grad_norm": 0.8145909309387207
},
{
"bits_per_byte": 1.458188613939892,
"learning_rate": 6.417910060713926e-05,
"loss_nats": 1.0107393264770508,
"step": 6240,
"unclipped_grad_norm": 0.8962635397911072
},
{
"bits_per_byte": 1.4595658510080016,
"learning_rate": 6.380884869494799e-05,
"loss_nats": 1.0116939544677734,
"step": 6250,
"unclipped_grad_norm": 0.855821430683136
},
{
"bits_per_byte": 1.5147881043388425,
"learning_rate": 6.34403258603367e-05,
"loss_nats": 1.049971103668213,
"step": 6260,
"unclipped_grad_norm": 0.8663405179977417
},
{
"bits_per_byte": 1.4764731214490225,
"learning_rate": 6.30735384003437e-05,
"loss_nats": 1.0234131813049316,
"step": 6270,
"unclipped_grad_norm": 0.8501848578453064
},
{
"bits_per_byte": 1.59573105103682,
"learning_rate": 6.270849258235484e-05,
"loss_nats": 1.1060764789581299,
"step": 6280,
"unclipped_grad_norm": 0.9987632036209106
},
{
"bits_per_byte": 1.4579987450933396,
"learning_rate": 6.234519464399581e-05,
"loss_nats": 1.0106077194213867,
"step": 6290,
"unclipped_grad_norm": 0.8792169690132141
},
{
"bits_per_byte": 1.4898052165439075,
"learning_rate": 6.198365079302614e-05,
"loss_nats": 1.0326542854309082,
"step": 6300,
"unclipped_grad_norm": 0.8601147532463074
},
{
"bits_per_byte": 1.537688110249178,
"learning_rate": 6.16238672072326e-05,
"loss_nats": 1.065844178199768,
"step": 6310,
"unclipped_grad_norm": 0.907733678817749
},
{
"bits_per_byte": 1.487509248154889,
"learning_rate": 6.126585003432406e-05,
"loss_nats": 1.0310628414154053,
"step": 6320,
"unclipped_grad_norm": 0.8578711152076721
},
{
"bits_per_byte": 1.4237945775157679,
"learning_rate": 6.090960539182614e-05,
"loss_nats": 0.986899197101593,
"step": 6330,
"unclipped_grad_norm": 0.845673143863678
},
{
"bits_per_byte": 1.4966876182661353,
"learning_rate": 6.0555139366976935e-05,
"loss_nats": 1.0374248027801514,
"step": 6340,
"unclipped_grad_norm": 0.8802160620689392
},
{
"bits_per_byte": 1.485631197607467,
"learning_rate": 6.020245801662274e-05,
"loss_nats": 1.0297610759735107,
"step": 6350,
"unclipped_grad_norm": 0.884225606918335
},
{
"bits_per_byte": 1.4432266392627804,
"learning_rate": 5.985156736711483e-05,
"loss_nats": 1.0003684759140015,
"step": 6360,
"unclipped_grad_norm": 0.8594306707382202
},
{
"bits_per_byte": 1.497669639202562,
"learning_rate": 5.950247341420618e-05,
"loss_nats": 1.0381054878234863,
"step": 6370,
"unclipped_grad_norm": 0.9075121283531189
},
{
"bits_per_byte": 1.4643473126672892,
"learning_rate": 5.915518212294935e-05,
"loss_nats": 1.0150082111358643,
"step": 6380,
"unclipped_grad_norm": 0.8724440932273865
},
{
"bits_per_byte": 1.4983035672536422,
"learning_rate": 5.880969942759425e-05,
"loss_nats": 1.0385448932647705,
"step": 6390,
"unclipped_grad_norm": 0.9329665303230286
},
{
"bits_per_byte": 1.4855978329731998,
"learning_rate": 5.846603123148688e-05,
"loss_nats": 1.029737949371338,
"step": 6400,
"unclipped_grad_norm": 0.8683961033821106
},
{
"bits_per_byte": 1.4585226042478678,
"learning_rate": 5.812418340696858e-05,
"loss_nats": 1.0109708309173584,
"step": 6410,
"unclipped_grad_norm": 0.8944246768951416
},
{
"bits_per_byte": 1.4417726979323862,
"learning_rate": 5.778416179527541e-05,
"loss_nats": 0.9993606805801392,
"step": 6420,
"unclipped_grad_norm": 0.8450288772583008
},
{
"bits_per_byte": 1.474598510554618,
"learning_rate": 5.7445972206438635e-05,
"loss_nats": 1.0221138000488281,
"step": 6430,
"unclipped_grad_norm": 0.8708812594413757
},
{
"bits_per_byte": 1.4773634756325393,
"learning_rate": 5.710962041918516e-05,
"loss_nats": 1.024030327796936,
"step": 6440,
"unclipped_grad_norm": 0.8653560876846313
},
{
"bits_per_byte": 1.4567353605401014,
"learning_rate": 5.677511218083908e-05,
"loss_nats": 1.0097320079803467,
"step": 6450,
"unclipped_grad_norm": 0.8851431012153625
},
{
"bits_per_byte": 1.4433678370191387,
"learning_rate": 5.644245320722316e-05,
"loss_nats": 1.0004663467407227,
"step": 6460,
"unclipped_grad_norm": 0.902314305305481
},
{
"bits_per_byte": 1.478021997202693,
"learning_rate": 5.61116491825615e-05,
"loss_nats": 1.024486780166626,
"step": 6470,
"unclipped_grad_norm": 0.8634862899780273
},
{
"bits_per_byte": 1.49569390250945,
"learning_rate": 5.5782705759382104e-05,
"loss_nats": 1.036736011505127,
"step": 6480,
"unclipped_grad_norm": 0.8877258896827698
},
{
"bits_per_byte": 1.4739315618345719,
"learning_rate": 5.5455628558420546e-05,
"loss_nats": 1.0216515064239502,
"step": 6490,
"unclipped_grad_norm": 0.9130761027336121
},
{
"bits_per_byte": 1.4798123365981748,
"learning_rate": 5.513042316852368e-05,
"loss_nats": 1.0257277488708496,
"step": 6500,
"unclipped_grad_norm": 0.9244931936264038
},
{
"bits_per_byte": 1.429752142532981,
"learning_rate": 5.480709514655443e-05,
"loss_nats": 0.9910286664962769,
"step": 6510,
"unclipped_grad_norm": 0.8583832383155823
},
{
"bits_per_byte": 1.3992985726054255,
"learning_rate": 5.448565001729653e-05,
"loss_nats": 0.9699198603630066,
"step": 6520,
"unclipped_grad_norm": 0.8968297243118286
},
{
"bits_per_byte": 1.5067831718544331,
"learning_rate": 5.41660932733604e-05,
"loss_nats": 1.0444225072860718,
"step": 6530,
"unclipped_grad_norm": 0.9009160399436951
},
{
"bits_per_byte": 1.468959715380779,
"learning_rate": 5.384843037508902e-05,
"loss_nats": 1.0182052850723267,
"step": 6540,
"unclipped_grad_norm": 0.8687822222709656
},
{
"bits_per_byte": 1.5110841139872102,
"learning_rate": 5.353266675046498e-05,
"loss_nats": 1.0474036931991577,
"step": 6550,
"unclipped_grad_norm": 0.9212287068367004
},
{
"bits_per_byte": 1.4657298811975754,
"learning_rate": 5.321880779501729e-05,
"loss_nats": 1.015966534614563,
"step": 6560,
"unclipped_grad_norm": 0.8736171126365662
},
{
"bits_per_byte": 1.3923212364599213,
"learning_rate": 5.290685887172962e-05,
"loss_nats": 0.9650835394859314,
"step": 6570,
"unclipped_grad_norm": 0.9459646344184875
},
{
"bits_per_byte": 1.5011586512406159,
"learning_rate": 5.2596825310948295e-05,
"loss_nats": 1.040523886680603,
"step": 6580,
"unclipped_grad_norm": 0.8902279138565063
},
{
"bits_per_byte": 1.4631396504929306,
"learning_rate": 5.228871241029154e-05,
"loss_nats": 1.0141711235046387,
"step": 6590,
"unclipped_grad_norm": 0.8934556245803833
},
{
"bits_per_byte": 1.3892133379761795,
"learning_rate": 5.198252543455865e-05,
"loss_nats": 0.9629293084144592,
"step": 6600,
"unclipped_grad_norm": 0.8376922607421875
},
{
"bits_per_byte": 1.5024242715683152,
"learning_rate": 5.167826961564027e-05,
"loss_nats": 1.0414011478424072,
"step": 6610,
"unclipped_grad_norm": 0.8684409856796265
},
{
"bits_per_byte": 1.5173028346597586,
"learning_rate": 5.137595015242884e-05,
"loss_nats": 1.0517141819000244,
"step": 6620,
"unclipped_grad_norm": 0.9338745474815369
},
{
"bits_per_byte": 1.4964176055042808,
"learning_rate": 5.1075572210729925e-05,
"loss_nats": 1.0372376441955566,
"step": 6630,
"unclipped_grad_norm": 0.9484766721725464
},
{
"bits_per_byte": 1.4696447222791655,
"learning_rate": 5.07771409231737e-05,
"loss_nats": 1.0186800956726074,
"step": 6640,
"unclipped_grad_norm": 0.9511991143226624
},
{
"bits_per_byte": 1.4476113369465278,
"learning_rate": 5.048066138912756e-05,
"loss_nats": 1.0034077167510986,
"step": 6650,
"unclipped_grad_norm": 0.8744685649871826
},
{
"bits_per_byte": 1.5102728718230902,
"learning_rate": 5.018613867460863e-05,
"loss_nats": 1.0468413829803467,
"step": 6660,
"unclipped_grad_norm": 0.9144671559333801
},
{
"bits_per_byte": 1.4729612357184039,
"learning_rate": 4.989357781219755e-05,
"loss_nats": 1.0209789276123047,
"step": 6670,
"unclipped_grad_norm": 0.918013870716095
},
{
"bits_per_byte": 1.4976474534406006,
"learning_rate": 4.960298380095222e-05,
"loss_nats": 1.0380901098251343,
"step": 6680,
"unclipped_grad_norm": 0.9484038352966309
},
{
"bits_per_byte": 1.4621077545877539,
"learning_rate": 4.931436160632256e-05,
"loss_nats": 1.013455867767334,
"step": 6690,
"unclipped_grad_norm": 0.9366878867149353
},
{
"bits_per_byte": 1.513278096664267,
"learning_rate": 4.9027716160065485e-05,
"loss_nats": 1.048924446105957,
"step": 6700,
"unclipped_grad_norm": 0.9588149785995483
},
{
"bits_per_byte": 1.448871969777352,
"learning_rate": 4.874305236016084e-05,
"loss_nats": 1.0042815208435059,
"step": 6710,
"unclipped_grad_norm": 0.8966986536979675
},
{
"bits_per_byte": 1.4475960304906008,
"learning_rate": 4.846037507072752e-05,
"loss_nats": 1.0033971071243286,
"step": 6720,
"unclipped_grad_norm": 0.9147130250930786
},
{
"bits_per_byte": 1.4208115384365525,
"learning_rate": 4.817968912194049e-05,
"loss_nats": 0.9848315119743347,
"step": 6730,
"unclipped_grad_norm": 0.9434075951576233
},
{
"bits_per_byte": 1.4530764296429952,
"learning_rate": 4.790099930994813e-05,
"loss_nats": 1.0071958303451538,
"step": 6740,
"unclipped_grad_norm": 0.9062299728393555
},
{
"bits_per_byte": 1.4277258429405155,
"learning_rate": 4.762431039679047e-05,
"loss_nats": 0.9896241426467896,
"step": 6750,
"unclipped_grad_norm": 0.9092971086502075
},
{
"bits_per_byte": 1.4128754850038137,
"learning_rate": 4.734962711031755e-05,
"loss_nats": 0.9793306589126587,
"step": 6760,
"unclipped_grad_norm": 0.8910818099975586
},
{
"bits_per_byte": 1.434429210723195,
"learning_rate": 4.707695414410892e-05,
"loss_nats": 0.9942705631256104,
"step": 6770,
"unclipped_grad_norm": 0.8744387030601501
},
{
"bits_per_byte": 1.4315308730995298,
"learning_rate": 4.680629615739322e-05,
"loss_nats": 0.9922615885734558,
"step": 6780,
"unclipped_grad_norm": 0.9047504663467407
},
{
"bits_per_byte": 1.53544803622169,
"learning_rate": 4.653765777496863e-05,
"loss_nats": 1.0642914772033691,
"step": 6790,
"unclipped_grad_norm": 0.9123643636703491
},
{
"bits_per_byte": 1.504298710480069,
"learning_rate": 4.6271043587123986e-05,
"loss_nats": 1.0427004098892212,
"step": 6800,
"unclipped_grad_norm": 0.978726327419281
},
{
"bits_per_byte": 1.4819994399691971,
"learning_rate": 4.600645814956006e-05,
"loss_nats": 1.027243733406067,
"step": 6810,
"unclipped_grad_norm": 0.9510568380355835
},
{
"bits_per_byte": 1.4201767504722178,
"learning_rate": 4.574390598331204e-05,
"loss_nats": 0.9843915104866028,
"step": 6820,
"unclipped_grad_norm": 0.9054594039916992
},
{
"bits_per_byte": 1.441129998766112,
"learning_rate": 4.548339157467199e-05,
"loss_nats": 0.9989151954650879,
"step": 6830,
"unclipped_grad_norm": 0.9485244154930115
},
{
"bits_per_byte": 1.4736460706341397,
"learning_rate": 4.52249193751124e-05,
"loss_nats": 1.021453619003296,
"step": 6840,
"unclipped_grad_norm": 0.9141860604286194
},
{
"bits_per_byte": 1.439767810179953,
"learning_rate": 4.4968493801209985e-05,
"loss_nats": 0.9979709982872009,
"step": 6850,
"unclipped_grad_norm": 0.9174992442131042
},
{
"bits_per_byte": 1.4496288654237992,
"learning_rate": 4.471411923457034e-05,
"loss_nats": 1.0048061609268188,
"step": 6860,
"unclipped_grad_norm": 0.9317811727523804
},
{
"bits_per_byte": 1.4588201342338605,
"learning_rate": 4.446180002175291e-05,
"loss_nats": 1.0111770629882812,
"step": 6870,
"unclipped_grad_norm": 0.9191097617149353
},
{
"bits_per_byte": 1.4995917173086046,
"learning_rate": 4.421154047419693e-05,
"loss_nats": 1.0394377708435059,
"step": 6880,
"unclipped_grad_norm": 0.9317643642425537
},
{
"bits_per_byte": 1.3990970949299397,
"learning_rate": 4.396334486814753e-05,
"loss_nats": 0.9697802066802979,
"step": 6890,
"unclipped_grad_norm": 0.9130836725234985
},
{
"bits_per_byte": 1.4230204836042344,
"learning_rate": 4.371721744458285e-05,
"loss_nats": 0.986362636089325,
"step": 6900,
"unclipped_grad_norm": 0.8910167217254639
},
{
"bits_per_byte": 1.5914774041330302,
"learning_rate": 4.347316240914145e-05,
"loss_nats": 1.1031280755996704,
"step": 6910,
"unclipped_grad_norm": 1.0252712965011597
},
{
"bits_per_byte": 1.394897880535148,
"learning_rate": 4.3231183932050546e-05,
"loss_nats": 0.9668695330619812,
"step": 6920,
"unclipped_grad_norm": 0.8750832676887512
},
{
"bits_per_byte": 1.42465509270936,
"learning_rate": 4.299128614805464e-05,
"loss_nats": 0.9874956607818604,
"step": 6930,
"unclipped_grad_norm": 0.8865536451339722
},
{
"bits_per_byte": 1.4536008047454758,
"learning_rate": 4.2753473156344965e-05,
"loss_nats": 1.0075592994689941,
"step": 6940,
"unclipped_grad_norm": 0.913940966129303
},
{
"bits_per_byte": 1.4401447961506444,
"learning_rate": 4.251774902048942e-05,
"loss_nats": 0.9982323050498962,
"step": 6950,
"unclipped_grad_norm": 0.941871702671051
},
{
"bits_per_byte": 1.4963969675861772,
"learning_rate": 4.228411776836306e-05,
"loss_nats": 1.0372233390808105,
"step": 6960,
"unclipped_grad_norm": 0.9974261522293091
},
{
"bits_per_byte": 1.4704031657694705,
"learning_rate": 4.2052583392079427e-05,
"loss_nats": 1.0192058086395264,
"step": 6970,
"unclipped_grad_norm": 0.9634589552879333
},
{
"bits_per_byte": 1.5157009882496222,
"learning_rate": 4.182314984792214e-05,
"loss_nats": 1.0506038665771484,
"step": 6980,
"unclipped_grad_norm": 0.9805173873901367
},
{
"bits_per_byte": 1.4572141602401034,
"learning_rate": 4.159582105627748e-05,
"loss_nats": 1.010063886642456,
"step": 6990,
"unclipped_grad_norm": 0.9425977468490601
},
{
"bits_per_byte": 1.4146713278441232,
"learning_rate": 4.137060090156724e-05,
"loss_nats": 0.980575442314148,
"step": 7000,
"unclipped_grad_norm": 0.9211949110031128
},
{
"bits_per_byte": 1.4423556331274856,
"learning_rate": 4.114749323218249e-05,
"loss_nats": 0.9997647404670715,
"step": 7010,
"unclipped_grad_norm": 0.9508169293403625
},
{
"bits_per_byte": 1.4268384124620634,
"learning_rate": 4.0926501860417706e-05,
"loss_nats": 0.9890090227127075,
"step": 7020,
"unclipped_grad_norm": 0.9370734095573425
},
{
"bits_per_byte": 1.4214180352548202,
"learning_rate": 4.070763056240571e-05,
"loss_nats": 0.9852519035339355,
"step": 7030,
"unclipped_grad_norm": 0.8771427273750305
},
{
"bits_per_byte": 1.5093840654834314,
"learning_rate": 4.049088307805306e-05,
"loss_nats": 1.0462253093719482,
"step": 7040,
"unclipped_grad_norm": 0.9992957711219788
},
{
"bits_per_byte": 1.466185635222362,
"learning_rate": 4.027626311097629e-05,
"loss_nats": 1.0162824392318726,
"step": 7050,
"unclipped_grad_norm": 0.9372990131378174
},
{
"bits_per_byte": 1.4783773133593756,
"learning_rate": 4.006377432843846e-05,
"loss_nats": 1.024733066558838,
"step": 7060,
"unclipped_grad_norm": 1.0182687044143677
},
{
"bits_per_byte": 1.4142496263842075,
"learning_rate": 3.985342036128656e-05,
"loss_nats": 0.9802831411361694,
"step": 7070,
"unclipped_grad_norm": 0.8880259394645691
},
{
"bits_per_byte": 1.3447492013960853,
"learning_rate": 3.964520480388956e-05,
"loss_nats": 0.9321091175079346,
"step": 7080,
"unclipped_grad_norm": 0.910698413848877
},
{
"bits_per_byte": 1.413691198716856,
"learning_rate": 3.943913121407678e-05,
"loss_nats": 0.979896068572998,
"step": 7090,
"unclipped_grad_norm": 0.9244340062141418
},
{
"bits_per_byte": 1.4025504205679395,
"learning_rate": 3.923520311307735e-05,
"loss_nats": 0.9721738696098328,
"step": 7100,
"unclipped_grad_norm": 0.9016135334968567
},
{
"bits_per_byte": 1.4533139376838367,
"learning_rate": 3.9033423985459816e-05,
"loss_nats": 1.0073604583740234,
"step": 7110,
"unclipped_grad_norm": 0.970280110836029
},
{
"bits_per_byte": 1.4606615524766484,
"learning_rate": 3.88337972790728e-05,
"loss_nats": 1.0124534368515015,
"step": 7120,
"unclipped_grad_norm": 0.9771916270256042
},
{
"bits_per_byte": 1.396041221198084,
"learning_rate": 3.8636326404985845e-05,
"loss_nats": 0.9676620364189148,
"step": 7130,
"unclipped_grad_norm": 0.8866344094276428
},
{
"bits_per_byte": 1.5095894127685614,
"learning_rate": 3.844101473743143e-05,
"loss_nats": 1.0463676452636719,
"step": 7140,
"unclipped_grad_norm": 0.9947805404663086
},
{
"bits_per_byte": 1.5291790966150898,
"learning_rate": 3.824786561374701e-05,
"loss_nats": 1.0599461793899536,
"step": 7150,
"unclipped_grad_norm": 0.9418985247612
},
{
"bits_per_byte": 1.4640222654571586,
"learning_rate": 3.805688233431824e-05,
"loss_nats": 1.0147829055786133,
"step": 7160,
"unclipped_grad_norm": 1.0358314514160156
},
{
"bits_per_byte": 1.4559156912260893,
"learning_rate": 3.786806816252236e-05,
"loss_nats": 1.0091638565063477,
"step": 7170,
"unclipped_grad_norm": 0.9539772272109985
},
{
"bits_per_byte": 1.3591113005869877,
"learning_rate": 3.768142632467269e-05,
"loss_nats": 0.9420641660690308,
"step": 7180,
"unclipped_grad_norm": 0.8915972709655762
},
{
"bits_per_byte": 1.3840849873101015,
"learning_rate": 3.7496960009963236e-05,
"loss_nats": 0.9593746066093445,
"step": 7190,
"unclipped_grad_norm": 0.9118176102638245
},
{
"bits_per_byte": 1.4715360154907038,
"learning_rate": 3.7314672370414325e-05,
"loss_nats": 1.0199910402297974,
"step": 7200,
"unclipped_grad_norm": 0.973518431186676
},
{
"bits_per_byte": 1.4678313372084681,
"learning_rate": 3.713456652081882e-05,
"loss_nats": 1.017423152923584,
"step": 7210,
"unclipped_grad_norm": 0.9181845188140869
},
{
"bits_per_byte": 1.4265923052886786,
"learning_rate": 3.695664553868873e-05,
"loss_nats": 0.9888384342193604,
"step": 7220,
"unclipped_grad_norm": 0.926552951335907
},
{
"bits_per_byte": 1.4337483314084294,
"learning_rate": 3.678091246420277e-05,
"loss_nats": 0.9937986135482788,
"step": 7230,
"unclipped_grad_norm": 0.9526586532592773
},
{
"bits_per_byte": 1.383381664259398,
"learning_rate": 3.660737030015427e-05,
"loss_nats": 0.9588871002197266,
"step": 7240,
"unclipped_grad_norm": 0.8890237212181091
},
{
"bits_per_byte": 1.431437056563484,
"learning_rate": 3.6436022011900056e-05,
"loss_nats": 0.9921965599060059,
"step": 7250,
"unclipped_grad_norm": 0.9829798340797424
},
{
"bits_per_byte": 1.4272578781475178,
"learning_rate": 3.626687052730954e-05,
"loss_nats": 0.9892997741699219,
"step": 7260,
"unclipped_grad_norm": 0.9537968039512634
},
{
"bits_per_byte": 1.5069463833901018,
"learning_rate": 3.6099918736714955e-05,
"loss_nats": 1.0445356369018555,
"step": 7270,
"unclipped_grad_norm": 0.9953339099884033
},
{
"bits_per_byte": 1.4318632295723221,
"learning_rate": 3.5935169492861716e-05,
"loss_nats": 0.9924919605255127,
"step": 7280,
"unclipped_grad_norm": 0.9815621972084045
},
{
"bits_per_byte": 1.435238647069481,
"learning_rate": 3.5772625610859886e-05,
"loss_nats": 0.9948316216468811,
"step": 7290,
"unclipped_grad_norm": 0.9373346567153931
},
{
"bits_per_byte": 1.44098209368637,
"learning_rate": 3.5612289868135887e-05,
"loss_nats": 0.9988126754760742,
"step": 7300,
"unclipped_grad_norm": 0.9782799482345581
},
{
"bits_per_byte": 1.467732619166873,
"learning_rate": 3.545416500438522e-05,
"loss_nats": 1.0173547267913818,
"step": 7310,
"unclipped_grad_norm": 0.9545702338218689
},
{
"bits_per_byte": 1.4680886232541588,
"learning_rate": 3.52982537215255e-05,
"loss_nats": 1.017601490020752,
"step": 7320,
"unclipped_grad_norm": 0.9820490479469299
},
{
"bits_per_byte": 1.4597027491980883,
"learning_rate": 3.5144558683650396e-05,
"loss_nats": 1.0117888450622559,
"step": 7330,
"unclipped_grad_norm": 1.0537965297698975
},
{
"bits_per_byte": 1.4358094574876945,
"learning_rate": 3.499308251698404e-05,
"loss_nats": 0.9952272772789001,
"step": 7340,
"unclipped_grad_norm": 0.9470505714416504
},
{
"bits_per_byte": 1.4763711357370608,
"learning_rate": 3.484382780983616e-05,
"loss_nats": 1.023342490196228,
"step": 7350,
"unclipped_grad_norm": 0.93792724609375
},
{
"bits_per_byte": 1.4524729425211176,
"learning_rate": 3.469679711255795e-05,
"loss_nats": 1.0067775249481201,
"step": 7360,
"unclipped_grad_norm": 0.959013044834137
},
{
"bits_per_byte": 1.4678055398108387,
"learning_rate": 3.4551992937498284e-05,
"loss_nats": 1.0174052715301514,
"step": 7370,
"unclipped_grad_norm": 0.9769075512886047
},
{
"bits_per_byte": 1.5064952728968888,
"learning_rate": 3.440941775896104e-05,
"loss_nats": 1.0442229509353638,
"step": 7380,
"unclipped_grad_norm": 1.0261726379394531
},
{
"bits_per_byte": 1.4743861119808024,
"learning_rate": 3.4269074013162596e-05,
"loss_nats": 1.021966576576233,
"step": 7390,
"unclipped_grad_norm": 0.977978527545929
},
{
"bits_per_byte": 1.4106443540741704,
"learning_rate": 3.41309640981904e-05,
"loss_nats": 0.9777841567993164,
"step": 7400,
"unclipped_grad_norm": 0.9216173887252808
},
{
"bits_per_byte": 1.4343313525948542,
"learning_rate": 3.3995090373961795e-05,
"loss_nats": 0.994202733039856,
"step": 7410,
"unclipped_grad_norm": 0.9778451919555664
},
{
"bits_per_byte": 1.5075760118749106,
"learning_rate": 3.3861455162183876e-05,
"loss_nats": 1.0449720621109009,
"step": 7420,
"unclipped_grad_norm": 1.0143206119537354
},
{
"bits_per_byte": 1.4176087055295346,
"learning_rate": 3.3730060746313664e-05,
"loss_nats": 0.9826114773750305,
"step": 7430,
"unclipped_grad_norm": 0.9865745902061462
},
{
"bits_per_byte": 1.4384498211350658,
"learning_rate": 3.3600909371519225e-05,
"loss_nats": 0.9970574378967285,
"step": 7440,
"unclipped_grad_norm": 0.9753794074058533
},
{
"bits_per_byte": 1.385612537215065,
"learning_rate": 3.3474003244641194e-05,
"loss_nats": 0.9604334235191345,
"step": 7450,
"unclipped_grad_norm": 0.9882722496986389
},
{
"bits_per_byte": 1.40335280562554,
"learning_rate": 3.334934453415514e-05,
"loss_nats": 0.9727300405502319,
"step": 7460,
"unclipped_grad_norm": 0.8938714861869812
},
{
"bits_per_byte": 1.4650283639647061,
"learning_rate": 3.3226935370134486e-05,
"loss_nats": 1.0154802799224854,
"step": 7470,
"unclipped_grad_norm": 0.95097815990448
},
{
"bits_per_byte": 1.378530205661209,
"learning_rate": 3.31067778442141e-05,
"loss_nats": 0.9555243253707886,
"step": 7480,
"unclipped_grad_norm": 0.9593113660812378
},
{
"bits_per_byte": 1.4079058743244819,
"learning_rate": 3.298887400955457e-05,
"loss_nats": 0.9758859872817993,
"step": 7490,
"unclipped_grad_norm": 0.9498793482780457
},
{
"bits_per_byte": 1.4795925427703722,
"learning_rate": 3.287322588080714e-05,
"loss_nats": 1.0255753993988037,
"step": 7500,
"unclipped_grad_norm": 0.9999135732650757
},
{
"bits_per_byte": 1.462181535144974,
"learning_rate": 3.275983543407924e-05,
"loss_nats": 1.0135070085525513,
"step": 7510,
"unclipped_grad_norm": 0.96773761510849
},
{
"bits_per_byte": 1.424363066168195,
"learning_rate": 3.2648704606900735e-05,
"loss_nats": 0.9872932434082031,
"step": 7520,
"unclipped_grad_norm": 0.9349227547645569
},
{
"bits_per_byte": 1.4575968216382733,
"learning_rate": 3.253983529819086e-05,
"loss_nats": 1.0103291273117065,
"step": 7530,
"unclipped_grad_norm": 0.9571959972381592
},
{
"bits_per_byte": 1.484410464751643,
"learning_rate": 3.243322936822571e-05,
"loss_nats": 1.0289149284362793,
"step": 7540,
"unclipped_grad_norm": 0.9889418482780457
},
{
"bits_per_byte": 1.4190770074112757,
"learning_rate": 3.232888863860648e-05,
"loss_nats": 0.9836292266845703,
"step": 7550,
"unclipped_grad_norm": 0.9660657048225403
},
{
"bits_per_byte": 1.458957376389249,
"learning_rate": 3.222681489222838e-05,
"loss_nats": 1.0112721920013428,
"step": 7560,
"unclipped_grad_norm": 0.9621381163597107
},
{
"bits_per_byte": 1.4744461339259534,
"learning_rate": 3.212700987325009e-05,
"loss_nats": 1.0220081806182861,
"step": 7570,
"unclipped_grad_norm": 0.9809478521347046
},
{
"bits_per_byte": 1.424117474942763,
"learning_rate": 3.202947528706403e-05,
"loss_nats": 0.9871230125427246,
"step": 7580,
"unclipped_grad_norm": 1.0034832954406738
},
{
"bits_per_byte": 1.4167208450944553,
"learning_rate": 3.1934212800267116e-05,
"loss_nats": 0.9819960594177246,
"step": 7590,
"unclipped_grad_norm": 0.9541960954666138
},
{
"bits_per_byte": 1.3862604818521906,
"learning_rate": 3.1841224040632484e-05,
"loss_nats": 0.9608825445175171,
"step": 7600,
"unclipped_grad_norm": 0.950609564781189
},
{
"bits_per_byte": 1.3616721222583348,
"learning_rate": 3.175051059708143e-05,
"loss_nats": 0.9438391923904419,
"step": 7610,
"unclipped_grad_norm": 0.9224535226821899
},
{
"bits_per_byte": 1.4445553772233464,
"learning_rate": 3.1662074019656414e-05,
"loss_nats": 1.0012894868850708,
"step": 7620,
"unclipped_grad_norm": 0.9717100858688354
},
{
"bits_per_byte": 1.4175787805482842,
"learning_rate": 3.157591581949455e-05,
"loss_nats": 0.9825907349586487,
"step": 7630,
"unclipped_grad_norm": 0.9714697599411011
},
{
"bits_per_byte": 1.4152020663046858,
"learning_rate": 3.149203746880175e-05,
"loss_nats": 0.9809433221817017,
"step": 7640,
"unclipped_grad_norm": 0.9795277118682861
},
{
"bits_per_byte": 1.5113582543326856,
"learning_rate": 3.141044040082763e-05,
"loss_nats": 1.0475937128067017,
"step": 7650,
"unclipped_grad_norm": 1.0563151836395264
},
{
"bits_per_byte": 1.4375178471500403,
"learning_rate": 3.133112600984091e-05,
"loss_nats": 0.9964114427566528,
"step": 7660,
"unclipped_grad_norm": 0.9650751948356628
},
{
"bits_per_byte": 1.4172367070557181,
"learning_rate": 3.125409565110571e-05,
"loss_nats": 0.9823536276817322,
"step": 7670,
"unclipped_grad_norm": 0.9894714951515198
},
{
"bits_per_byte": 1.3937812831744212,
"learning_rate": 3.117935064085835e-05,
"loss_nats": 0.9660955667495728,
"step": 7680,
"unclipped_grad_norm": 0.9648637175559998
},
{
"bits_per_byte": 1.4266868097553278,
"learning_rate": 3.110689225628479e-05,
"loss_nats": 0.9889039397239685,
"step": 7690,
"unclipped_grad_norm": 0.9686183333396912
},
{
"bits_per_byte": 1.4503658110827464,
"learning_rate": 3.1036721735498926e-05,
"loss_nats": 1.005316972732544,
"step": 7700,
"unclipped_grad_norm": 0.9993582367897034
},
{
"bits_per_byte": 1.3691781330725912,
"learning_rate": 3.096884027752132e-05,
"loss_nats": 0.9490419626235962,
"step": 7710,
"unclipped_grad_norm": 0.9620972275733948
},
{
"bits_per_byte": 1.3650053180147068,
"learning_rate": 3.090324904225885e-05,
"loss_nats": 0.9461495876312256,
"step": 7720,
"unclipped_grad_norm": 0.9128993153572083
},
{
"bits_per_byte": 1.4252105106803215,
"learning_rate": 3.083994915048472e-05,
"loss_nats": 0.9878806471824646,
"step": 7730,
"unclipped_grad_norm": 0.9594771265983582
},
{
"bits_per_byte": 1.4081796707046554,
"learning_rate": 3.077894168381942e-05,
"loss_nats": 0.9760757684707642,
"step": 7740,
"unclipped_grad_norm": 0.9255442023277283
},
{
"bits_per_byte": 1.433292233418341,
"learning_rate": 3.0720227684712254e-05,
"loss_nats": 0.9934824705123901,
"step": 7750,
"unclipped_grad_norm": 0.9785766005516052
},
{
"bits_per_byte": 1.3999133245909345,
"learning_rate": 3.0663808156423456e-05,
"loss_nats": 0.9703459739685059,
"step": 7760,
"unclipped_grad_norm": 1.0032256841659546
},
{
"bits_per_byte": 1.4048038232508695,
"learning_rate": 3.060968406300711e-05,
"loss_nats": 0.9737358093261719,
"step": 7770,
"unclipped_grad_norm": 1.0204466581344604
},
{
"bits_per_byte": 1.536527399338505,
"learning_rate": 3.055785632929463e-05,
"loss_nats": 1.0650396347045898,
"step": 7780,
"unclipped_grad_norm": 1.0556561946868896
},
{
"bits_per_byte": 1.4128328333063997,
"learning_rate": 3.0508325840878984e-05,
"loss_nats": 0.9793010950088501,
"step": 7790,
"unclipped_grad_norm": 1.0078489780426025
},
{
"bits_per_byte": 1.451735308931567,
"learning_rate": 3.046109344409957e-05,
"loss_nats": 1.0062662363052368,
"step": 7800,
"unclipped_grad_norm": 1.0289596319198608
},
{
"bits_per_byte": 1.3852072600983067,
"learning_rate": 3.0416159946027727e-05,
"loss_nats": 0.9601525068283081,
"step": 7810,
"unclipped_grad_norm": 0.9891387224197388
},
{
"bits_per_byte": 1.3953124447150527,
"learning_rate": 3.0373526114452946e-05,
"loss_nats": 0.9671568870544434,
"step": 7820,
"unclipped_grad_norm": 0.9778857231140137
},
{
"bits_per_byte": 1.45392653988621,
"learning_rate": 3.0333192677869803e-05,
"loss_nats": 1.0077850818634033,
"step": 7830,
"unclipped_grad_norm": 1.0185887813568115
},
{
"bits_per_byte": 1.4113820736550464,
"learning_rate": 3.029516032546544e-05,
"loss_nats": 0.9782955050468445,
"step": 7840,
"unclipped_grad_norm": 0.9330953359603882
},
{
"bits_per_byte": 1.380841996454107,
"learning_rate": 3.02594297071078e-05,
"loss_nats": 0.9571267366409302,
"step": 7850,
"unclipped_grad_norm": 0.9742368459701538
},
{
"bits_per_byte": 1.3846048048723343,
"learning_rate": 3.0226001433334603e-05,
"loss_nats": 0.9597349166870117,
"step": 7860,
"unclipped_grad_norm": 0.9645414352416992
},
{
"bits_per_byte": 1.4429803601067448,
"learning_rate": 3.019487607534278e-05,
"loss_nats": 1.0001977682113647,
"step": 7870,
"unclipped_grad_norm": 0.9958038926124573
},
{
"bits_per_byte": 1.4256185395194934,
"learning_rate": 3.016605416497886e-05,
"loss_nats": 0.9881634712219238,
"step": 7880,
"unclipped_grad_norm": 0.9739922881126404
},
{
"bits_per_byte": 1.4834710955126302,
"learning_rate": 3.0139536194729734e-05,
"loss_nats": 1.028263807296753,
"step": 7890,
"unclipped_grad_norm": 0.9912284016609192
},
{
"bits_per_byte": 1.4598313922209336,
"learning_rate": 3.0115322617714362e-05,
"loss_nats": 1.0118780136108398,
"step": 7900,
"unclipped_grad_norm": 0.9979069232940674
},
{
"bits_per_byte": 1.465345499972897,
"learning_rate": 3.0093413847675965e-05,
"loss_nats": 1.015700101852417,
"step": 7910,
"unclipped_grad_norm": 1.0503500699996948
},
{
"bits_per_byte": 1.3660396216769959,
"learning_rate": 3.0073810258974985e-05,
"loss_nats": 0.946866512298584,
"step": 7920,
"unclipped_grad_norm": 0.9824531078338623
},
{
"bits_per_byte": 1.3848080023743288,
"learning_rate": 3.0056512186582658e-05,
"loss_nats": 0.959875762462616,
"step": 7930,
"unclipped_grad_norm": 1.0091050863265991
},
{
"bits_per_byte": 1.4375059803471308,
"learning_rate": 3.0041519926075316e-05,
"loss_nats": 0.9964032173156738,
"step": 7940,
"unclipped_grad_norm": 1.0658900737762451
},
{
"bits_per_byte": 1.4049340141175726,
"learning_rate": 3.0028833733629313e-05,
"loss_nats": 0.9738260507583618,
"step": 7950,
"unclipped_grad_norm": 1.0148085355758667
},
{
"bits_per_byte": 1.415199744538899,
"learning_rate": 3.0018453826016686e-05,
"loss_nats": 0.9809417128562927,
"step": 7960,
"unclipped_grad_norm": 0.9737400412559509
},
{
"bits_per_byte": 1.4008050546356583,
"learning_rate": 3.001038038060143e-05,
"loss_nats": 0.9709640741348267,
"step": 7970,
"unclipped_grad_norm": 0.9626508951187134
},
{
"bits_per_byte": 1.4372590132604919,
"learning_rate": 3.0004613535336433e-05,
"loss_nats": 0.9962320327758789,
"step": 7980,
"unclipped_grad_norm": 0.9673776626586914
},
{
"bits_per_byte": 1.4403348369798477,
"learning_rate": 3.000115338876117e-05,
"loss_nats": 0.9983640313148499,
"step": 7990,
"unclipped_grad_norm": 0.9839751720428467
},
{
"bits_per_byte": 1.3635582559903472,
"learning_rate": 3e-05,
"loss_nats": 0.9451465606689453,
"step": 8000,
"unclipped_grad_norm": 0.9773924946784973
}
]
}
Reference in New Issue View Git Blame Copy Permalink