wuyang/llm-atlas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
204f5993049e320580841e1fefeaeebd29fd65a6
llm-atlas/experiments/k3/attnres_gradient/results/raw/formal-depth-32-block-seed-2026073002.json
T
wuyang 26fc824409 research: audit AttnRes gradient scale study
2026-07-30 09:49:53 +08:00

13073 lines
366 KiB
JSON
Raw Blame History

{
"architecture": "block",
"batch_size": 32,
"canonical_sha256_without_self": "b4262e697e2269457fdebf31f75008383c6e8024ee1bbf96e5962fb2ba1dec18",
"depth": 32,
"diagnostics": [
{
"activation_grad_rms_by_block": [
0.00040311424527317286,
0.00035970969474874437,
0.00032544726855121553,
0.00029308689408935606,
0.00026978139067068696,
0.00025443214690312743,
0.00023969457834027708,
0.0002233884297311306,
0.0002095247182296589,
0.0001984660339076072,
0.0001901928335428238,
0.00018121300672646612,
0.0001734284160193056,
0.0001677417167229578,
0.00016109299031086266,
0.00015624657680746168,
0.0001518603676231578,
0.00014820833166595548,
0.00014440594532061368,
0.0001414585713064298,
0.00013923499500378966,
0.00013687640603166074,
0.00013471563579514623,
0.00013265138841234148,
0.00013098257477395236,
0.0001293915556743741,
0.00012844328011851758,
0.0001269879430765286,
0.0001256384130101651,
0.00012456141121219844,
0.0001231025526067242,
0.00012163266364950687
],
"activation_grad_statistics": {
"first_quartile_mean": 0.00029608183103846386,
"first_to_last_ratio": 2.343484699021421,
"imbalance_abs_log_ratio": 0.8516390092453431,
"last_quartile_mean": 0.0001263425492652459,
"mean": 0.0001858347804954974,
"normalized": [
2.16920774571012,
1.9356424765570717,
1.751272112160408,
1.5771369240348272,
1.451727119925344,
1.369130935687747,
1.289826251583108,
1.2020808437231323,
1.127478492836466,
1.067970341065492,
1.0234512239081746,
0.9751296621832141,
0.9332398141881068,
0.9026389800429313,
0.8668613586828855,
0.8407822065969367,
0.8171794710239251,
0.7975274126338607,
0.777066300159633,
0.7612061150730458,
0.7492407752334779,
0.7365489154759011,
0.7249215428669998,
0.7138135717041164,
0.7048334785593375,
0.6962720074755282,
0.6911692191098191,
0.683337870004401,
0.6760758813563923,
0.6702804011180051,
0.662430102042749,
0.6545204472768429
],
"population_cv": 0.39438035124793563
},
"activation_output_rms_by_block": [
0.0036361459642648697,
0.005152894649654627,
0.006386642809957266,
0.007284422870725393,
0.0037505170330405235,
0.0054016136564314365,
0.006585344672203064,
0.007922882214188576,
0.0044892593286931515,
0.006135277915745974,
0.007316684816032648,
0.0082859443500638,
0.0044554853811860085,
0.006464940495789051,
0.007893112488090992,
0.009196000173687935,
0.004961193539202213,
0.006907128728926182,
0.008217990398406982,
0.009676996618509293,
0.005381465889513493,
0.007502692751586437,
0.0091153085231781,
0.010543142445385456,
0.006136673036962748,
0.008469858206808567,
0.010564134456217289,
0.012301022186875343,
0.005773921962827444,
0.008156762458384037,
0.010457510128617287,
0.012218860909342766
],
"activation_output_statistics": {
"first_quartile_mean": 0.005765057983808219,
"first_to_last_ratio": 0.622587017371887,
"imbalance_abs_log_ratio": 0.4738718734697682,
"last_quartile_mean": 0.009259842918254435,
"mean": 0.007398182220640592,
"normalized": [
0.49149180917985336,
0.6965082091758006,
0.8632718983507628,
0.9846233376628902,
0.5069511565390686,
0.730127144119427,
0.8901300989627224,
1.0709228264321597,
0.6068057253534959,
0.8292953231982888,
0.9889841312125875,
1.119997332175246,
0.6022405569783624,
0.8738552664669654,
1.0668989020126547,
1.2430080659586231,
0.6705962885532487,
0.9336251153230055,
1.1108121094231989,
1.3080235563150786,
0.7274038039370601,
1.01412651484229,
1.232101109614035,
1.4250990487866828,
0.8294838994154143,
1.1448566626512724,
1.427936504016329,
1.6627087330393202,
0.7804514393709395,
1.1025360304896303,
1.4135242707919937,
1.6516031296515918
],
"population_cv": 0.30590027610070303
},
"bits_per_byte": 8.038491115106986,
"branch_output_rms_by_sublayer": [
0.0023513585329055786,
0.0027412420604377985,
0.0023208872880786657,
0.002683233004063368,
0.002778682392090559,
0.002711885841563344,
0.002497476991266012,
0.0027249811682850122,
0.002628475893288851,
0.0027012599166482687,
0.002920303260907531,
0.002700963756069541,
0.002932395786046982,
0.0026683746837079525,
0.0032228652853518724,
0.002738022478297353,
0.003561305580660701,
0.0027370047755539417,
0.003354643704369664,
0.002654933836311102,
0.0034162327647209167,
0.002663377905264497,
0.0036349669098854065,
0.002684312639757991,
0.0035210708156228065,
0.0027356327045708895,
0.0038089773152023554,
0.002652956871315837,
0.003915763925760984,
0.002679797587916255,
0.0037244565319269896,
0.002698024269193411,
0.004162859171628952,
0.0026484597474336624,
0.004167158622294664,
0.0026680538430809975,
0.004440344404429197,
0.002654345240443945,
0.0041700867004692554,
0.002591830212622881,
0.004737259820103645,
0.002620960585772991,
0.004552462603896856,
0.002651234855875373,
0.004706811159849167,
0.0026270023081451654,
0.005186605267226696,
0.002615687670186162,
0.005580419674515724,
0.00258000404573977,
0.004905648995190859,
0.002626793459057808,
0.005227713380008936,
0.002573782578110695,
0.005349088925868273,
0.002592637902125716,
0.005235649179667234,
0.002581534907221794,
0.004925078712403774,
0.002551236655563116,
0.005810525268316269,
0.002563327783718705,
0.005564783234149218,
0.002589041367173195
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.010012756197717748,
0.008904953530422711,
0.008980942528705073,
0.007787912846174446,
0.007907551845183929,
0.008289745083748344,
0.007663833460373458,
0.007350080301903749,
0.00709564946584838,
0.006310668475383808,
0.006285078227083486,
0.006362421438950662,
0.005604026016423943,
0.005959901601525332,
0.006167718108848569,
0.005523196426059091,
0.0054641843609823,
0.00557989785603791,
0.005260407000687673,
0.004763357766419076,
0.004970214176820314,
0.004845025235106639,
0.004921304265254701,
0.004820106373346748,
0.0054804096177630225,
0.0042302744374162005,
0.004588429583035324,
0.004469681656036737,
0.004349111730454971,
0.003928502929307311,
0.004468266803351117,
0.003904642007649288
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.008362221974278682,
"first_to_last_ratio": 1.8887369414995314,
"imbalance_abs_log_ratio": 0.6359083207487641,
"last_quartile_mean": 0.004427414845626747,
"mean": 0.00600782035481319,
"normalized": [
1.6666204390908572,
1.4822269982304765,
1.4948753455169401,
1.2962958920592769,
1.3162097696294732,
1.3798257261648947,
1.2756429133626719,
1.2234187888150154,
1.1810688480662828,
1.0504089840715678,
1.0461494944748422,
1.0590232502297414,
0.9327885465041001,
0.9920239370590588,
1.0266149359654666,
0.9193344840336578,
0.9095119424808776,
0.9287724210274617,
0.875593258455752,
0.7928595538984271,
0.8272907449435313,
0.8064530809788657,
0.8191497039874,
0.8023053434820334,
0.9122126318860999,
0.7041279844573081,
0.7637428072161453,
0.7439772483303089,
0.7239084182952745,
0.6538982022256998,
0.7437417465006834,
0.6499265585598059
],
"population_cv": 0.2648157248292805
},
"depth_weights": [
{
"entropy_mean": 0.0,
"mean_weights": [
1.0
],
"sources": 1
},
{
"entropy_mean": 0.6931471824645996,
"mean_weights": [
0.5,
0.5
],
"sources": 2
},
{
"entropy_mean": 0.6931471824645996,
"mean_weights": [
0.5,
0.5
],
"sources": 2
},
{
"entropy_mean": 0.6931471824645996,
"mean_weights": [
0.5,
0.5
],
"sources": 2
},
{
"entropy_mean": 0.6931471824645996,
"mean_weights": [
0.5,
0.5
],
"sources": 2
},
{
"entropy_mean": 0.6931471824645996,
"mean_weights": [
0.5,
0.5
],
"sources": 2
},
{
"entropy_mean": 0.6931471824645996,
"mean_weights": [
0.5,
0.5
],
"sources": 2
},
{
"entropy_mean": 0.6931471824645996,
"mean_weights": [
0.5,
0.5
],
"sources": 2
},
{
"entropy_mean": 0.6931471824645996,
"mean_weights": [
0.5,
0.5
],
"sources": 2
},
{
"entropy_mean": 1.0986123085021973,
"mean_weights": [
0.3333333432674408,
0.3333333432674408,
0.3333333432674408
],
"sources": 3
},
{
"entropy_mean": 1.0986123085021973,
"mean_weights": [
0.3333333432674408,
0.3333333432674408,
0.3333333432674408
],
"sources": 3
},
{
"entropy_mean": 1.0986123085021973,
"mean_weights": [
0.3333333432674408,
0.3333333432674408,
0.3333333432674408
],
"sources": 3
},
{
"entropy_mean": 1.0986123085021973,
"mean_weights": [
0.3333333432674408,
0.3333333432674408,
0.3333333432674408
],
"sources": 3
},
{
"entropy_mean": 1.0986123085021973,
"mean_weights": [
0.3333333432674408,
0.3333333432674408,
0.3333333432674408
],
"sources": 3
},
{
"entropy_mean": 1.0986123085021973,
"mean_weights": [
0.3333333432674408,
0.3333333432674408,
0.3333333432674408
],
"sources": 3
},
{
"entropy_mean": 1.0986123085021973,
"mean_weights": [
0.3333333432674408,
0.3333333432674408,
0.3333333432674408
],
"sources": 3
},
{
"entropy_mean": 1.0986123085021973,
"mean_weights": [
0.3333333432674408,
0.3333333432674408,
0.3333333432674408
],
"sources": 3
},
{
"entropy_mean": 1.3862943649291992,
"mean_weights": [
0.25,
0.25,
0.25,
0.25
],
"sources": 4
},
{
"entropy_mean": 1.3862943649291992,
"mean_weights": [
0.25,
0.25,
0.25,
0.25
],
"sources": 4
},
{
"entropy_mean": 1.3862943649291992,
"mean_weights": [
0.25,
0.25,
0.25,
0.25
],
"sources": 4
},
{
"entropy_mean": 1.3862943649291992,
"mean_weights": [
0.25,
0.25,
0.25,
0.25
],
"sources": 4
},
{
"entropy_mean": 1.3862943649291992,
"mean_weights": [
0.25,
0.25,
0.25,
0.25
],
"sources": 4
},
{
"entropy_mean": 1.3862943649291992,
"mean_weights": [
0.25,
0.25,
0.25,
0.25
],
"sources": 4
},
{
"entropy_mean": 1.3862943649291992,
"mean_weights": [
0.25,
0.25,
0.25,
0.25
],
"sources": 4
},
{
"entropy_mean": 1.3862943649291992,
"mean_weights": [
0.25,
0.25,
0.25,
0.25
],
"sources": 4
},
{
"entropy_mean": 1.6094379425048828,
"mean_weights": [
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343
],
"sources": 5
},
{
"entropy_mean": 1.6094379425048828,
"mean_weights": [
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343
],
"sources": 5
},
{
"entropy_mean": 1.6094379425048828,
"mean_weights": [
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343
],
"sources": 5
},
{
"entropy_mean": 1.6094379425048828,
"mean_weights": [
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343
],
"sources": 5
},
{
"entropy_mean": 1.6094379425048828,
"mean_weights": [
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343
],
"sources": 5
},
{
"entropy_mean": 1.6094379425048828,
"mean_weights": [
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343
],
"sources": 5
},
{
"entropy_mean": 1.6094379425048828,
"mean_weights": [
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343
],
"sources": 5
},
{
"entropy_mean": 1.6094379425048828,
"mean_weights": [
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343,
0.20000001788139343
],
"sources": 5
},
{
"entropy_mean": 1.791759729385376,
"mean_weights": [
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592
],
"sources": 6
},
{
"entropy_mean": 1.791759729385376,
"mean_weights": [
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592
],
"sources": 6
},
{
"entropy_mean": 1.791759729385376,
"mean_weights": [
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592
],
"sources": 6
},
{
"entropy_mean": 1.791759729385376,
"mean_weights": [
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592
],
"sources": 6
},
{
"entropy_mean": 1.791759729385376,
"mean_weights": [
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592
],
"sources": 6
},
{
"entropy_mean": 1.791759729385376,
"mean_weights": [
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592
],
"sources": 6
},
{
"entropy_mean": 1.791759729385376,
"mean_weights": [
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592
],
"sources": 6
},
{
"entropy_mean": 1.791759729385376,
"mean_weights": [
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592,
0.1666666567325592
],
"sources": 6
},
{
"entropy_mean": 1.9459102153778076,
"mean_weights": [
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548
],
"sources": 7
},
{
"entropy_mean": 1.9459102153778076,
"mean_weights": [
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548
],
"sources": 7
},
{
"entropy_mean": 1.9459102153778076,
"mean_weights": [
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548
],
"sources": 7
},
{
"entropy_mean": 1.9459102153778076,
"mean_weights": [
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548
],
"sources": 7
},
{
"entropy_mean": 1.9459102153778076,
"mean_weights": [
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548
],
"sources": 7
},
{
"entropy_mean": 1.9459102153778076,
"mean_weights": [
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548
],
"sources": 7
},
{
"entropy_mean": 1.9459102153778076,
"mean_weights": [
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548
],
"sources": 7
},
{
"entropy_mean": 1.9459102153778076,
"mean_weights": [
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548,
0.1428571492433548
],
"sources": 7
},
{
"entropy_mean": 2.079441547393799,
"mean_weights": [
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125
],
"sources": 8
},
{
"entropy_mean": 2.079441547393799,
"mean_weights": [
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125
],
"sources": 8
},
{
"entropy_mean": 2.079441547393799,
"mean_weights": [
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125
],
"sources": 8
},
{
"entropy_mean": 2.079441547393799,
"mean_weights": [
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125
],
"sources": 8
},
{
"entropy_mean": 2.079441547393799,
"mean_weights": [
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125
],
"sources": 8
},
{
"entropy_mean": 2.079441547393799,
"mean_weights": [
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125
],
"sources": 8
},
{
"entropy_mean": 2.079441547393799,
"mean_weights": [
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125
],
"sources": 8
},
{
"entropy_mean": 2.079441547393799,
"mean_weights": [
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125,
0.125
],
"sources": 8
},
{
"entropy_mean": 2.1972246170043945,
"mean_weights": [
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519
],
"sources": 9
},
{
"entropy_mean": 2.1972246170043945,
"mean_weights": [
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519
],
"sources": 9
},
{
"entropy_mean": 2.1972246170043945,
"mean_weights": [
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519
],
"sources": 9
},
{
"entropy_mean": 2.1972246170043945,
"mean_weights": [
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519
],
"sources": 9
},
{
"entropy_mean": 2.1972246170043945,
"mean_weights": [
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519
],
"sources": 9
},
{
"entropy_mean": 2.1972246170043945,
"mean_weights": [
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519
],
"sources": 9
},
{
"entropy_mean": 2.1972246170043945,
"mean_weights": [
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519
],
"sources": 9
}
],
"layer_input_rms_by_sublayer": [
0.027978340163826942,
0.014031016267836094,
0.0140878576785326,
0.014151837676763535,
0.014187770895659924,
0.014265215955674648,
0.014337530359625816,
0.014401593245565891,
0.014477464370429516,
0.009722241200506687,
0.009757626801729202,
0.009786579757928848,
0.009825476445257664,
0.009839320555329323,
0.00987118761986494,
0.00996283907443285,
0.009999874979257584,
0.007562503684312105,
0.007595349568873644,
0.007635751739144325,
0.007658981718122959,
0.007660485804080963,
0.007689184043556452,
0.007726665120571852,
0.007755559403449297,
0.006238248664885759,
0.00625597033649683,
0.006290202960371971,
0.006315375678241253,
0.006327399518340826,
0.00635211868211627,
0.006382675375789404,
0.006408242508769035,
0.005390914157032967,
0.005408597644418478,
0.0054517327807843685,
0.005475745070725679,
0.005509276874363422,
0.005517482291907072,
0.0055727302096784115,
0.005585856735706329,
0.0048178439028561115,
0.004830927588045597,
0.004884926602244377,
0.004899685736745596,
0.004938251338899136,
0.004948681220412254,
0.004975184798240662,
0.004989070352166891,
0.004412420094013214,
0.004419817589223385,
0.004455919377505779,
0.004463871940970421,
0.004530393984168768,
0.004542885348200798,
0.0045819394290447235,
0.004592607729136944,
0.004127359017729759,
0.004137600306421518,
0.004185030702501535,
0.004193917382508516,
0.004221425391733646,
0.004230021964758635,
0.004281661007553339
],
"loss_nats": 5.571857452392578,
"loss_scale": 1.0,
"output_weights": {
"entropy_mean": 2.1972246170043945,
"mean_weights": [
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519,
0.11111113429069519
],
"sources": 9
},
"step": 0,
"stream_state_rms_by_sublayer": [
0.0023513585329055786,
0.0036361459642648697,
0.004408807028084993,
0.005152894649654627,
0.005783648230135441,
0.006386642809957266,
0.006751363631337881,
0.007284422870725393,
0.002628475893288851,
0.0037505170330405235,
0.004723479505628347,
0.0054016136564314365,
0.006024772301316261,
0.006585344672203064,
0.007445912808179855,
0.007922882214188576,
0.003561305580660701,
0.0044892593286931515,
0.005543973762542009,
0.006135277915745974,
0.006855612155050039,
0.007316684816032648,
0.007876631803810596,
0.0082859443500638,
0.0035210708156228065,
0.0044554853811860085,
0.0058924350887537,
0.006464940495789051,
0.007390674669295549,
0.007893112488090992,
0.008790239691734314,
0.009196000173687935,
0.004162859171628952,
0.004961193539202213,
0.006355749908834696,
0.006907128728926182,
0.007784162648022175,
0.008217990398406982,
0.00937180407345295,
0.009676996618509293,
0.004737259820103645,
0.005381465889513493,
0.006999304983764887,
0.007502692751586437,
0.008728929795324802,
0.0091153085231781,
0.010217384435236454,
0.010543142445385456,
0.005580419674515724,
0.006136673036962748,
0.008008211851119995,
0.008469858206808567,
0.010209310799837112,
0.010564134456217289,
0.012040933594107628,
0.012301022186875343,
0.005235649179667234,
0.005773921962827444,
0.007745372597128153,
0.008156762458384037,
0.010180563665926456,
0.010457510128617287,
0.011906592175364494,
0.012218860909342766
]
},
{
"activation_grad_rms_by_block": [
6.95402777637355e-05,
6.603818474104628e-05,
6.243807729333639e-05,
5.902352859266102e-05,
5.3786254284204915e-05,
5.022022014600225e-05,
4.6831908548483625e-05,
4.3414886022219434e-05,
3.877186099998653e-05,
3.6122120945947245e-05,
3.3541862649144605e-05,
3.1072035199031234e-05,
2.835266786860302e-05,
2.6703228286351077e-05,
2.5144798200926743e-05,
2.3762264390825294e-05,
2.2584137695957907e-05,
2.1571571778622456e-05,
2.0759547624038532e-05,
1.9997522031189874e-05,
1.9395642084418796e-05,
1.8848633771995082e-05,
1.8380347682978027e-05,
1.797073673515115e-05,
1.755586163199041e-05,
1.7233234757441096e-05,
1.6949628843576647e-05,
1.6709716874174774e-05,
1.630509359529242e-05,
1.611644620425068e-05,
1.594743662280962e-05,
1.580910975462757e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 5.641166717396118e-05,
"first_to_last_ratio": 3.40273807382454,
"imbalance_abs_log_ratio": 1.2245804233600555,
"last_quartile_mean": 1.6578316035520402e-05,
"mean": 3.084058886315688e-05,
"normalized": [
2.254829765809383,
2.1412750915381364,
2.0245423189025695,
1.913826251974467,
1.7440086673720954,
1.6283807150646488,
1.518515381022133,
1.407719100788124,
1.2571699318719751,
1.1712526341901288,
1.0875882687575642,
1.0075046017085245,
0.9193296533476374,
0.8658469008110146,
0.8153151132261776,
0.770486727612793,
0.7322862023214354,
0.6994539525278819,
0.6731242297658762,
0.648415700488765,
0.6288998621420432,
0.6111632256967778,
0.5959791417905044,
0.5826975877435189,
0.5692453444999938,
0.5587842318415797,
0.5495883661230995,
0.5418092679201958,
0.5286894380532074,
0.5225725836741036,
0.517092481391006,
0.5126072600226392
],
"population_cv": 0.5330470009230729
},
"activation_output_rms_by_block": [
0.005864492617547512,
0.009892112575471401,
0.014509912580251694,
0.019168438389897346,
0.006793031003326178,
0.01223041582852602,
0.017751524224877357,
0.023574935272336006,
0.008267601951956749,
0.014755276031792164,
0.02151178941130638,
0.02830074541270733,
0.009282662533223629,
0.01676790602505207,
0.02516039088368416,
0.0331411249935627,
0.010437680408358574,
0.01991143450140953,
0.029664477333426476,
0.04036801680922508,
0.01357664167881012,
0.025029508396983147,
0.03699437528848648,
0.05095698684453964,
0.015438608825206757,
0.029965175315737724,
0.04443887993693352,
0.060807984322309494,
0.019359532743692398,
0.037317242473363876,
0.05564062297344208,
0.07935244590044022
],
"activation_output_statistics": {
"first_quartile_mean": 0.01372310781152919,
"first_to_last_ratio": 0.3207078305283738,
"imbalance_abs_log_ratio": 1.137224755592113,
"last_quartile_mean": 0.04279006156139076,
"mean": 0.02613224917149637,
"normalized": [
0.224415915333617,
0.37854041994445614,
0.5552492816453899,
0.7335165933901026,
0.2599481949963819,
0.4680200218611986,
0.6792956897196492,
0.9021395409795135,
0.31637544467372514,
0.5646385787522038,
0.8231893577216564,
1.0829816150527234,
0.35521865998999624,
0.6416556766702496,
0.9628100022529916,
1.268207905720998,
0.3994176061868959,
0.7619487465750876,
1.1351673994362057,
1.5447586062839287,
0.519535903309034,
0.9578015360531602,
1.4156598249812316,
1.9499655965365867,
0.5907876020884704,
1.1466741771474498,
1.700537892674199,
2.3269326694097012,
0.7408291807047638,
1.4280149492094805,
2.1291938021979306,
3.0365716085010215
],
"population_cv": 0.6534988385002395
},
"bits_per_byte": 5.118322357706281,
"branch_output_rms_by_sublayer": [
0.003087003482505679,
0.0049620093777775764,
0.0026560353580862284,
0.004914142191410065,
0.0032712947577238083,
0.00555405905470252,
0.002734958194196224,
0.005823355168104172,
0.003094236133620143,
0.006141432095319033,
0.0030397549271583557,
0.006136075593531132,
0.0035086015705019236,
0.006357426289469004,
0.0038121570833027363,
0.0064498125575482845,
0.0047028325498104095,
0.006525229662656784,
0.004038033075630665,
0.007023691199719906,
0.004860103130340576,
0.006682637147605419,
0.005031581036746502,
0.006716069765388966,
0.005196469835937023,
0.006977807730436325,
0.005135253071784973,
0.006669311318546534,
0.006614624988287687,
0.006923797074705362,
0.005866772960871458,
0.007155857048928738,
0.006203304044902325,
0.007240671198815107,
0.007167549338191748,
0.007543859537690878,
0.007609309628605843,
0.007281317841261625,
0.008147990331053734,
0.007711030542850494,
0.00901782512664795,
0.00788753479719162,
0.008936969563364983,
0.007908415980637074,
0.009206709451973438,
0.008078642189502716,
0.01152731105685234,
0.008181036449968815,
0.010577243752777576,
0.007894126698374748,
0.011347576975822449,
0.008452145382761955,
0.011137762106955051,
0.008351319469511509,
0.012936098501086235,
0.008498509414494038,
0.013407763093709946,
0.009082214906811714,
0.013788831420242786,
0.009139914996922016,
0.015265416353940964,
0.008129997178912163,
0.0202538650482893,
0.00888112559914589
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.0002932670153511829,
0.0002289733734981176,
0.00023382191596673618,
0.00020236837455671756,
0.0002080571281121399,
0.00019123637931049836,
0.00021124835961838567,
0.00020444780259018514,
0.00020817260281626697,
0.00017921191649714578,
0.0001893830879389061,
0.0001891854298096901,
0.00018418310100442068,
0.00017033524110907422,
0.00020556753369166975,
0.00017823357128584385,
0.0001749817385153215,
0.00018220334134876856,
0.00017289789528954973,
0.00017141475273833558,
0.00017603112434287998,
0.00016603952795395127,
0.0001609386975915838,
0.00016666055126527177,
0.00016172141650657888,
0.00015460675258205218,
0.00014513294616478987,
0.00015851211030580876,
0.00015100080713109572,
0.00014721653732759288,
0.00014725738129728224,
0.00017222312685716471
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.0002216775436254954,
"first_to_last_ratio": 1.4328688617518024,
"imbalance_abs_log_ratio": 0.35967863157527663,
"last_quartile_mean": 0.00015470888477154567,
"mean": 0.000183954110636719,
"normalized": [
1.5942400761586677,
1.2447309424376207,
1.2710882902122171,
1.1001024867357483,
1.1310273382420937,
1.0395874201917712,
1.1483753142954691,
1.1114065452657267,
1.1316550747124958,
0.974220776458003,
1.02951267184732,
1.0284381749060347,
1.0012448233252798,
0.9259659407419281,
1.1174935584757544,
0.9689023565112261,
0.9512249435995668,
0.9904825758886794,
0.9398968834732723,
0.9318343153355962,
0.956929550166532,
0.9026138496130359,
0.8748850299377811,
0.9059898182672343,
0.879139998268121,
0.8404637006855296,
0.7889627780669988,
0.8616937656742324,
0.8208612822428254,
0.8002894679441159,
0.8005115014151157,
0.936228748904007
],
"population_cv": 0.1635293006829388
},
"depth_weights": [
{
"entropy_mean": 0.0,
"mean_weights": [
1.0
],
"sources": 1
},
{
"entropy_mean": 0.692298173904419,
"mean_weights": [
0.48010045289993286,
0.5198994874954224
],
"sources": 2
},
{
"entropy_mean": 0.6927915811538696,
"mean_weights": [
0.5130036473274231,
0.4869963526725769
],
"sources": 2
},
{
"entropy_mean": 0.6928644776344299,
"mean_weights": [
0.5081723928451538,
0.4918276071548462
],
"sources": 2
},
{
"entropy_mean": 0.692646861076355,
"mean_weights": [
0.5149973034858704,
0.48500269651412964
],
"sources": 2
},
{
"entropy_mean": 0.6925361156463623,
"mean_weights": [
0.5161343812942505,
0.4838656485080719
],
"sources": 2
},
{
"entropy_mean": 0.6928029656410217,
"mean_weights": [
0.5107344388961792,
0.4892655611038208
],
"sources": 2
},
{
"entropy_mean": 0.6924350261688232,
"mean_weights": [
0.5171386003494263,
0.48286139965057373
],
"sources": 2
},
{
"entropy_mean": 0.6929639577865601,
"mean_weights": [
0.5072796940803528,
0.49272027611732483
],
"sources": 2
},
{
"entropy_mean": 1.0978727340698242,
"mean_weights": [
0.34929555654525757,
0.32294148206710815,
0.3277629613876343
],
"sources": 3
},
{
"entropy_mean": 1.098320722579956,
"mean_weights": [
0.3432672619819641,
0.3268624544143677,
0.329870343208313
],
"sources": 3
},
{
"entropy_mean": 1.098150610923767,
"mean_weights": [
0.3442404866218567,
0.33018797636032104,
0.32557153701782227
],
"sources": 3
},
{
"entropy_mean": 1.0984500646591187,
"mean_weights": [
0.34007447957992554,
0.32975536584854126,
0.3301700949668884
],
"sources": 3
},
{
"entropy_mean": 1.0982636213302612,
"mean_weights": [
0.3434537351131439,
0.33091628551483154,
0.3256300091743469
],
"sources": 3
},
{
"entropy_mean": 1.0984963178634644,
"mean_weights": [
0.33822712302207947,
0.33379626274108887,
0.32797661423683167
],
"sources": 3
},
{
"entropy_mean": 1.0980933904647827,
"mean_weights": [
0.34460341930389404,
0.33179372549057007,
0.3236028254032135
],
"sources": 3
},
{
"entropy_mean": 1.0985530614852905,
"mean_weights": [
0.3358767628669739,
0.3345070779323578,
0.32961612939834595
],
"sources": 3
},
{
"entropy_mean": 1.3860312700271606,
"mean_weights": [
0.25749218463897705,
0.24992284178733826,
0.24571210145950317,
0.24687285721302032
],
"sources": 4
},
{
"entropy_mean": 1.3862019777297974,
"mean_weights": [
0.25314557552337646,
0.25236955285072327,
0.24678170680999756,
0.2477031946182251
],
"sources": 4
},
{
"entropy_mean": 1.385892391204834,
"mean_weights": [
0.25815317034721375,
0.25234687328338623,
0.2448991984128952,
0.24460072815418243
],
"sources": 4
},
{
"entropy_mean": 1.3861613273620605,
"mean_weights": [
0.25378289818763733,
0.25339171290397644,
0.24767157435417175,
0.24515381455421448
],
"sources": 4
},
{
"entropy_mean": 1.3855845928192139,
"mean_weights": [
0.2592235505580902,
0.2543467879295349,
0.2462996244430542,
0.24013003706932068
],
"sources": 4
},
{
"entropy_mean": 1.3861030340194702,
"mean_weights": [
0.25138330459594727,
0.25586679577827454,
0.24899962544441223,
0.24375030398368835
],
"sources": 4
},
{
"entropy_mean": 1.3856689929962158,
"mean_weights": [
0.2574368119239807,
0.25204822421073914,
0.2460038810968399,
0.24451106786727905
],
"sources": 4
},
{
"entropy_mean": 1.386252999305725,
"mean_weights": [
0.25002047419548035,
0.25257009267807007,
0.2502796947956085,
0.24712975323200226
],
"sources": 4
},
{
"entropy_mean": 1.608945608139038,
"mean_weights": [
0.20495271682739258,
0.2066430151462555,
0.2000001072883606,
0.19580814242362976,
0.19259600341320038
],
"sources": 5
},
{
"entropy_mean": 1.6093038320541382,
"mean_weights": [
0.20154938101768494,
0.20420850813388824,
0.2010108083486557,
0.19671520590782166,
0.19651609659194946
],
"sources": 5
},
{
"entropy_mean": 1.6085759401321411,
"mean_weights": [
0.20549985766410828,
0.209174245595932,
0.20157773792743683,
0.19417354464530945,
0.18957462906837463
],
"sources": 5
},
{
"entropy_mean": 1.6093413829803467,
"mean_weights": [
0.20104944705963135,
0.20404505729675293,
0.2002137303352356,
0.19850106537342072,
0.1961907148361206
],
"sources": 5
},
{
"entropy_mean": 1.6080443859100342,
"mean_weights": [
0.20538949966430664,
0.21283748745918274,
0.20341980457305908,
0.19302619993686676,
0.18532699346542358
],
"sources": 5
},
{
"entropy_mean": 1.6093816757202148,
"mean_weights": [
0.2010989934206009,
0.20253092050552368,
0.19950580596923828,
0.19837552309036255,
0.1984887421131134
],
"sources": 5
},
{
"entropy_mean": 1.6076874732971191,
"mean_weights": [
0.20581099390983582,
0.21552260220050812,
0.20295079052448273,
0.19259145855903625,
0.18312416970729828
],
"sources": 5
},
{
"entropy_mean": 1.609355092048645,
"mean_weights": [
0.20059309899806976,
0.203348308801651,
0.2011984884738922,
0.1984248012304306,
0.19643528759479523
],
"sources": 5
},
{
"entropy_mean": 1.7901467084884644,
"mean_weights": [
0.17125868797302246,
0.18124164640903473,
0.17160004377365112,
0.1620369851589203,
0.15427924692630768,
0.15958338975906372
],
"sources": 6
},
{
"entropy_mean": 1.7916979789733887,
"mean_weights": [
0.16689208149909973,
0.16927623748779297,
0.1683787703514099,
0.16569086909294128,
0.1651553511619568,
0.1646067053079605
],
"sources": 6
},
{
"entropy_mean": 1.789665699005127,
"mean_weights": [
0.1720278263092041,
0.18269017338752747,
0.17356479167938232,
0.16241812705993652,
0.15660038590431213,
0.15269872546195984
],
"sources": 6
},
{
"entropy_mean": 1.7917176485061646,
"mean_weights": [
0.16644492745399475,
0.16854418814182281,
0.16665583848953247,
0.16677430272102356,
0.16461026668548584,
0.16697049140930176
],
"sources": 6
},
{
"entropy_mean": 1.7893871068954468,
"mean_weights": [
0.17303556203842163,
0.18359911441802979,
0.17276406288146973,
0.16385790705680847,
0.1554735004901886,
0.15126986801624298
],
"sources": 6
},
{
"entropy_mean": 1.7917413711547852,
"mean_weights": [
0.16618916392326355,
0.16740113496780396,
0.16675519943237305,
0.1668717861175537,
0.1653555929660797,
0.16742712259292603
],
"sources": 6
},
{
"entropy_mean": 1.7882882356643677,
"mean_weights": [
0.17288538813591003,
0.18687990307807922,
0.17607632279396057,
0.162863090634346,
0.15389539301395416,
0.14739990234375
],
"sources": 6
},
{
"entropy_mean": 1.7917335033416748,
"mean_weights": [
0.16660982370376587,
0.16799160838127136,
0.16742658615112305,
0.1664734035730362,
0.16542598605155945,
0.1660725623369217
],
"sources": 6
},
{
"entropy_mean": 1.941649079322815,
"mean_weights": [
0.15037035942077637,
0.16432975232601166,
0.15334898233413696,
0.1418340504169464,
0.13212506473064423,
0.12697139382362366,
0.13102038204669952
],
"sources": 7
},
{
"entropy_mean": 1.9458887577056885,
"mean_weights": [
0.1431310474872589,
0.14324922859668732,
0.14309079945087433,
0.142366424202919,
0.14222830533981323,
0.14392223954200745,
0.14201194047927856
],
"sources": 7
},
{
"entropy_mean": 1.9409761428833008,
"mean_weights": [
0.1505419909954071,
0.16567251086235046,
0.1550290882587433,
0.14243587851524353,
0.13249853253364563,
0.12776729464530945,
0.12605470418930054
],
"sources": 7
},
{
"entropy_mean": 1.9458999633789062,
"mean_weights": [
0.14249610900878906,
0.14256571233272552,
0.1435922086238861,
0.14210085570812225,
0.14355877041816711,
0.1427185833454132,
0.14296773076057434
],
"sources": 7
},
{
"entropy_mean": 1.941575527191162,
"mean_weights": [
0.15046924352645874,
0.1640411913394928,
0.15395450592041016,
0.14198558032512665,
0.13400861620903015,
0.1288735270500183,
0.126667320728302
],
"sources": 7
},
{
"entropy_mean": 1.9458911418914795,
"mean_weights": [
0.14260169863700867,
0.14355596899986267,
0.1428615152835846,
0.14244115352630615,
0.14214617013931274,
0.14205437898635864,
0.14433911442756653
],
"sources": 7
},
{
"entropy_mean": 1.9400205612182617,
"mean_weights": [
0.1507154405117035,
0.16689133644104004,
0.15648439526557922,
0.1427520215511322,
0.13353285193443298,
0.12712189555168152,
0.12250205874443054
],
"sources": 7
},
{
"entropy_mean": 1.9458441734313965,
"mean_weights": [
0.1422245353460312,
0.14142528176307678,
0.14198407530784607,
0.14165860414505005,
0.14288201928138733,
0.14370128512382507,
0.14612418413162231
],
"sources": 7
},
{
"entropy_mean": 2.0741422176361084,
"mean_weights": [
0.13300861418247223,
0.14654949307441711,
0.1379268318414688,
0.12711459398269653,
0.1185426190495491,
0.1134917140007019,
0.10956315696239471,
0.11380299925804138
],
"sources": 8
},
{
"entropy_mean": 2.0794055461883545,
"mean_weights": [
0.12428364902734756,
0.1234029084444046,
0.12436957657337189,
0.12428068369626999,
0.12555459141731262,
0.12575605511665344,
0.1263071894645691,
0.1260453760623932
],
"sources": 8
},
{
"entropy_mean": 2.071772813796997,
"mean_weights": [
0.134201318025589,
0.1495765745639801,
0.14145472645759583,
0.12840476632118225,
0.11872729659080505,
0.11272042989730835,
0.10793697834014893,
0.1069779098033905
],
"sources": 8
},
{
"entropy_mean": 2.079404592514038,
"mean_weights": [
0.12455718219280243,
0.12374450266361237,
0.12391386926174164,
0.12458803504705429,
0.1251734495162964,
0.1251622885465622,
0.1266230344772339,
0.1262376308441162
],
"sources": 8
},
{
"entropy_mean": 2.0732176303863525,
"mean_weights": [
0.13281872868537903,
0.1473538726568222,
0.13968363404273987,
0.12848341464996338,
0.12016644328832626,
0.11505335569381714,
0.10907343029975891,
0.1073671206831932
],
"sources": 8
},
{
"entropy_mean": 2.0794081687927246,
"mean_weights": [
0.12450858950614929,
0.12424902617931366,
0.12409761548042297,
0.12468484044075012,
0.12504076957702637,
0.1255234032869339,
0.12684062123298645,
0.12505511939525604
],
"sources": 8
},
{
"entropy_mean": 2.072478771209717,
"mean_weights": [
0.13302236795425415,
0.14877822995185852,
0.14031866192817688,
0.12882456183433533,
0.11896270513534546,
0.11445862054824829,
0.11000129580497742,
0.10563355684280396
],
"sources": 8
},
{
"entropy_mean": 2.079291820526123,
"mean_weights": [
0.12426379323005676,
0.12249694019556046,
0.12262053787708282,
0.12315947562456131,
0.1255876123905182,
0.126260906457901,
0.1273583620786667,
0.12825237214565277
],
"sources": 8
},
{
"entropy_mean": 2.1905503273010254,
"mean_weights": [
0.11892962455749512,
0.13242119550704956,
0.12570281326770782,
0.11684690415859222,
0.10851435363292694,
0.10355254262685776,
0.09824737906455994,
0.0950399786233902,
0.10074522346258163
],
"sources": 9
},
{
"entropy_mean": 2.1971964836120605,
"mean_weights": [
0.11044953763484955,
0.11010265350341797,
0.1108412891626358,
0.11044170707464218,
0.11131919175386429,
0.11133328080177307,
0.11097346246242523,
0.11225177347660065,
0.11228710412979126
],
"sources": 9
},
{
"entropy_mean": 2.1902995109558105,
"mean_weights": [
0.11926670372486115,
0.13237422704696655,
0.12644407153129578,
0.11611802130937576,
0.11051423847675323,
0.10466914623975754,
0.09992008656263351,
0.09535142779350281,
0.09534208476543427
],
"sources": 9
},
{
"entropy_mean": 2.197113037109375,
"mean_weights": [
0.11037027835845947,
0.10917964577674866,
0.10961420089006424,
0.10947252064943314,
0.11109665036201477,
0.11174711585044861,
0.1118166521191597,
0.11308293044567108,
0.11361998319625854
],
"sources": 9
},
{
"entropy_mean": 2.1894307136535645,
"mean_weights": [
0.1198490560054779,
0.13425537943840027,
0.12682178616523743,
0.11624304950237274,
0.10967148095369339,
0.10377678275108337,
0.09994889795780182,
0.0946216955780983,
0.09481187164783478
],
"sources": 9
},
{
"entropy_mean": 2.196977138519287,
"mean_weights": [
0.10978136211633682,
0.1075383648276329,
0.10839901119470596,
0.11003260314464569,
0.11053797602653503,
0.11152469366788864,
0.11391061544418335,
0.11424341052770615,
0.11403194814920425
],
"sources": 9
},
{
"entropy_mean": 2.1905651092529297,
"mean_weights": [
0.11943811178207397,
0.13202990591526031,
0.12506362795829773,
0.11625386774539948,
0.11043960601091385,
0.1047937273979187,
0.1010652631521225,
0.09624719619750977,
0.0946686640381813
],
"sources": 9
}
],
"layer_input_rms_by_sublayer": [
0.02856997773051262,
0.013703026808798313,
0.015031684190034866,
0.014947476796805859,
0.015670204535126686,
0.015775443986058235,
0.016530683264136314,
0.016744175925850868,
0.01773575134575367,
0.012103146873414516,
0.01292925700545311,
0.013081531040370464,
0.014021075330674648,
0.01416850183159113,
0.015247474424540997,
0.015494133345782757,
0.01669454388320446,
0.012773395515978336,
0.01373021025210619,
0.013903867453336716,
0.01499613095074892,
0.015176740474998951,
0.01623532734811306,
0.016410861164331436,
0.01758282259106636,
0.014334342442452908,
0.015234442427754402,
0.015468464232981205,
0.01637510396540165,
0.01670125499367714,
0.01772587187588215,
0.017849057912826538,
0.01901337131857872,
0.01606663130223751,
0.017032917588949203,
0.0173187293112278,
0.01834850199520588,
0.018571525812149048,
0.019697511568665504,
0.019770435988903046,
0.021141810342669487,
0.018272221088409424,
0.019469235092401505,
0.01950262486934662,
0.02084389142692089,
0.02077464945614338,
0.022254284471273422,
0.0220524650067091,
0.02407064661383629,
0.020764723420143127,
0.022380243986845016,
0.02203267626464367,
0.023947052657604218,
0.023422541096806526,
0.025450291112065315,
0.024811791256070137,
0.027382872998714447,
0.023658527061343193,
0.025761734694242477,
0.025185640901327133,
0.02756396308541298,
0.026624467223882675,
0.02946169301867485,
0.02854793891310692
],
"loss_nats": 3.54775071144104,
"loss_scale": 1.0,
"output_weights": {
"entropy_mean": 2.1935477256774902,
"mean_weights": [
0.10072024166584015,
0.09480113536119461,
0.10205776989459991,
0.11059839278459549,
0.1157325804233551,
0.11904929578304291,
0.11986880004405975,
0.11937490850687027,
0.11779685318470001
],
"sources": 9
},
"step": 100,
"stream_state_rms_by_sublayer": [
0.003087003482505679,
0.005864492617547512,
0.006683253683149815,
0.009892112575471401,
0.010629965923726559,
0.014509912580251694,
0.01484723947942257,
0.019168438389897346,
0.003094236133620143,
0.006793031003326178,
0.007696541491895914,
0.01223041582852602,
0.012912816368043423,
0.017751524224877357,
0.018475037068128586,
0.023574935272336006,
0.0047028325498104095,
0.008267601951956749,
0.0095678074285388,
0.014755276031792164,
0.01631263643503189,
0.02151178941130638,
0.023062339052557945,
0.02830074541270733,
0.005196469835937023,
0.009282662533223629,
0.011978921480476856,
0.01676790602505207,
0.01994149386882782,
0.02516039088368416,
0.027569252997636795,
0.0331411249935627,
0.006203304044902325,
0.010437680408358574,
0.014378104358911514,
0.01991143450140953,
0.024205319583415985,
0.029664477333426476,
0.03444278985261917,
0.04036801680922508,
0.00901782512664795,
0.01357664167881012,
0.01929495297372341,
0.025029508396983147,
0.030904190614819527,
0.03699437528848648,
0.04446368291974068,
0.05095698684453964,
0.010577243752777576,
0.015438608825206757,
0.023803669959306717,
0.029965175315737724,
0.03807948902249336,
0.04443887993693352,
0.05412018671631813,
0.060807984322309494,
0.013407763093709946,
0.019359532743692398,
0.030261604115366936,
0.037317242473363876,
0.04958754777908325,
0.05564062297344208,
0.07224015146493912,
0.07935244590044022
]
},
{
"activation_grad_rms_by_block": [
0.00014133188233245164,
0.0001488620473537594,
0.00014702342741657048,
0.0001400294277118519,
0.00013315585965756327,
0.00011939816613448784,
0.0001078194472938776,
9.923135075950995e-05,
8.953637006925419e-05,
8.09328121249564e-05,
7.468360854545608e-05,
6.987994856899604e-05,
7.115858170436695e-05,
6.467441562563181e-05,
5.974166560918093e-05,
5.531677743420005e-05,
5.705624425900169e-05,
5.295562368701212e-05,
4.8012458137236536e-05,
4.413425631355494e-05,
5.2346804295666516e-05,
4.681985228671692e-05,
4.172014814685099e-05,
3.8159589166752994e-05,
3.1379553547594696e-05,
3.02761654893402e-05,
2.9319184250198305e-05,
2.8667578590102494e-05,
2.2355405235430226e-05,
2.2064739823690616e-05,
2.201247843913734e-05,
2.2052488930057734e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 0.000129606451082509,
"first_to_last_ratio": 4.9818074922726145,
"imbalance_abs_log_ratio": 1.6057927754415957,
"last_quartile_mean": 2.601594928819395e-05,
"mean": 6.850338621688934e-05,
"normalized": [
2.0631371693799077,
2.173061151786652,
2.146221311616304,
2.04412417319785,
1.9437850741564366,
1.7429527608527258,
1.5739287245233284,
1.448561249882341,
1.3070356812110293,
1.181442503713818,
1.0902177639657034,
1.0200948073975256,
1.038760061861336,
0.9441053821903842,
0.8720979926456799,
0.8075042780959905,
0.832896699126015,
0.7730365841967101,
0.7008771505867426,
0.6442638641806842,
0.7641491492103867,
0.6834677068149604,
0.6090231512754765,
0.5570467574542299,
0.45807302792658366,
0.4419659692950432,
0.4279961308389998,
0.4184841096663127,
0.3263401485679923,
0.32209706763737483,
0.3213341654300619,
0.32191823131541414
],
"population_cv": 0.5858213540607353
},
"activation_output_rms_by_block": [
0.021054552868008614,
0.038058556616306305,
0.04883526638150215,
0.05791458487510681,
0.016990283504128456,
0.027475843206048012,
0.03445403650403023,
0.040619369596242905,
0.019055280834436417,
0.029903952032327652,
0.037492990493774414,
0.04475866258144379,
0.0285621490329504,
0.03507128730416298,
0.04228900745511055,
0.05219251662492752,
0.027010982856154442,
0.04389137029647827,
0.054182786494493484,
0.06662839651107788,
0.03837457299232483,
0.041143495589494705,
0.05716610327363014,
0.07329611480236053,
0.028997479006648064,
0.053451184183359146,
0.08026835322380066,
0.10854664444923401,
0.027061885222792625,
0.05247335880994797,
0.08323372155427933,
0.11460508406162262
],
"activation_output_statistics": {
"first_quartile_mean": 0.035675311693921685,
"first_to_last_ratio": 0.5202021080271609,
"imbalance_abs_log_ratio": 0.6535378736360024,
"last_quartile_mean": 0.06857971381396055,
"mean": 0.047658121038693935,
"normalized": [
0.4417831087153916,
0.798574425236075,
1.0246997849926245,
1.2152091524566315,
0.3565034276180115,
0.5765196488493577,
0.7229415628043072,
0.8523074076559539,
0.3998328179779751,
0.6274681288431082,
0.7867072741565622,
0.9391612931005807,
0.5993133680146686,
0.7358932022453922,
0.8873410561187638,
1.095144237485823,
0.5667655851187262,
0.92096308750491,
1.136905637771622,
1.398049168933493,
0.8052053281993276,
0.8633050295056751,
1.1995039256209161,
1.5379564532737442,
0.6084478022770731,
1.121554585418124,
1.6842534173479116,
2.277610658655726,
0.567833658419326,
1.1010370880409766,
1.746475096798326,
2.404733580842896
],
"population_cv": 0.48875915937608194
},
"bits_per_byte": 3.059021014366605,
"branch_output_rms_by_sublayer": [
0.01139580737799406,
0.01652366667985916,
0.014748631045222282,
0.013563322834670544,
0.012339585460722446,
0.012220539152622223,
0.0120800556614995,
0.0097816726192832,
0.01439172774553299,
0.008202088065445423,
0.0171960536390543,
0.008006753399968147,
0.01820177212357521,
0.008192610926926136,
0.01519495714455843,
0.008106541819870472,
0.016362585127353668,
0.008609342388808727,
0.015944689512252808,
0.008666405454277992,
0.017555685713887215,
0.009109876118600368,
0.01772458106279373,
0.008975221775472164,
0.025544000789523125,
0.009615846909582615,
0.019797103479504585,
0.009856111370027065,
0.02002105489373207,
0.009463186375796795,
0.017730148509144783,
0.009926749393343925,
0.022795414552092552,
0.010939708910882473,
0.02134518511593342,
0.010758070275187492,
0.021650495007634163,
0.011536303907632828,
0.03255617991089821,
0.012654256075620651,
0.033973678946495056,
0.014695252291858196,
0.031387828290462494,
0.016267452389001846,
0.019825322553515434,
0.01879013702273369,
0.013387269340455532,
0.02227928675711155,
0.015395586378872395,
0.021389417350292206,
0.011897368356585503,
0.023454682901501656,
0.014087310060858727,
0.024781694635748863,
0.014466723427176476,
0.026336129754781723,
0.006911881268024445,
0.026170482859015465,
0.005224450025707483,
0.026235800236463547,
0.00758614856749773,
0.030007388442754745,
0.005234629847109318,
0.031163375824689865
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.00048079443816738296,
0.0004352800282264352,
0.00039416580024316875,
0.0003828745093102654,
0.0003643213588911513,
0.0002989310042525185,
0.0002673721517191021,
0.00023887606182770654,
0.0002716851528017292,
0.00026407625379019375,
0.00023115090959535868,
0.00019271062698547964,
0.00023181255047936123,
0.00019176941572124543,
0.00020689480536159778,
0.0001935784151883457,
0.00017612030874433946,
0.00017372524780256995,
0.0001906158264031438,
0.00020699034417808569,
0.0002469606938485226,
0.00020806313134476826,
0.00017307229934302834,
0.00014129096626273445,
0.00011634997010006574,
0.00010725656087507846,
0.00011122939555071074,
0.00011004876520390145,
7.494181919924042e-05,
7.493373834181984e-05,
7.712439386045853e-05,
7.424519704760952e-05
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.00035782691907971636,
"first_to_last_ratio": 3.8366182378544447,
"imbalance_abs_log_ratio": 1.3445913113590247,
"last_quartile_mean": 9.326623002236059e-05,
"mean": 0.0002159144418958475,
"normalized": [
2.226782210331757,
2.015983851772199,
1.8255647782620577,
1.7732695689478515,
1.6873413176636518,
1.384488233523149,
1.2383245389767708,
1.1063459198478962,
1.258300048927641,
1.2230597058328248,
1.070567154127008,
0.892532362788586,
1.0736315201703026,
0.8881731765481019,
0.9582258766247762,
0.896551492751577,
0.8156948983954307,
0.8046022594744787,
0.8828303689620391,
0.9586683612295531,
1.1437896033265396,
0.9636369365469933,
0.801578152083568,
0.6543840468572741,
0.5388707169305075,
0.49675491798189547,
0.5151549594091785,
0.5096869122677149,
0.34709034996089216,
0.3470529237593392,
0.3571988662882573,
0.34386396943018666
],
"population_cv": 0.48652454891086055
},
"depth_weights": [
{
"entropy_mean": 0.0,
"mean_weights": [
1.0
],
"sources": 1
},
{
"entropy_mean": 0.6572662591934204,
"mean_weights": [
0.3911391496658325,
0.6088608503341675
],
"sources": 2
},
{
"entropy_mean": 0.6751130819320679,
"mean_weights": [
0.44147175550460815,
0.5585282444953918
],
"sources": 2
},
{
"entropy_mean": 0.6885842084884644,
"mean_weights": [
0.4896422326564789,
0.5103577375411987
],
"sources": 2
},
{
"entropy_mean": 0.668085515499115,
"mean_weights": [
0.4332469701766968,
0.5667530298233032
],
"sources": 2
},
{
"entropy_mean": 0.6886032223701477,
"mean_weights": [
0.51185142993927,
0.4881485104560852
],
"sources": 2
},
{
"entropy_mean": 0.6789880990982056,
"mean_weights": [
0.4699174761772156,
0.5300825834274292
],
"sources": 2
},
{
"entropy_mean": 0.6772009134292603,
"mean_weights": [
0.4395230710506439,
0.5604768991470337
],
"sources": 2
},
{
"entropy_mean": 0.6781555414199829,
"mean_weights": [
0.4541339576244354,
0.5458660125732422
],
"sources": 2
},
{
"entropy_mean": 1.0917184352874756,
"mean_weights": [
0.30861830711364746,
0.355538010597229,
0.33584368228912354
],
"sources": 3
},
{
"entropy_mean": 1.0860013961791992,
"mean_weights": [
0.30104559659957886,
0.3248867988586426,
0.37406760454177856
],
"sources": 3
},
{
"entropy_mean": 1.0827364921569824,
"mean_weights": [
0.2735002934932709,
0.34776103496551514,
0.3787387013435364
],
"sources": 3
},
{
"entropy_mean": 1.0724949836730957,
"mean_weights": [
0.2886848449707031,
0.3024986684322357,
0.40881651639938354
],
"sources": 3
},
{
"entropy_mean": 1.0897927284240723,
"mean_weights": [
0.2973601222038269,
0.3298739790916443,
0.3727658987045288
],
"sources": 3
},
{
"entropy_mean": 1.0697213411331177,
"mean_weights": [
0.30285951495170593,
0.28946101665496826,
0.4076794683933258
],
"sources": 3
},
{
"entropy_mean": 1.0826778411865234,
"mean_weights": [
0.2862996757030487,
0.3235614597797394,
0.3901388645172119
],
"sources": 3
},
{
"entropy_mean": 1.0579445362091064,
"mean_weights": [
0.28999701142311096,
0.2830814719200134,
0.4269214868545532
],
"sources": 3
},
{
"entropy_mean": 1.3809051513671875,
"mean_weights": [
0.23795948922634125,
0.2348281741142273,
0.26517200469970703,
0.26204031705856323
],
"sources": 4
},
{
"entropy_mean": 1.3605632781982422,
"mean_weights": [
0.24186556041240692,
0.22099661827087402,
0.2607640027999878,
0.27637380361557007
],
"sources": 4
},
{
"entropy_mean": 1.3829561471939087,
"mean_weights": [
0.23731295764446259,
0.2382979840040207,
0.2565648853778839,
0.2678241431713104
],
"sources": 4
},
{
"entropy_mean": 1.35977041721344,
"mean_weights": [
0.23858636617660522,
0.21978360414505005,
0.2459542453289032,
0.29567578434944153
],
"sources": 4
},
{
"entropy_mean": 1.3790044784545898,
"mean_weights": [
0.24452988803386688,
0.2358427345752716,
0.24750955402851105,
0.27211782336235046
],
"sources": 4
},
{
"entropy_mean": 1.3556556701660156,
"mean_weights": [
0.2410905957221985,
0.22088873386383057,
0.23317404091358185,
0.3048466145992279
],
"sources": 4
},
{
"entropy_mean": 1.3804312944412231,
"mean_weights": [
0.26568183302879333,
0.22685569524765015,
0.2384241223335266,
0.2690383493900299
],
"sources": 4
},
{
"entropy_mean": 1.3533532619476318,
"mean_weights": [
0.24537262320518494,
0.2171444594860077,
0.23335666954517365,
0.30412623286247253
],
"sources": 4
},
{
"entropy_mean": 1.6053518056869507,
"mean_weights": [
0.20177139341831207,
0.18913303315639496,
0.18395328521728516,
0.1997503787279129,
0.22539189457893372
],
"sources": 5
},
{
"entropy_mean": 1.585533618927002,
"mean_weights": [
0.19244730472564697,
0.1780712604522705,
0.17078758776187897,
0.20516318082809448,
0.25353068113327026
],
"sources": 5
},
{
"entropy_mean": 1.6022908687591553,
"mean_weights": [
0.1985350400209427,
0.1897844821214676,
0.17792725563049316,
0.1998080015182495,
0.23394520580768585
],
"sources": 5
},
{
"entropy_mean": 1.5833771228790283,
"mean_weights": [
0.1926397681236267,
0.1801614761352539,
0.17439623177051544,
0.19375719130039215,
0.2590453326702118
],
"sources": 5
},
{
"entropy_mean": 1.601109504699707,
"mean_weights": [
0.19747966527938843,
0.19028550386428833,
0.18151578307151794,
0.19478440284729004,
0.23593464493751526
],
"sources": 5
},
{
"entropy_mean": 1.5755603313446045,
"mean_weights": [
0.1969345360994339,
0.17712506651878357,
0.16844677925109863,
0.18347150087356567,
0.27402210235595703
],
"sources": 5
},
{
"entropy_mean": 1.5980476140975952,
"mean_weights": [
0.19043894112110138,
0.1923028826713562,
0.1810823678970337,
0.19125500321388245,
0.24492081999778748
],
"sources": 5
},
{
"entropy_mean": 1.5842485427856445,
"mean_weights": [
0.1964084804058075,
0.17743000388145447,
0.17621047794818878,
0.18783687055110931,
0.26211416721343994
],
"sources": 5
},
{
"entropy_mean": 1.7856926918029785,
"mean_weights": [
0.15997162461280823,
0.16431675851345062,
0.15234553813934326,
0.15752777457237244,
0.18046650290489197,
0.18537181615829468
],
"sources": 6
},
{
"entropy_mean": 1.7751184701919556,
"mean_weights": [
0.16064736247062683,
0.15155571699142456,
0.14907033741474152,
0.14832919836044312,
0.18695974349975586,
0.2034376710653305
],
"sources": 6
},
{
"entropy_mean": 1.7829411029815674,
"mean_weights": [
0.14932186901569366,
0.16724151372909546,
0.15592646598815918,
0.1555134654045105,
0.17433607578277588,
0.19766061007976532
],
"sources": 6
},
{
"entropy_mean": 1.7661566734313965,
"mean_weights": [
0.165928453207016,
0.143815815448761,
0.1375015527009964,
0.1444939523935318,
0.17902101576328278,
0.22923919558525085
],
"sources": 6
},
{
"entropy_mean": 1.7807259559631348,
"mean_weights": [
0.14958028495311737,
0.16952179372310638,
0.15695258975028992,
0.1499582827091217,
0.1711740493774414,
0.20281299948692322
],
"sources": 6
},
{
"entropy_mean": 1.781292200088501,
"mean_weights": [
0.1682473123073578,
0.15929323434829712,
0.14874719083309174,
0.15058940649032593,
0.1693388819694519,
0.20378397405147552
],
"sources": 6
},
{
"entropy_mean": 1.7818934917449951,
"mean_weights": [
0.14812925457954407,
0.18386004865169525,
0.16644306480884552,
0.15062814950942993,
0.15809310972690582,
0.19284634292125702
],
"sources": 6
},
{
"entropy_mean": 1.77958083152771,
"mean_weights": [
0.19125157594680786,
0.15093471109867096,
0.14643779397010803,
0.1490234136581421,
0.16375312209129333,
0.19859938323497772
],
"sources": 6
},
{
"entropy_mean": 1.9289655685424805,
"mean_weights": [
0.12320417165756226,
0.15015369653701782,
0.13329030573368073,
0.12274879962205887,
0.13076716661453247,
0.1441647708415985,
0.19567111134529114
],
"sources": 7
},
{
"entropy_mean": 1.9340521097183228,
"mean_weights": [
0.14550700783729553,
0.14042077958583832,
0.13418063521385193,
0.1260203719139099,
0.12906627357006073,
0.1355920135974884,
0.18921294808387756
],
"sources": 7
},
{
"entropy_mean": 1.9230798482894897,
"mean_weights": [
0.12067099660634995,
0.15888366103172302,
0.14002522826194763,
0.12021138519048691,
0.12209220230579376,
0.13571104407310486,
0.20240548253059387
],
"sources": 7
},
{
"entropy_mean": 1.9308109283447266,
"mean_weights": [
0.1450628787279129,
0.14527732133865356,
0.13644321262836456,
0.12497064471244812,
0.12634633481502533,
0.1283716857433319,
0.19352790713310242
],
"sources": 7
},
{
"entropy_mean": 1.9273775815963745,
"mean_weights": [
0.1260491907596588,
0.16657930612564087,
0.14038512110710144,
0.12195964902639389,
0.12248437106609344,
0.13317090272903442,
0.18937145173549652
],
"sources": 7
},
{
"entropy_mean": 1.918284296989441,
"mean_weights": [
0.13994047045707703,
0.15779416263103485,
0.13759920001029968,
0.11949826776981354,
0.11786683648824692,
0.12190048396587372,
0.20540057122707367
],
"sources": 7
},
{
"entropy_mean": 1.9060933589935303,
"mean_weights": [
0.11515206843614578,
0.18610316514968872,
0.15217435359954834,
0.11993157863616943,
0.11246243864297867,
0.12337306141853333,
0.19080331921577454
],
"sources": 7
},
{
"entropy_mean": 1.9199097156524658,
"mean_weights": [
0.13630829751491547,
0.1662534922361374,
0.14684012532234192,
0.12322510778903961,
0.12947364151477814,
0.1381947100162506,
0.15970461070537567
],
"sources": 7
},
{
"entropy_mean": 2.0490760803222656,
"mean_weights": [
0.10742150247097015,
0.1722080111503601,
0.14153704047203064,
0.1159135103225708,
0.10868977755308151,
0.11180940270423889,
0.1477845162153244,
0.09463626146316528
],
"sources": 8
},
{
"entropy_mean": 2.051286220550537,
"mean_weights": [
0.1131708174943924,
0.16089829802513123,
0.14448881149291992,
0.11956015229225159,
0.10795272141695023,
0.11352235078811646,
0.13960374891757965,
0.10080308467149734
],
"sources": 8
},
{
"entropy_mean": 2.0247201919555664,
"mean_weights": [
0.1011081114411354,
0.1894242763519287,
0.1531166434288025,
0.11718504130840302,
0.10284577310085297,
0.10429200530052185,
0.14699342846870422,
0.08503472805023193
],
"sources": 8
},
{
"entropy_mean": 2.047537326812744,
"mean_weights": [
0.1111423671245575,
0.16533835232257843,
0.14553973078727722,
0.12043843418359756,
0.10876712203025818,
0.11067016422748566,
0.13975468277931213,
0.09834913164377213
],
"sources": 8
},
{
"entropy_mean": 2.0282247066497803,
"mean_weights": [
0.100978784263134,
0.18681633472442627,
0.15267831087112427,
0.11643697321414948,
0.10204557329416275,
0.10493149608373642,
0.14740025997161865,
0.08871228992938995
],
"sources": 8
},
{
"entropy_mean": 2.053830623626709,
"mean_weights": [
0.11781682819128036,
0.1550440490245819,
0.1398259550333023,
0.11913557350635529,
0.10805961489677429,
0.11164884269237518,
0.13677509129047394,
0.11169403791427612
],
"sources": 8
},
{
"entropy_mean": 2.017721176147461,
"mean_weights": [
0.09720280021429062,
0.19308976829051971,
0.16317345201969147,
0.11856384575366974,
0.10023905336856842,
0.10130266845226288,
0.14337101578712463,
0.08305738866329193
],
"sources": 8
},
{
"entropy_mean": 2.0608584880828857,
"mean_weights": [
0.12434182316064835,
0.14232873916625977,
0.13082580268383026,
0.11684279143810272,
0.10329097509384155,
0.10883669555187225,
0.1425117552280426,
0.13102145493030548
],
"sources": 8
},
{
"entropy_mean": 2.142296314239502,
"mean_weights": [
0.08636107295751572,
0.16583400964736938,
0.14278221130371094,
0.1055997684597969,
0.08843953907489777,
0.08854378759860992,
0.12657995522022247,
0.07546606659889221,
0.12039357423782349
],
"sources": 9
},
{
"entropy_mean": 2.191580057144165,
"mean_weights": [
0.1060912162065506,
0.12397187203168869,
0.11601399630308151,
0.1091354638338089,
0.10782144963741302,
0.10360677540302277,
0.12169313430786133,
0.10359254479408264,
0.10807354003190994
],
"sources": 9
},
{
"entropy_mean": 2.1432788372039795,
"mean_weights": [
0.08927536010742188,
0.16873395442962646,
0.14930182695388794,
0.112123504281044,
0.09766747802495956,
0.0965762585401535,
0.1252625584602356,
0.07491002976894379,
0.08614903688430786
],
"sources": 9
},
{
"entropy_mean": 2.1724743843078613,
"mean_weights": [
0.1105542778968811,
0.12893787026405334,
0.11523330211639404,
0.10665617883205414,
0.10366716235876083,
0.10828493535518646,
0.11819782853126526,
0.10312190651893616,
0.10534654557704926
],
"sources": 9
},
{
"entropy_mean": 2.1452555656433105,
"mean_weights": [
0.09202822297811508,
0.1701684445142746,
0.14628148078918457,
0.11057832837104797,
0.09616327285766602,
0.09830240160226822,
0.12624149024486542,
0.08024715632200241,
0.07998920977115631
],
"sources": 9
},
{
"entropy_mean": 2.1762189865112305,
"mean_weights": [
0.09908370673656464,
0.13803134858608246,
0.13144394755363464,
0.11672185361385345,
0.1074555367231369,
0.10124591737985611,
0.12084682285785675,
0.08855879306793213,
0.09661206603050232
],
"sources": 9
},
{
"entropy_mean": 2.1389729976654053,
"mean_weights": [
0.09089216589927673,
0.17082780599594116,
0.14892268180847168,
0.11308471858501434,
0.10078474879264832,
0.09951519966125488,
0.12574949860572815,
0.07681785523891449,
0.07340531051158905
],
"sources": 9
}
],
"layer_input_rms_by_sublayer": [
0.04405451565980911,
0.018189694732427597,
0.0240375604480505,
0.02713710628449917,
0.030349314212799072,
0.03182194381952286,
0.034919846802949905,
0.03608781099319458,
0.038470037281513214,
0.026185201480984688,
0.02659580484032631,
0.02762730047106743,
0.028048591688275337,
0.028690988197922707,
0.029997555539011955,
0.03034786321222782,
0.031361404806375504,
0.02331089787185192,
0.0246393084526062,
0.024178674444556236,
0.025546060875058174,
0.024928271770477295,
0.02669469080865383,
0.0256717037409544,
0.027990806847810745,
0.020905915647745132,
0.02195541374385357,
0.021578654646873474,
0.023044798523187637,
0.022400982677936554,
0.02394026517868042,
0.023724498227238655,
0.025360919535160065,
0.020535636693239212,
0.02148214913904667,
0.021988606080412865,
0.02322523109614849,
0.023164646700024605,
0.023834573104977608,
0.024238524958491325,
0.02502058818936348,
0.019457563757896423,
0.01980534754693508,
0.019637931138277054,
0.02057415433228016,
0.02042427472770214,
0.02201046608388424,
0.021937908604741096,
0.02379545010626316,
0.021518733352422714,
0.022529911249876022,
0.02267260104417801,
0.023956971243023872,
0.024102909490466118,
0.02642984129488468,
0.025153208523988724,
0.030454812571406364,
0.023361310362815857,
0.02776513062417507,
0.025513608008623123,
0.030121317133307457,
0.027568820863962173,
0.03041756898164749,
0.028944427147507668
],
"loss_nats": 2.120351791381836,
"loss_scale": 1.0,
"output_weights": {
"entropy_mean": 2.1635239124298096,
"mean_weights": [
0.1242082342505455,
0.0880277156829834,
0.0938715785741806,
0.09339353442192078,
0.09749531745910645,
0.10569863021373749,
0.13438579440116882,
0.14857196807861328,
0.11434720456600189
],
"sources": 9
},
"step": 500,
"stream_state_rms_by_sublayer": [
0.01139580737799406,
0.021054552868008614,
0.02950318343937397,
0.038058556616306305,
0.04104945436120033,
0.04883526638150215,
0.05214058980345726,
0.05791458487510681,
0.01439172774553299,
0.016990283504128456,
0.024067245423793793,
0.027475843206048012,
0.030465533956885338,
0.03445403650403023,
0.036618731915950775,
0.040619369596242905,
0.016362585127353668,
0.019055280834436417,
0.02618422359228134,
0.029903952032327652,
0.033149950206279755,
0.037492990493774414,
0.04017872363328934,
0.04475866258144379,
0.025544000789523125,
0.0285621490329504,
0.030907167121767998,
0.03507128730416298,
0.03762345016002655,
0.04228900745511055,
0.04704070836305618,
0.05219251662492752,
0.022795414552092552,
0.027010982856154442,
0.03890290856361389,
0.04389137029647827,
0.04891306161880493,
0.054182786494493484,
0.060658931732177734,
0.06662839651107788,
0.033973678946495056,
0.03837457299232483,
0.03364448994398117,
0.041143495589494705,
0.04609670862555504,
0.05716610327363014,
0.05855504423379898,
0.07329611480236053,
0.015395586378872395,
0.028997479006648064,
0.03421308845281601,
0.053451184183359146,
0.058928847312927246,
0.08026835322380066,
0.08527950197458267,
0.10854664444923401,
0.006911881268024445,
0.027061885222792625,
0.028406694531440735,
0.05247335880994797,
0.05527278408408165,
0.08323372155427933,
0.0854114517569542,
0.11460508406162262
]
},
{
"activation_grad_rms_by_block": [
5.134936509421095e-05,
5.603071258519776e-05,
4.972576425643638e-05,
4.7571254981448874e-05,
4.3214979086769745e-05,
3.9845475839683786e-05,
3.5890228900825605e-05,
3.406162795727141e-05,
3.5466175177134573e-05,
3.383069270057604e-05,
3.1626921554561704e-05,
3.0198520107660443e-05,
2.883204069803469e-05,
2.77392227872042e-05,
2.7800235329777934e-05,
3.112062404397875e-05,
6.127587403170764e-05,
6.331028998829424e-05,
5.8612553402781487e-05,
5.708334720111452e-05,
0.00015888184134382755,
0.00012872362276539207,
0.00010434311116114259,
8.657111175125465e-05,
5.805831096949987e-05,
5.078948743175715e-05,
4.287513002054766e-05,
3.959577952628024e-05,
2.2001073375577107e-05,
1.997212712012697e-05,
1.8872458895202726e-05,
1.86532452062238e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 4.4711176087730564e-05,
"first_to_last_ratio": 1.3207760209544162,
"imbalance_abs_log_ratio": 0.27821945851834845,
"last_quartile_mean": 3.385220156815194e-05,
"mean": 4.981010016535947e-05,
"normalized": [
1.0309026668033479,
1.1248865671658381,
0.9983068512481783,
0.9550523854303026,
0.8675947035501784,
0.799947715571839,
0.7205411910772573,
0.68382974224492,
0.7120277827066004,
0.6791934284063869,
0.6349499689734955,
0.6062730250974693,
0.5788392434931497,
0.5568995584252106,
0.5581244614543389,
0.6247854138149606,
1.2301897371875206,
1.271033179578376,
1.1767202476646232,
1.146019522378179,
3.189751492495938,
2.584287570955602,
2.0948183363362958,
1.7380232415485222,
1.165593138274326,
1.019662423146035,
0.8607718088943986,
0.7949347500773989,
0.4416990390008851,
0.4009654076948961,
0.3788881940118565,
0.3744872052916737
],
"population_cv": 0.6175097196118516
},
"activation_output_rms_by_block": [
0.0955304205417633,
0.19925470650196075,
0.2359117865562439,
0.25103330612182617,
0.09304249286651611,
0.11085781455039978,
0.12910984456539154,
0.14378803968429565,
0.04010729491710663,
0.06472750008106232,
0.09987020492553711,
0.14899125695228577,
0.05702618882060051,
0.10170809924602509,
0.15923865139484406,
0.38537684082984924,
0.05919196084141731,
0.08540526777505875,
0.1145314872264862,
0.15640178322792053,
0.06262174993753433,
0.08427268266677856,
0.11030438542366028,
0.13170360028743744,
0.05311359465122223,
0.08614693582057953,
0.12623678147792816,
0.17503507435321808,
0.07505694776773453,
0.13522638380527496,
0.2187158316373825,
0.3292153775691986
],
"activation_output_statistics": {
"first_quartile_mean": 0.15731605142354965,
"first_to_last_ratio": 1.0498699791884785,
"imbalance_abs_log_ratio": 0.04866632715763954,
"last_quartile_mean": 0.14984336588531733,
"mean": 0.13496107165701687,
"normalized": [
0.7078368552417796,
1.4763865169086419,
1.7479987648273716,
1.86004232953773,
0.6894024456398029,
0.8214058556983611,
0.9566450753555414,
1.06540380806779,
0.2971767667867461,
0.479601260469815,
0.7399926786247083,
1.1039572754054923,
0.4225380557552472,
0.7536106374770113,
1.1798857955094264,
2.8554666623367226,
0.4385854388578413,
0.632814090214868,
0.8486260950679959,
1.158865895977708,
0.4639986121085199,
0.624422151010662,
0.8173051982276947,
0.9758636225277033,
0.39354751706627217,
0.6383095122385285,
0.935356987967166,
1.2969300866107762,
0.5561377576971261,
1.0019658420387498,
1.6205845800722127,
2.439335828673987
],
"population_cv": 0.5772709429538598
},
"bits_per_byte": 2.064953725142933,
"branch_output_rms_by_sublayer": [
0.051234737038612366,
0.07594770938158035,
0.11323792487382889,
0.04037529230117798,
0.05108218640089035,
0.04642988368868828,
0.03771067038178444,
0.02321692928671837,
0.08815877884626389,
0.015561277978122234,
0.03706873208284378,
0.01583549566566944,
0.0438041090965271,
0.020819898694753647,
0.030363978818058968,
0.027020027860999107,
0.03348636254668236,
0.0216005090624094,
0.027010800316929817,
0.024715786799788475,
0.032826121896505356,
0.03509491682052612,
0.031702958047389984,
0.05490833893418312,
0.038218315690755844,
0.037195492535829544,
0.03341353312134743,
0.042240556329488754,
0.04180334508419037,
0.04918703809380531,
0.2476748526096344,
0.04376247525215149,
0.05078785493969917,
0.02373618632555008,
0.0355411171913147,
0.01774558797478676,
0.052597854286432266,
0.022557877004146576,
0.06600882858037949,
0.03756217285990715,
0.050489071756601334,
0.03271714597940445,
0.022040290758013725,
0.03932206705212593,
0.028325051069259644,
0.04476166144013405,
0.02171224355697632,
0.049177221953868866,
0.024430278688669205,
0.0453680194914341,
0.015834001824259758,
0.0437605082988739,
0.018936052918434143,
0.052028607577085495,
0.02182060107588768,
0.060354240238666534,
0.030554499477148056,
0.06733348965644836,
0.016542606055736542,
0.07210637629032135,
0.015565099194645882,
0.09804103523492813,
0.013472299091517925,
0.1243291050195694
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.00020065673277085515,
0.0001817740596446822,
0.0001465045892684325,
9.755666249346124e-05,
0.00012624719322665892,
7.565538032235173e-05,
8.200655559893429e-05,
7.97391790009874e-05,
8.09162329894288e-05,
7.508564486922528e-05,
9.13364471873345e-05,
0.00011039800739152168,
8.766456722124108e-05,
8.422239419368561e-05,
9.398125423258295e-05,
0.00017161898601766447,
0.0001497631172902175,
0.00014237628599353527,
0.00022365597941099996,
0.00025566927173363617,
0.0008281082302532628,
0.0006115617552883831,
0.0005788343529396405,
0.00045039612289279527,
0.0002758540055807746,
0.00022394859978287228,
0.00021964130134669458,
0.00019854079025805258,
0.00011545923673689007,
0.00010570174900205617,
9.906174415914337e-05,
0.0001053958635417218
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.00012376754404079543,
"first_to_last_ratio": 0.7369290916409895,
"imbalance_abs_log_ratio": 0.30526360357753907,
"last_quartile_mean": 0.00016795041130102567,
"mean": 0.00019904163414499135,
"normalized": [
1.0081143758329842,
0.9132464191500225,
0.7360499721465892,
0.49013194105106844,
0.6342753051087516,
0.3800982676179234,
0.4120070454164219,
0.40061557645221907,
0.4065291833892681,
0.3772358742519635,
0.4588811159016144,
0.5546478145929135,
0.4404333173700838,
0.42313958361261256,
0.4721688235543854,
0.8622265726207266,
0.7524210597121753,
0.7153090689047581,
1.123664304564935,
1.2845014704179658,
4.16047744890413,
3.072531824386513,
2.908106916555911,
2.2628236792142964,
1.385911077176092,
1.1251344511155768,
1.1034942628470255,
0.9974837229954918,
0.5800758079225982,
0.5310534625386868,
0.49769358347903286,
0.5295166711952659
],
"population_cv": 0.8756839263617585
},
"depth_weights": [
{
"entropy_mean": 0.0,
"mean_weights": [
1.0
],
"sources": 1
},
{
"entropy_mean": 0.4580664336681366,
"mean_weights": [
0.81428062915802,
0.18571940064430237
],
"sources": 2
},
{
"entropy_mean": 0.6350213885307312,
"mean_weights": [
0.5189062356948853,
0.48109376430511475
],
"sources": 2
},
{
"entropy_mean": 0.4277062714099884,
"mean_weights": [
0.8402144908905029,
0.15978553891181946
],
"sources": 2
},
{
"entropy_mean": 0.6868079900741577,
"mean_weights": [
0.493515282869339,
0.5064846873283386
],
"sources": 2
},
{
"entropy_mean": 0.38686758279800415,
"mean_weights": [
0.8638634085655212,
0.13613657653331757
],
"sources": 2
},
{
"entropy_mean": 0.6772255897521973,
"mean_weights": [
0.5679300427436829,
0.43206995725631714
],
"sources": 2
},
{
"entropy_mean": 0.5690979361534119,
"mean_weights": [
0.7181376218795776,
0.28186243772506714
],
"sources": 2
},
{
"entropy_mean": 0.6592018008232117,
"mean_weights": [
0.5172787308692932,
0.4827212691307068
],
"sources": 2
},
{
"entropy_mean": 1.038456916809082,
"mean_weights": [
0.4540913999080658,
0.23118522763252258,
0.3147233724594116
],
"sources": 3
},
{
"entropy_mean": 1.0903832912445068,
"mean_weights": [
0.3152569830417633,
0.30281925201416016,
0.38192376494407654
],
"sources": 3
},
{
"entropy_mean": 1.0411981344223022,
"mean_weights": [
0.4020236134529114,
0.2803056836128235,
0.31767070293426514
],
"sources": 3
},
{
"entropy_mean": 1.0905810594558716,
"mean_weights": [
0.3278552293777466,
0.2872077226638794,
0.384937047958374
],
"sources": 3
},
{
"entropy_mean": 1.0150185823440552,
"mean_weights": [
0.48287349939346313,
0.22136211395263672,
0.29576438665390015
],
"sources": 3
},
{
"entropy_mean": 1.0816482305526733,
"mean_weights": [
0.3306047320365906,
0.2639211118221283,
0.40547415614128113
],
"sources": 3
},
{
"entropy_mean": 1.0114538669586182,
"mean_weights": [
0.4296638071537018,
0.22373110055923462,
0.346605122089386
],
"sources": 3
},
{
"entropy_mean": 1.0918333530426025,
"mean_weights": [
0.36806726455688477,
0.3041593134403229,
0.32777342200279236
],
"sources": 3
},
{
"entropy_mean": 1.3290088176727295,
"mean_weights": [
0.35130923986434937,
0.17179939150810242,
0.21658103168010712,
0.2603103518486023
],
"sources": 4
},
{
"entropy_mean": 1.379033088684082,
"mean_weights": [
0.29040855169296265,
0.23445811867713928,
0.2306380271911621,
0.24449528753757477
],
"sources": 4
},
{
"entropy_mean": 1.3394957780838013,
"mean_weights": [
0.3262045383453369,
0.18669262528419495,
0.22525924444198608,
0.26184359192848206
],
"sources": 4
},
{
"entropy_mean": 1.3677098751068115,
"mean_weights": [
0.29889944195747375,
0.22409284114837646,
0.1924189329147339,
0.2845887541770935
],
"sources": 4
},
{
"entropy_mean": 1.2910308837890625,
"mean_weights": [
0.3990757167339325,
0.18164023756980896,
0.16051796078681946,
0.2587660551071167
],
"sources": 4
},
{
"entropy_mean": 1.3623619079589844,
"mean_weights": [
0.3024129569530487,
0.21113446354866028,
0.18768057227134705,
0.29877203702926636
],
"sources": 4
},
{
"entropy_mean": 1.2281851768493652,
"mean_weights": [
0.46696996688842773,
0.16045212745666504,
0.13728894293308258,
0.23528897762298584
],
"sources": 4
},
{
"entropy_mean": 1.3772211074829102,
"mean_weights": [
0.28748467564582825,
0.24638661742210388,
0.22340166568756104,
0.24272704124450684
],
"sources": 4
},
{
"entropy_mean": 1.5519046783447266,
"mean_weights": [
0.30400484800338745,
0.1470816731452942,
0.13316048681735992,
0.1807912439107895,
0.23496176302433014
],
"sources": 5
},
{
"entropy_mean": 1.572178602218628,
"mean_weights": [
0.21885336935520172,
0.2029750645160675,
0.14561443030834198,
0.14960341155529022,
0.28295373916625977
],
"sources": 5
},
{
"entropy_mean": 1.5391805171966553,
"mean_weights": [
0.2898171544075012,
0.15984714031219482,
0.1228363886475563,
0.16408921778202057,
0.26341012120246887
],
"sources": 5
},
{
"entropy_mean": 1.5521924495697021,
"mean_weights": [
0.210292249917984,
0.185291588306427,
0.14173434674739838,
0.14211969077587128,
0.32056212425231934
],
"sources": 5
},
{
"entropy_mean": 1.5468778610229492,
"mean_weights": [
0.2756267488002777,
0.16966477036476135,
0.1333748996257782,
0.1590600460767746,
0.26227355003356934
],
"sources": 5
},
{
"entropy_mean": 1.5309922695159912,
"mean_weights": [
0.21606209874153137,
0.27386677265167236,
0.12764990329742432,
0.10704417526721954,
0.2753770351409912
],
"sources": 5
},
{
"entropy_mean": 1.5832164287567139,
"mean_weights": [
0.22763225436210632,
0.24636119604110718,
0.17577087879180908,
0.1817018985748291,
0.1685338020324707
],
"sources": 5
},
{
"entropy_mean": 1.5793015956878662,
"mean_weights": [
0.21581211686134338,
0.21387425065040588,
0.1464191973209381,
0.17280855774879456,
0.25108590722084045
],
"sources": 5
},
{
"entropy_mean": 1.7719972133636475,
"mean_weights": [
0.16962555050849915,
0.20328447222709656,
0.1659279465675354,
0.14008209109306335,
0.12920910120010376,
0.1918708235025406
],
"sources": 6
},
{
"entropy_mean": 1.689546823501587,
"mean_weights": [
0.16601097583770752,
0.1523149460554123,
0.0875597670674324,
0.101267509162426,
0.1655988097190857,
0.3272479772567749
],
"sources": 6
},
{
"entropy_mean": 1.5251271724700928,
"mean_weights": [
0.13745036721229553,
0.11702707409858704,
0.07592719793319702,
0.08887732028961182,
0.11522199213504791,
0.4654960632324219
],
"sources": 6
},
{
"entropy_mean": 1.7189464569091797,
"mean_weights": [
0.1744517982006073,
0.15982553362846375,
0.09849169105291367,
0.10780535638332367,
0.16236206889152527,
0.29706352949142456
],
"sources": 6
},
{
"entropy_mean": 1.5877950191497803,
"mean_weights": [
0.14238224923610687,
0.1503014713525772,
0.10275131464004517,
0.10159259289503098,
0.08505517989397049,
0.4179171919822693
],
"sources": 6
},
{
"entropy_mean": 1.7369492053985596,
"mean_weights": [
0.17980654537677765,
0.1706826537847519,
0.10907262563705444,
0.1106589287519455,
0.15735921263694763,
0.2724200189113617
],
"sources": 6
},
{
"entropy_mean": 1.5987625122070312,
"mean_weights": [
0.15493065118789673,
0.15333226323127747,
0.10736657679080963,
0.1186048835515976,
0.07128532975912094,
0.39448028802871704
],
"sources": 6
},
{
"entropy_mean": 1.75795578956604,
"mean_weights": [
0.1946164071559906,
0.17554126679897308,
0.11572675406932831,
0.1155121773481369,
0.1708213835954666,
0.2277819961309433
],
"sources": 6
},
{
"entropy_mean": 1.5344648361206055,
"mean_weights": [
0.0905577763915062,
0.0786280632019043,
0.06272733211517334,
0.07359343767166138,
0.050107162445783615,
0.13821640610694885,
0.5061697959899902
],
"sources": 7
},
{
"entropy_mean": 1.7138500213623047,
"mean_weights": [
0.13302072882652283,
0.10984362661838531,
0.07243296504020691,
0.07083481550216675,
0.08885611593723297,
0.11790697276592255,
0.4071047604084015
],
"sources": 7
},
{
"entropy_mean": 1.4565383195877075,
"mean_weights": [
0.09418222308158875,
0.07457256317138672,
0.057202234864234924,
0.06210128590464592,
0.04410415142774582,
0.11846546828746796,
0.5493720769882202
],
"sources": 7
},
{
"entropy_mean": 1.7785513401031494,
"mean_weights": [
0.1471671462059021,
0.12308044731616974,
0.07605892419815063,
0.07563325762748718,
0.10904586315155029,
0.11226382851600647,
0.3567505478858948
],
"sources": 7
},
{
"entropy_mean": 1.5135828256607056,
"mean_weights": [
0.10121430456638336,
0.07933001220226288,
0.061728835105895996,
0.06817515194416046,
0.048567138612270355,
0.12093988060951233,
0.5200446844100952
],
"sources": 7
},
{
"entropy_mean": 1.7546210289001465,
"mean_weights": [
0.14427465200424194,
0.12224892526865005,
0.08193168044090271,
0.07876674830913544,
0.09007281810045242,
0.10526382178068161,
0.3774413466453552
],
"sources": 7
},
{
"entropy_mean": 1.5284571647644043,
"mean_weights": [
0.10831618309020996,
0.08754566311836243,
0.06345342099666595,
0.06688661873340607,
0.05242554843425751,
0.11085730791091919,
0.5105152726173401
],
"sources": 7
},
{
"entropy_mean": 1.8309729099273682,
"mean_weights": [
0.1468513160943985,
0.13658970594406128,
0.09501184523105621,
0.0928751677274704,
0.10057065635919571,
0.11721575260162354,
0.31088554859161377
],
"sources": 7
},
{
"entropy_mean": 1.6937754154205322,
"mean_weights": [
0.09603364765644073,
0.06767555326223373,
0.045112669467926025,
0.047895483672618866,
0.03816477954387665,
0.07518067210912704,
0.3669250011444092,
0.26301223039627075
],
"sources": 8
},
{
"entropy_mean": 1.9070472717285156,
"mean_weights": [
0.10919073224067688,
0.11743452399969101,
0.07601121068000793,
0.0681401938199997,
0.0658918172121048,
0.07979628443717957,
0.24526360630989075,
0.2382715940475464
],
"sources": 8
},
{
"entropy_mean": 1.776512861251831,
"mean_weights": [
0.08731270581483841,
0.08430957794189453,
0.056895699352025986,
0.06295455247163773,
0.04594648256897926,
0.09297705441713333,
0.33455103635787964,
0.23505286872386932
],
"sources": 8
},
{
"entropy_mean": 1.9573358297348022,
"mean_weights": [
0.11900999397039413,
0.11752337962388992,
0.07979564368724823,
0.07266241312026978,
0.07412992417812347,
0.08811671286821365,
0.22605092823505402,
0.2227109968662262
],
"sources": 8
},
{
"entropy_mean": 1.7231189012527466,
"mean_weights": [
0.09091638028621674,
0.07795695960521698,
0.05235600098967552,
0.053361356258392334,
0.04162060469388962,
0.07854442298412323,
0.3417898118495941,
0.2634544372558594
],
"sources": 8
},
{
"entropy_mean": 1.8967883586883545,
"mean_weights": [
0.1089041531085968,
0.1000017300248146,
0.07706406712532043,
0.07702124863862991,
0.06017591059207916,
0.07969298958778381,
0.261149525642395,
0.23599036037921906
],
"sources": 8
},
{
"entropy_mean": 1.801592230796814,
"mean_weights": [
0.09432460367679596,
0.09541776776313782,
0.06706254929304123,
0.07247316837310791,
0.048089414834976196,
0.08817503601312637,
0.3473600149154663,
0.1870974600315094
],
"sources": 8
},
{
"entropy_mean": 1.919801950454712,
"mean_weights": [
0.10887958109378815,
0.10445678234100342,
0.0838727131485939,
0.0852852314710617,
0.05865198373794556,
0.08557502925395966,
0.27338963747024536,
0.19988907873630524
],
"sources": 8
},
{
"entropy_mean": 1.9332969188690186,
"mean_weights": [
0.07704077661037445,
0.08837630599737167,
0.05746292322874069,
0.06001049652695656,
0.04015304520726204,
0.06584024429321289,
0.2511133551597595,
0.10944834351539612,
0.25055450201034546
],
"sources": 9
},
{
"entropy_mean": 2.0115504264831543,
"mean_weights": [
0.08485299348831177,
0.08635596185922623,
0.06204982101917267,
0.06870189309120178,
0.0476498119533062,
0.08640170097351074,
0.2653427720069885,
0.13704031705856323,
0.16160473227500916
],
"sources": 9
},
{
"entropy_mean": 1.9781036376953125,
"mean_weights": [
0.08389372378587723,
0.08841055631637573,
0.060679104179143906,
0.0658775195479393,
0.045975662767887115,
0.08423998951911926,
0.29481440782546997,
0.12572166323661804,
0.15038736164569855
],
"sources": 9
},
{
"entropy_mean": 2.0586161613464355,
"mean_weights": [
0.1038442850112915,
0.09918913245201111,
0.08148118108510971,
0.0698266476392746,
0.059573929756879807,
0.08160553872585297,
0.22791330516338348,
0.13585451245307922,
0.1407114714384079
],
"sources": 9
},
{
"entropy_mean": 2.022735118865967,
"mean_weights": [
0.09707076847553253,
0.09551473706960678,
0.06701318919658661,
0.0673459842801094,
0.04480960965156555,
0.08329527080059052,
0.24251294136047363,
0.15929152071475983,
0.14314597845077515
],
"sources": 9
},
{
"entropy_mean": 1.9317231178283691,
"mean_weights": [
0.07746735960245132,
0.08971680700778961,
0.06909166276454926,
0.08098623156547546,
0.04421798139810562,
0.0916394367814064,
0.3483121693134308,
0.10497690737247467,
0.09359145164489746
],
"sources": 9
},
{
"entropy_mean": 2.053220272064209,
"mean_weights": [
0.09178322553634644,
0.10647953301668167,
0.07523074746131897,
0.08394691348075867,
0.04770101606845856,
0.09626083076000214,
0.26201331615448,
0.13223087787628174,
0.10435353964567184
],
"sources": 9
}
],
"layer_input_rms_by_sublayer": [
0.0558863990008831,
0.04748110845685005,
0.061753273010253906,
0.06085919216275215,
0.11003358662128448,
0.06251973658800125,
0.1138310506939888,
0.08963073790073395,
0.13082429766654968,
0.08407715708017349,
0.10403942316770554,
0.1004372090101242,
0.1062953919172287,
0.09022688120603561,
0.10766621679067612,
0.09703134745359421,
0.11238158494234085,
0.0712415874004364,
0.08696430176496506,
0.07720176130533218,
0.08355452120304108,
0.07461326569318771,
0.08609789609909058,
0.0729745551943779,
0.100697360932827,
0.06531906127929688,
0.07805952429771423,
0.06964895129203796,
0.08197034895420074,
0.08001933246850967,
0.10499614477157593,
0.11749708652496338,
0.13756641745567322,
0.09965827316045761,
0.09882861375808716,
0.0854005292057991,
0.10110001266002655,
0.08426972478628159,
0.10269922018051147,
0.0886540412902832,
0.10988812148571014,
0.04304825887084007,
0.0575832724571228,
0.0459144227206707,
0.06622003018856049,
0.05584317073225975,
0.06460635364055634,
0.06311307847499847,
0.07165247201919556,
0.05157631263136864,
0.055589381605386734,
0.05562245845794678,
0.05917651578783989,
0.059894196689128876,
0.06219335272908211,
0.06387592852115631,
0.06750614196062088,
0.050557736307382584,
0.06024058535695076,
0.06106090545654297,
0.06803743541240692,
0.06874485313892365,
0.06952574104070663,
0.07015988230705261
],
"loss_nats": 1.43131685256958,
"loss_scale": 1.0,
"output_weights": {
"entropy_mean": 1.843801498413086,
"mean_weights": [
0.11151862144470215,
0.061874739825725555,
0.05308101326227188,
0.040848225355148315,
0.042743273079395294,
0.053944505751132965,
0.1199670359492302,
0.3250349462032318,
0.19098761677742004
],
"sources": 9
},
"step": 2000,
"stream_state_rms_by_sublayer": [
0.051234737038612366,
0.0955304205417633,
0.1796528398990631,
0.19925470650196075,
0.21105031669139862,
0.2359117865562439,
0.23972436785697937,
0.25103330612182617,
0.08815877884626389,
0.09304249286651611,
0.1068187803030014,
0.11085781455039978,
0.12298524379730225,
0.12910984456539154,
0.13432784378528595,
0.14378803968429565,
0.03348636254668236,
0.04010729491710663,
0.051035381853580475,
0.06472750008106232,
0.07822930067777634,
0.09987020492553711,
0.11166217923164368,
0.14899125695228577,
0.038218315690755844,
0.05702618882060051,
0.07459890842437744,
0.10170809924602509,
0.12532564997673035,
0.15923865139484406,
0.36271119117736816,
0.38537684082984924,
0.05078785493969917,
0.05919196084141731,
0.0794382095336914,
0.08540526777505875,
0.10824274271726608,
0.1145314872264862,
0.14269457757472992,
0.15640178322792053,
0.050489071756601334,
0.06262174993753433,
0.06721649318933487,
0.08427268266677856,
0.09199884533882141,
0.11030438542366028,
0.11444488167762756,
0.13170360028743744,
0.024430278688669205,
0.05311359465122223,
0.05695623904466629,
0.08614693582057953,
0.09002252668142319,
0.12623678147792816,
0.130380317568779,
0.17503507435321808,
0.030554499477148056,
0.07505694776773453,
0.07779927551746368,
0.13522638380527496,
0.13734298944473267,
0.2187158316373825,
0.22044414281845093,
0.3292153775691986
]
},
{
"activation_grad_rms_by_block": [
3.57800890924409e-05,
3.942943658330478e-05,
3.388081677258015e-05,
3.245583502575755e-05,
3.3783475373638794e-05,
2.915231198130641e-05,
2.5950168492272496e-05,
2.450849387969356e-05,
2.7269536076346412e-05,
2.568827403592877e-05,
2.400098310317844e-05,
2.2796592020313255e-05,
2.010442767641507e-05,
1.8936527339974418e-05,
1.9039980543311685e-05,
2.3072669137036428e-05,
4.836148946196772e-05,
4.9383146688342094e-05,
4.5993543608346954e-05,
4.280430221115239e-05,
0.00014426946290768683,
0.00011744296352844685,
9.347966988570988e-05,
7.698025729041547e-05,
6.582303467439488e-05,
5.6237244280055165e-05,
4.4205091398907825e-05,
3.852568261208944e-05,
2.7434407456894405e-05,
2.285257687617559e-05,
2.028334711212665e-05,
1.9826158677460626e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 3.186757840012433e-05,
"first_to_last_ratio": 0.8636564556008468,
"imbalance_abs_log_ratio": 0.146580210082559,
"last_quartile_mean": 3.689844288601307e-05,
"mean": 4.2179749868864747e-05,
"normalized": [
0.8482764645044092,
0.9347954102594115,
0.8032484042203742,
0.7694648528419803,
0.8009406285876599,
0.6911447334785021,
0.6152281265998635,
0.5810488197746441,
0.646507771172811,
0.6090191173677569,
0.569016724323795,
0.5404629494292167,
0.4766369582304063,
0.4489483081063157,
0.4514009827584627,
0.5470082020108817,
1.146557047216301,
1.170778557053383,
1.090417647125437,
1.014806923801809,
3.420348943656972,
2.7843447126541196,
2.216221532283493,
1.825052484420704,
1.5605363919661974,
1.3332759073938238,
1.048016916561615,
0.913369157756135,
0.6504165515961319,
0.5417902416971032,
0.48087879077488155,
0.470039740375403
],
"population_cv": 0.6827037752697015
},
"activation_output_rms_by_block": [
0.15143299102783203,
0.30262914299964905,
0.3626292943954468,
0.38562148809432983,
0.13662196695804596,
0.15442071855068207,
0.18618692457675934,
0.2074802666902542,
0.04909220710396767,
0.0881529152393341,
0.14954733848571777,
0.23951733112335205,
0.08217982947826385,
0.14409078657627106,
0.22605396807193756,
0.6592875719070435,
0.06389845162630081,
0.09458807110786438,
0.14630787074565887,
0.21908092498779297,
0.07773751765489578,
0.10736925154924393,
0.1424422264099121,
0.17246152460575104,
0.06572399288415909,
0.10123347491025925,
0.14551502466201782,
0.19983991980552673,
0.09023923426866531,
0.15600618720054626,
0.24885882437229156,
0.3860291540622711
],
"activation_output_statistics": {
"first_quartile_mean": 0.2358778491616249,
"first_to_last_ratio": 1.3542132581102162,
"imbalance_abs_log_ratio": 0.30322066438495643,
"last_quartile_mean": 0.17418072652071714,
"mean": 0.18569613725412637,
"normalized": [
0.8154881047449846,
1.6297007976288655,
1.9528101109566227,
2.076626330501449,
0.7357286423846173,
0.8315774406193291,
1.0026429592445498,
1.1173106223869165,
0.26436848838048077,
0.4747159340137284,
0.8053335987331889,
1.2898347518967022,
0.442550021198342,
0.7759492938675503,
1.2173326349948854,
3.5503569522547695,
0.3441022122008675,
0.5093701598026241,
0.7878886061342009,
1.1797818103667892,
0.41862754284711634,
0.5781986267291517,
0.7670715638795383,
0.9287297364173834,
0.3539330104331431,
0.5451566004936365,
0.7836190176798323,
1.0761662722797756,
0.4859510574803844,
0.8401154138551131,
1.3401400161152877,
2.078821669478174
],
"population_cv": 0.6621635595354842
},
"bits_per_byte": 1.9288425276664853,
"branch_output_rms_by_sublayer": [
0.08832799643278122,
0.11756844818592072,
0.17041416466236115,
0.05185626819729805,
0.08745641261339188,
0.0881219431757927,
0.05030680447816849,
0.03964918479323387,
0.12814420461654663,
0.022397184744477272,
0.05527547001838684,
0.022468235343694687,
0.06746833771467209,
0.031668391078710556,
0.04974694177508354,
0.045926205813884735,
0.03680809214711189,
0.03180550038814545,
0.03558321297168732,
0.03912416473031044,
0.051462169736623764,
0.06056958809494972,
0.0526844747364521,
0.10351218283176422,
0.05210389941930771,
0.05620818957686424,
0.03620988130569458,
0.0675361230969429,
0.04957791045308113,
0.0823286920785904,
0.4858623445034027,
0.07062793523073196,
0.04707946628332138,
0.0371064618229866,
0.04491531476378441,
0.023919809609651566,
0.07858023792505264,
0.035764746367931366,
0.09729185700416565,
0.07103991508483887,
0.05887215584516525,
0.045838613063097,
0.026784580200910568,
0.05455324053764343,
0.041033755987882614,
0.06383209675550461,
0.032232560217380524,
0.07182251662015915,
0.02874751202762127,
0.05687069892883301,
0.022572210058569908,
0.052836012095212936,
0.02659173309803009,
0.06559387594461441,
0.025225715711712837,
0.07689779996871948,
0.041177187114953995,
0.07740812748670578,
0.027006076648831367,
0.08842137455940247,
0.023083338513970375,
0.11899886280298233,
0.016772178933024406,
0.16321057081222534
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.00017320581296885245,
0.0001375362640366496,
0.00014375813569971144,
8.981648567965897e-05,
0.00010459593200616139,
6.156395218407552e-05,
7.281931709149581e-05,
7.667720183186952e-05,
7.236475672264944e-05,
7.890209128468536e-05,
9.134838485444255e-05,
0.00011250740948005864,
7.70610699389613e-05,
7.611216582142322e-05,
8.146874247135892e-05,
0.00017928549432252573,
0.00013712162717798398,
0.00013184310037104538,
0.00022367504753246386,
0.0002691717754589954,
0.0008753648193242955,
0.0006614972326966637,
0.0006052862860379005,
0.0005306195765578692,
0.00037252876038501256,
0.00029795485316373564,
0.0002868510641817651,
0.00023542884481748448,
0.00018331496216173493,
0.00013991664250432274,
0.00012258431160994327,
0.00012212563876750547
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.00010749663768730933,
"first_to_last_ratio": 0.4884254111851857,
"imbalance_abs_log_ratio": 0.7165685086453754,
"last_quartile_mean": 0.00022008813469893802,
"mean": 0.0002132596174732282,
"normalized": [
0.812182892481255,
0.6449240867362775,
0.6740991914128248,
0.42116030565859097,
0.4904629073494984,
0.2886807775119639,
0.3414585375059921,
0.3595486231306557,
0.33932704925597945,
0.3699813974138373,
0.428343565166106,
0.5275607769210333,
0.36134862686150726,
0.35689910130771973,
0.3820167335786683,
0.840691249692561,
0.6429798046280267,
0.6182281574597519,
1.0488392044524941,
1.262178834644052,
4.104690938190328,
3.101840097690814,
2.838260207016858,
2.4881390243724,
1.746832170097925,
1.3971461484082413,
1.3450791461621905,
1.1039541738230836,
0.8595859091079332,
0.6560859677143862,
0.5748125831902143,
0.5726618110568292
],
"population_cv": 0.9042592730039112
},
"depth_weights": [
{
"entropy_mean": 0.0,
"mean_weights": [
1.0
],
"sources": 1
},
{
"entropy_mean": 0.2508508265018463,
"mean_weights": [
0.9251874089241028,
0.07481260597705841
],
"sources": 2
},
{
"entropy_mean": 0.5796181559562683,
"mean_weights": [
0.6163991689682007,
0.38360080122947693
],
"sources": 2
},
{
"entropy_mean": 0.3817020058631897,
"mean_weights": [
0.8651303648948669,
0.13486966490745544
],
"sources": 2
},
{
"entropy_mean": 0.6884468793869019,
"mean_weights": [
0.4932539463043213,
0.5067461133003235
],
"sources": 2
},
{
"entropy_mean": 0.2797870337963104,
"mean_weights": [
0.9142746329307556,
0.08572537451982498
],
"sources": 2
},
{
"entropy_mean": 0.6642253994941711,
"mean_weights": [
0.605044960975647,
0.39495497941970825
],
"sources": 2
},
{
"entropy_mean": 0.5038842558860779,
"mean_weights": [
0.7756115198135376,
0.2243884801864624
],
"sources": 2
},
{
"entropy_mean": 0.6434851884841919,
"mean_weights": [
0.5374104976654053,
0.4625895023345947
],
"sources": 2
},
{
"entropy_mean": 1.0016276836395264,
"mean_weights": [
0.4353652000427246,
0.18586981296539307,
0.3787649869918823
],
"sources": 3
},
{
"entropy_mean": 1.0655895471572876,
"mean_weights": [
0.2726097106933594,
0.2790643572807312,
0.4483259320259094
],
"sources": 3
},
{
"entropy_mean": 1.0300546884536743,
"mean_weights": [
0.40803641080856323,
0.2817816734313965,
0.3101818859577179
],
"sources": 3
},
{
"entropy_mean": 1.0874333381652832,
"mean_weights": [
0.36608952283859253,
0.27405625581741333,
0.35985422134399414
],
"sources": 3
},
{
"entropy_mean": 0.9672750234603882,
"mean_weights": [
0.5343947410583496,
0.1846013069152832,
0.2810039520263672
],
"sources": 3
},
{
"entropy_mean": 1.0731852054595947,
"mean_weights": [
0.3801807165145874,
0.24373164772987366,
0.37608763575553894
],
"sources": 3
},
{
"entropy_mean": 0.9721636772155762,
"mean_weights": [
0.49137914180755615,
0.2109004259109497,
0.29772043228149414
],
"sources": 3
},
{
"entropy_mean": 1.0864832401275635,
"mean_weights": [
0.39626431465148926,
0.29088544845581055,
0.3128502070903778
],
"sources": 3
},
{
"entropy_mean": 1.3038380146026611,
"mean_weights": [
0.35696184635162354,
0.1472446322441101,
0.19980239868164062,
0.29599112272262573
],
"sources": 4
},
{
"entropy_mean": 1.3611116409301758,
"mean_weights": [
0.3264957070350647,
0.22676822543144226,
0.20138126611709595,
0.2453548014163971
],
"sources": 4
},
{
"entropy_mean": 1.3161137104034424,
"mean_weights": [
0.3259783387184143,
0.17862647771835327,
0.20532535016536713,
0.2900698482990265
],
"sources": 4
},
{
"entropy_mean": 1.3568220138549805,
"mean_weights": [
0.3223523497581482,
0.21851874887943268,
0.1804703176021576,
0.27865859866142273
],
"sources": 4
},
{
"entropy_mean": 1.2413711547851562,
"mean_weights": [
0.4272037148475647,
0.16667884588241577,
0.1365978717803955,
0.269519567489624
],
"sources": 4
},
{
"entropy_mean": 1.3540384769439697,
"mean_weights": [
0.32447606325149536,
0.2030642032623291,
0.18181107938289642,
0.2906486392021179
],
"sources": 4
},
{
"entropy_mean": 1.1544246673583984,
"mean_weights": [
0.5128031969070435,
0.13654224574565887,
0.12138728052377701,
0.22926729917526245
],
"sources": 4
},
{
"entropy_mean": 1.3712997436523438,
"mean_weights": [
0.3103735148906708,
0.24219870567321777,
0.2247106432914734,
0.22271710634231567
],
"sources": 4
},
{
"entropy_mean": 1.5205519199371338,
"mean_weights": [
0.3307538628578186,
0.1283297836780548,
0.12636876106262207,
0.17177441716194153,
0.2427731454372406
],
"sources": 5
},
{
"entropy_mean": 1.518624186515808,
"mean_weights": [
0.2150207757949829,
0.17106646299362183,
0.13557285070419312,
0.11898039281368256,
0.3593595027923584
],
"sources": 5
},
{
"entropy_mean": 1.4772696495056152,
"mean_weights": [
0.30324018001556396,
0.13238273561000824,
0.09796822816133499,
0.15785877406597137,
0.30855005979537964
],
"sources": 5
},
{
"entropy_mean": 1.4979676008224487,
"mean_weights": [
0.20932656526565552,
0.16542977094650269,
0.1272144764661789,
0.1160244345664978,
0.3820047676563263
],
"sources": 5
},
{
"entropy_mean": 1.49334716796875,
"mean_weights": [
0.28671571612358093,
0.13947691023349762,
0.11353637278079987,
0.14176231622695923,
0.31850868463516235
],
"sources": 5
},
{
"entropy_mean": 1.5120060443878174,
"mean_weights": [
0.21571135520935059,
0.28788846731185913,
0.131511390209198,
0.08343321830034256,
0.2814555764198303
],
"sources": 5
},
{
"entropy_mean": 1.5766459703445435,
"mean_weights": [
0.2565065026283264,
0.22934341430664062,
0.1593727320432663,
0.1524350941181183,
0.20234227180480957
],
"sources": 5
},
{
"entropy_mean": 1.5796699523925781,
"mean_weights": [
0.2356114238500595,
0.21223093569278717,
0.1439642310142517,
0.1658918261528015,
0.2423015832901001
],
"sources": 5
},
{
"entropy_mean": 1.7626371383666992,
"mean_weights": [
0.16389364004135132,
0.20886825025081635,
0.18027648329734802,
0.1462823748588562,
0.11240866780281067,
0.18827059864997864
],
"sources": 6
},
{
"entropy_mean": 1.6546597480773926,
"mean_weights": [
0.17577320337295532,
0.14207448065280914,
0.08111412078142166,
0.0931762307882309,
0.15011057257652283,
0.35775139927864075
],
"sources": 6
},
{
"entropy_mean": 1.430190086364746,
"mean_weights": [
0.1354856789112091,
0.09479984641075134,
0.07003438472747803,
0.0834486186504364,
0.09165525436401367,
0.5245761871337891
],
"sources": 6
},
{
"entropy_mean": 1.7008471488952637,
"mean_weights": [
0.18653985857963562,
0.149653822183609,
0.09563066065311432,
0.09960515052080154,
0.15559062361717224,
0.31297987699508667
],
"sources": 6
},
{
"entropy_mean": 1.514115333557129,
"mean_weights": [
0.14724519848823547,
0.13274209201335907,
0.096603162586689,
0.11347334086894989,
0.0527515783905983,
0.4571846127510071
],
"sources": 6
},
{
"entropy_mean": 1.7213332653045654,
"mean_weights": [
0.19527462124824524,
0.16114139556884766,
0.10406571626663208,
0.10423287749290466,
0.14977452158927917,
0.2855108380317688
],
"sources": 6
},
{
"entropy_mean": 1.503009557723999,
"mean_weights": [
0.1677132397890091,
0.12867671251296997,
0.09363879263401031,
0.12237933278083801,
0.04204780235886574,
0.44554415345191956
],
"sources": 6
},
{
"entropy_mean": 1.7531253099441528,
"mean_weights": [
0.21246549487113953,
0.17559614777565002,
0.11484666168689728,
0.11058571934700012,
0.1656094789505005,
0.22089649736881256
],
"sources": 6
},
{
"entropy_mean": 1.3981997966766357,
"mean_weights": [
0.08501995354890823,
0.06276540458202362,
0.0543542318046093,
0.06455785036087036,
0.03336101397871971,
0.12979380786418915,
0.5701477527618408
],
"sources": 7
},
{
"entropy_mean": 1.6064693927764893,
"mean_weights": [
0.12975823879241943,
0.09198254346847534,
0.06163205951452255,
0.06021754443645477,
0.078399658203125,
0.10503693670034409,
0.4729730188846588
],
"sources": 7
},
{
"entropy_mean": 1.3297276496887207,
"mean_weights": [
0.09028719365596771,
0.057085148990154266,
0.04790058359503746,
0.05685827136039734,
0.031231366097927094,
0.11244906485080719,
0.6041883826255798
],
"sources": 7
},
{
"entropy_mean": 1.6772459745407104,
"mean_weights": [
0.1455874890089035,
0.10527922213077545,
0.06669703871011734,
0.06069611385464668,
0.09951996803283691,
0.09685218334197998,
0.4253680109977722
],
"sources": 7
},
{
"entropy_mean": 1.3742375373840332,
"mean_weights": [
0.09774194657802582,
0.06367622315883636,
0.05300067365169525,
0.05981069803237915,
0.03241024166345596,
0.10956928879022598,
0.5837908983230591
],
"sources": 7
},
{
"entropy_mean": 1.672610878944397,
"mean_weights": [
0.14811497926712036,
0.10026681423187256,
0.0711982250213623,
0.06714795529842377,
0.08556141704320908,
0.09809904545545578,
0.42961156368255615
],
"sources": 7
},
{
"entropy_mean": 1.4325350522994995,
"mean_weights": [
0.11123399436473846,
0.07182648032903671,
0.05620439350605011,
0.06410455703735352,
0.03741488978266716,
0.10516737401485443,
0.5540482997894287
],
"sources": 7
},
{
"entropy_mean": 1.7746202945709229,
"mean_weights": [
0.15383490920066833,
0.11435382068157196,
0.09083531051874161,
0.08329388499259949,
0.08837151527404785,
0.11093884706497192,
0.3583717346191406
],
"sources": 7
},
{
"entropy_mean": 1.5557539463043213,
"mean_weights": [
0.09889799356460571,
0.05227670818567276,
0.03886467218399048,
0.039664674550294876,
0.02741250954568386,
0.06309205293655396,
0.40426212549209595,
0.27552926540374756
],
"sources": 8
},
{
"entropy_mean": 1.7942585945129395,
"mean_weights": [
0.10892648994922638,
0.08669497072696686,
0.060229551047086716,
0.05406671017408371,
0.05429235100746155,
0.07220236957073212,
0.24074554443359375,
0.322842001914978
],
"sources": 8
},
{
"entropy_mean": 1.6406831741333008,
"mean_weights": [
0.08106359094381332,
0.06122073531150818,
0.04382575303316116,
0.0495278537273407,
0.031025834381580353,
0.08371539413928986,
0.3212447166442871,
0.3283761143684387
],
"sources": 8
},
{
"entropy_mean": 1.8771746158599854,
"mean_weights": [
0.11823929846286774,
0.09187044948339462,
0.06966835260391235,
0.05918115749955177,
0.06588791310787201,
0.07802121341228485,
0.22871823608875275,
0.2884133756160736
],
"sources": 8
},
{
"entropy_mean": 1.612527847290039,
"mean_weights": [
0.09160974621772766,
0.05898914486169815,
0.039636120200157166,
0.04581688344478607,
0.032106831669807434,
0.06671778112649918,
0.3060194253921509,
0.35910406708717346
],
"sources": 8
},
{
"entropy_mean": 1.8253357410430908,
"mean_weights": [
0.10355446487665176,
0.08380813896656036,
0.06609268486499786,
0.07030215114355087,
0.04850628599524498,
0.0775601714849472,
0.25825831294059753,
0.2919177711009979
],
"sources": 8
},
{
"entropy_mean": 1.714170217514038,
"mean_weights": [
0.09696917235851288,
0.07828262448310852,
0.055068351328372955,
0.0616651251912117,
0.03598042577505112,
0.07802210748195648,
0.3351396918296814,
0.25887250900268555
],
"sources": 8
},
{
"entropy_mean": 1.875918984413147,
"mean_weights": [
0.11156761646270752,
0.0880264937877655,
0.07785218954086304,
0.07930342108011246,
0.04990004748106003,
0.08684437721967697,
0.2776731252670288,
0.22883272171020508
],
"sources": 8
},
{
"entropy_mean": 1.731968641281128,
"mean_weights": [
0.06617744266986847,
0.05556279793381691,
0.04319432005286217,
0.045391663908958435,
0.026374589651823044,
0.054588086903095245,
0.21148523688316345,
0.10446193069219589,
0.3927639424800873
],
"sources": 9
},
{
"entropy_mean": 1.9649744033813477,
"mean_weights": [
0.07813189923763275,
0.08751581609249115,
0.05837181955575943,
0.07608447968959808,
0.03110998496413231,
0.09630586206912994,
0.27242881059646606,
0.1154000535607338,
0.18465125560760498
],
"sources": 9
},
{
"entropy_mean": 1.8779311180114746,
"mean_weights": [
0.07922039180994034,
0.064256951212883,
0.04481552541255951,
0.05368471518158913,
0.03291679173707962,
0.07845190167427063,
0.26414477825164795,
0.13897250592708588,
0.24353644251823425
],
"sources": 9
},
{
"entropy_mean": 1.9959204196929932,
"mean_weights": [
0.10073552280664444,
0.0823616087436676,
0.0686727911233902,
0.0658436119556427,
0.048376426100730896,
0.07448098808526993,
0.23861894011497498,
0.14355987310409546,
0.177350252866745
],
"sources": 9
},
{
"entropy_mean": 1.9298079013824463,
"mean_weights": [
0.09570661187171936,
0.07176163792610168,
0.05499213933944702,
0.057467732578516006,
0.03367581591010094,
0.06890923529863358,
0.21427617967128754,
0.18804281949996948,
0.2151678204536438
],
"sources": 9
},
{
"entropy_mean": 1.8635971546173096,
"mean_weights": [
0.07685244828462601,
0.07133444398641586,
0.05962139368057251,
0.08040933310985565,
0.033340416848659515,
0.10067727416753769,
0.36328762769699097,
0.11175162345170975,
0.10272547602653503
],
"sources": 9
},
{
"entropy_mean": 2.0009379386901855,
"mean_weights": [
0.09139549732208252,
0.08807491511106491,
0.06457877159118652,
0.07007958739995956,
0.03462113067507744,
0.09155915677547455,
0.2692604660987854,
0.15421991050243378,
0.13621056079864502
],
"sources": 9
}
],
"layer_input_rms_by_sublayer": [
0.059407927095890045,
0.05594591051340103,
0.07886742800474167,
0.07073435187339783,
0.16177938878536224,
0.06750936061143875,
0.15622827410697937,
0.10916222631931305,
0.1895449161529541,
0.11270010471343994,
0.15217198431491852,
0.14727257192134857,
0.14916078746318817,
0.11801524460315704,
0.14915435016155243,
0.13372685015201569,
0.16036193072795868,
0.09399056434631348,
0.12247956544160843,
0.10827244818210602,
0.12010690569877625,
0.10214843600988388,
0.12369577586650848,
0.09725081920623779,
0.14881008863449097,
0.0902416780591011,
0.10321364551782608,
0.09460548311471939,
0.1168464794754982,
0.11164536327123642,
0.16267083585262299,
0.19675607979297638,
0.21974295377731323,
0.15372900664806366,
0.1438056081533432,
0.10524243861436844,
0.14797262847423553,
0.10666082799434662,
0.15148812532424927,
0.11902136355638504,
0.1692686378955841,
0.05537491664290428,
0.07994630187749863,
0.06347332894802094,
0.09600856900215149,
0.07961282879114151,
0.09550122171640396,
0.09120825678110123,
0.10489360988140106,
0.07538540661334991,
0.07298607379198074,
0.07336290925741196,
0.08083610236644745,
0.07823792845010757,
0.0843365490436554,
0.08682125806808472,
0.09205732494592667,
0.057621169835329056,
0.07988084107637405,
0.0771951824426651,
0.08835897594690323,
0.09140052646398544,
0.09466564655303955,
0.09293081611394882
],
"loss_nats": 1.3369717597961426,
"loss_scale": 1.0,
"output_weights": {
"entropy_mean": 1.7290174961090088,
"mean_weights": [
0.10267993807792664,
0.05008202791213989,
0.044380687177181244,
0.0326685830950737,
0.033161722123622894,
0.043591417372226715,
0.1060870811343193,
0.3520195186138153,
0.2353290319442749
],
"sources": 9
},
"step": 4000,
"stream_state_rms_by_sublayer": [
0.08832799643278122,
0.15143299102783203,
0.2790028154850006,
0.30262914299964905,
0.315830796957016,
0.3626292943954468,
0.3664044439792633,
0.38562148809432983,
0.12814420461654663,
0.13662196695804596,
0.14859555661678314,
0.15442071855068207,
0.17815031111240387,
0.18618692457675934,
0.19290222227573395,
0.2074802666902542,
0.03680809214711189,
0.04909220710396767,
0.06554166972637177,
0.0881529152393341,
0.11251214146614075,
0.14954733848571777,
0.16864995658397675,
0.23951733112335205,
0.05210389941930771,
0.08217982947826385,
0.09766767174005508,
0.14409078657627106,
0.16688434779644012,
0.22605396807193756,
0.6339711546897888,
0.6592875719070435,
0.04707946628332138,
0.06389845162630081,
0.084575355052948,
0.09458807110786438,
0.13482175767421722,
0.14630787074565887,
0.19051826000213623,
0.21908092498779297,
0.05887215584516525,
0.07773751765489578,
0.08281512558460236,
0.10736925154924393,
0.11754488199949265,
0.1424422264099121,
0.14818532764911652,
0.17246152460575104,
0.02874751202762127,
0.06572399288415909,
0.07124026119709015,
0.10123347491025925,
0.10666517913341522,
0.14551502466201782,
0.14956052601337433,
0.19983991980552673,
0.041177187114953995,
0.09023923426866531,
0.09537366032600403,
0.15600618720054626,
0.15850447118282318,
0.24885882437229156,
0.25068944692611694,
0.3860291540622711
]
},
{
"activation_grad_rms_by_block": [
4.789723243447952e-05,
5.257939483271912e-05,
4.447350511327386e-05,
4.274819002603181e-05,
4.96310785820242e-05,
4.17958217440173e-05,
3.729292075149715e-05,
3.545484287315048e-05,
4.479578274185769e-05,
4.205213917884976e-05,
3.91026733268518e-05,
3.680578447529115e-05,
3.1189843866741285e-05,
2.9126334993634373e-05,
2.9360509870457463e-05,
3.65210507879965e-05,
8.83217144291848e-05,
8.915644866647199e-05,
8.140744466800243e-05,
7.45641955290921e-05,
0.0002545347670093179,
0.0001978082291316241,
0.000154864537762478,
0.00012389873154461384,
0.00012786213483195752,
0.00010459556506248191,
7.644320430699736e-05,
6.138104072306305e-05,
6.131517875473946e-05,
4.588167939800769e-05,
3.516174911055714e-05,
3.2676212867954746e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 4.398412329464918e-05,
"first_to_last_ratio": 0.6452634668607965,
"imbalance_abs_log_ratio": 0.43809656983670076,
"last_quartile_mean": 6.816459563196986e-05,
"mean": 7.03343731061068e-05,
"normalized": [
0.6809932372926891,
0.7475632825133391,
0.6323153694166137,
0.6077851857944574,
0.7056447138179579,
0.5942446047107566,
0.530223262177042,
0.504089839823597,
0.6368974480554273,
0.5978888745536934,
0.5559539610577222,
0.5232972563751404,
0.4434509399790644,
0.41411238498840724,
0.4174418363858031,
0.5192489699581264,
1.2557404086895352,
1.2676084925356499,
1.1574347089891701,
1.060138766241709,
3.6189242296270336,
2.81239769967395,
2.2018329150221976,
1.7615672973682381,
1.8179181698124194,
1.4871187506667403,
1.086854135909831,
0.872703317202576,
0.8717669049561169,
0.6523365087620864,
0.4999226915339381,
0.46458383610897114
],
"population_cv": 0.7299683279122143
},
"activation_output_rms_by_block": [
0.15345558524131775,
0.3249167501926422,
0.3818626403808594,
0.40600064396858215,
0.1351035088300705,
0.15324510633945465,
0.18001793324947357,
0.20391276478767395,
0.04374876990914345,
0.08211024105548859,
0.1380448192358017,
0.2295975536108017,
0.08199761807918549,
0.13892096281051636,
0.21687114238739014,
0.7360114455223083,
0.057135771960020065,
0.08251863718032837,
0.12661415338516235,
0.19619159400463104,
0.06753352284431458,
0.095451220870018,
0.1274663656949997,
0.15628865361213684,
0.058436937630176544,
0.08794140070676804,
0.12247690558433533,
0.1640285700559616,
0.07537907361984253,
0.12290703505277634,
0.19157102704048157,
0.29249706864356995
],
"activation_output_statistics": {
"first_quartile_mean": 0.24231436662375927,
"first_to_last_ratio": 1.7382073612286646,
"imbalance_abs_log_ratio": 0.5528543299844475,
"last_quartile_mean": 0.139404752291739,
"mean": 0.17594548198394477,
"normalized": [
0.872176901111452,
1.8466899321818973,
2.1703463827261262,
2.3075366266331874,
0.767871429869389,
0.870980627700565,
1.0231460974138593,
1.1589542538312023,
0.24864957835709345,
0.4666800200245053,
0.7845885991457198,
1.3049357734104974,
0.46603991847126747,
0.789568230136157,
1.2326042132027004,
4.183179000808161,
0.32473565854469494,
0.4690011715552792,
0.7196215097851506,
1.1150703717560293,
0.38383209436703286,
0.5425045292082467,
0.7244651255474189,
0.8882788682598759,
0.3321309360788757,
0.49982187502145003,
0.6961071361611404,
0.9322692927740504,
0.42842289992260757,
0.6985518108614562,
1.0888090156129309,
1.6624301195199809
],
"population_cv": 0.7619463065887785
},
"bits_per_byte": 1.877417995231997,
"branch_output_rms_by_sublayer": [
0.09253507107496262,
0.116500623524189,
0.19514523446559906,
0.0468219518661499,
0.07917427271604538,
0.09375449270009995,
0.04215213656425476,
0.041188232600688934,
0.12552575767040253,
0.024237047880887985,
0.05027730390429497,
0.022429833188652992,
0.054851483553647995,
0.03027583472430706,
0.036243390291929245,
0.04546776041388512,
0.031880609691143036,
0.030174262821674347,
0.035054463893175125,
0.03663193807005882,
0.050445348024368286,
0.05748353153467178,
0.05600574240088463,
0.10596779733896255,
0.053015924990177155,
0.05394246056675911,
0.032114963978528976,
0.06537146866321564,
0.048427484929561615,
0.08060523867607117,
0.5742782950401306,
0.06690707802772522,
0.04075472429394722,
0.033449217677116394,
0.03505101054906845,
0.022565271705389023,
0.06577442586421967,
0.036697570234537125,
0.08553697913885117,
0.07367803901433945,
0.0493597537279129,
0.042231619358062744,
0.023477228358387947,
0.052135858684778214,
0.03512806445360184,
0.06269875913858414,
0.030729109421372414,
0.0728035643696785,
0.023035654798150063,
0.051585447043180466,
0.019568851217627525,
0.04862172156572342,
0.02376176416873932,
0.05972515419125557,
0.02176014520227909,
0.06787826865911484,
0.03424545377492905,
0.06439156830310822,
0.021422436460852623,
0.07232685387134552,
0.019477492198348045,
0.09627991914749146,
0.017491821199655533,
0.1271906942129135
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.00028365814152869,
0.0002107335686261075,
0.00020559059302825965,
0.00013034830149957224,
0.00015811506316286034,
8.078727112532468e-05,
9.745502139202278e-05,
0.00010158534732968689,
0.00010687310625858127,
0.00011249947044062199,
0.00014466041390513986,
0.00018570148639523412,
0.0001198924920321974,
0.00012288623237884872,
0.00013187077826681118,
0.000321118955435273,
0.0002173403860671073,
0.00022706794561596516,
0.0003846029635623724,
0.00045310883243459936,
0.001499135691470673,
0.0011394398675334944,
0.0010162826769661553,
0.0008650696174683238,
0.0006964490714203023,
0.0005559793522399823,
0.0005128319422021078,
0.0003792784055626199,
0.00037680605737877213,
0.0002696585960614849,
0.00021598580145167561,
0.00019464643295656736
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.00015853416346156552,
"first_to_last_ratio": 0.39613292787359505,
"imbalance_abs_log_ratio": 0.9260054476059185,
"last_quartile_mean": 0.000400204457409189,
"mean": 0.0003599206213499198,
"normalized": [
0.7881130579981791,
0.5855001245433764,
0.5712109305023165,
0.36215846986118033,
0.439305373973387,
0.2244585787341764,
0.27076809610549,
0.28224375404965807,
0.29693521270813145,
0.3125674489521634,
0.4019231056074973,
0.5159512275199496,
0.3331081491872516,
0.34142592863379456,
0.36638850469921974,
0.892193823823939,
0.6038564427121444,
0.6308834008018855,
1.0685771827128963,
1.258913231298522,
4.165184217142125,
3.165808792116179,
2.8236300358520197,
2.4035011226191765,
1.9350074158246267,
1.544727696220138,
1.4248473462806275,
1.0537834818691318,
1.0469143334036313,
0.7492168552335295,
0.6000928778173319,
0.540803781196325
],
"population_cv": 0.9317377718256953
},
"depth_weights": [
{
"entropy_mean": 0.0,
"mean_weights": [
1.0
],
"sources": 1
},
{
"entropy_mean": 0.2200722098350525,
"mean_weights": [
0.9384047985076904,
0.06159520894289017
],
"sources": 2
},
{
"entropy_mean": 0.5651201605796814,
"mean_weights": [
0.6557347774505615,
0.3442652225494385
],
"sources": 2
},
{
"entropy_mean": 0.4018177092075348,
"mean_weights": [
0.856421947479248,
0.14357808232307434
],
"sources": 2
},
{
"entropy_mean": 0.687358021736145,
"mean_weights": [
0.46872246265411377,
0.5312775373458862
],
"sources": 2
},
{
"entropy_mean": 0.2678314447402954,
"mean_weights": [
0.920531153678894,
0.07946885377168655
],
"sources": 2
},
{
"entropy_mean": 0.657238245010376,
"mean_weights": [
0.6222448348999023,
0.37775513529777527
],
"sources": 2
},
{
"entropy_mean": 0.5044373869895935,
"mean_weights": [
0.7792346477508545,
0.2207653522491455
],
"sources": 2
},
{
"entropy_mean": 0.650418758392334,
"mean_weights": [
0.5397791862487793,
0.4602207541465759
],
"sources": 2
},
{
"entropy_mean": 1.0032464265823364,
"mean_weights": [
0.4337552487850189,
0.18956376612186432,
0.37668097019195557
],
"sources": 3
},
{
"entropy_mean": 1.0385558605194092,
"mean_weights": [
0.24478834867477417,
0.2602521479129791,
0.4949595034122467
],
"sources": 3
},
{
"entropy_mean": 1.031773328781128,
"mean_weights": [
0.40370821952819824,
0.2651250958442688,
0.33116668462753296
],
"sources": 3
},
{
"entropy_mean": 1.0838522911071777,
"mean_weights": [
0.35612422227859497,
0.2625962495803833,
0.38127952814102173
],
"sources": 3
},
{
"entropy_mean": 0.96053147315979,
"mean_weights": [
0.5271762013435364,
0.1581989824771881,
0.3146248161792755
],
"sources": 3
},
{
"entropy_mean": 1.0704245567321777,
"mean_weights": [
0.3788670599460602,
0.23852786421775818,
0.38260507583618164
],
"sources": 3
},
{
"entropy_mean": 0.9674104452133179,
"mean_weights": [
0.48829180002212524,
0.18356263637542725,
0.3281455934047699
],
"sources": 3
},
{
"entropy_mean": 1.0832805633544922,
"mean_weights": [
0.40388280153274536,
0.2811351418495178,
0.3149820566177368
],
"sources": 3
},
{
"entropy_mean": 1.2917464971542358,
"mean_weights": [
0.3349481225013733,
0.1244347095489502,
0.20291922986507416,
0.33769792318344116
],
"sources": 4
},
{
"entropy_mean": 1.3533791303634644,
"mean_weights": [
0.32115638256073,
0.20602071285247803,
0.18808895349502563,
0.28473395109176636
],
"sources": 4
},
{
"entropy_mean": 1.308367371559143,
"mean_weights": [
0.31139707565307617,
0.16179001331329346,
0.2296154499053955,
0.29719749093055725
],
"sources": 4
},
{
"entropy_mean": 1.3437637090682983,
"mean_weights": [
0.3217787742614746,
0.1920534074306488,
0.1763250231742859,
0.3098427951335907
],
"sources": 4
},
{
"entropy_mean": 1.2280659675598145,
"mean_weights": [
0.425346314907074,
0.14848899841308594,
0.13232898712158203,
0.29383569955825806
],
"sources": 4
},
{
"entropy_mean": 1.3462648391723633,
"mean_weights": [
0.3221249580383301,
0.18557843565940857,
0.18214309215545654,
0.3101535141468048
],
"sources": 4
},
{
"entropy_mean": 1.147221326828003,
"mean_weights": [
0.505380392074585,
0.12089012563228607,
0.11514648050069809,
0.2585829496383667
],
"sources": 4
},
{
"entropy_mean": 1.3729398250579834,
"mean_weights": [
0.30785465240478516,
0.23079943656921387,
0.23862706124782562,
0.22271886467933655
],
"sources": 4
},
{
"entropy_mean": 1.5232064723968506,
"mean_weights": [
0.3136816620826721,
0.11285122483968735,
0.13196846842765808,
0.19688263535499573,
0.24461600184440613
],
"sources": 5
},
{
"entropy_mean": 1.4876129627227783,
"mean_weights": [
0.2035508155822754,
0.15372204780578613,
0.13167491555213928,
0.11437506973743439,
0.396677166223526
],
"sources": 5
},
{
"entropy_mean": 1.4587602615356445,
"mean_weights": [
0.28519001603126526,
0.10532154142856598,
0.09675675630569458,
0.17811742424964905,
0.33461427688598633
],
"sources": 5
},
{
"entropy_mean": 1.4565696716308594,
"mean_weights": [
0.20232918858528137,
0.14928825199604034,
0.12029382586479187,
0.1059027910232544,
0.4221859574317932
],
"sources": 5
},
{
"entropy_mean": 1.4788098335266113,
"mean_weights": [
0.2697942852973938,
0.11948063224554062,
0.11090938746929169,
0.15228068828582764,
0.34753501415252686
],
"sources": 5
},
{
"entropy_mean": 1.5018762350082397,
"mean_weights": [
0.208971306681633,
0.27331358194351196,
0.12766750156879425,
0.07916607707738876,
0.31088152527809143
],
"sources": 5
},
{
"entropy_mean": 1.5627843141555786,
"mean_weights": [
0.27452588081359863,
0.21448183059692383,
0.143874391913414,
0.14059773087501526,
0.22652018070220947
],
"sources": 5
},
{
"entropy_mean": 1.580754280090332,
"mean_weights": [
0.2518634498119354,
0.18758198618888855,
0.13903988897800446,
0.18044015765190125,
0.24107450246810913
],
"sources": 5
},
{
"entropy_mean": 1.7564777135849,
"mean_weights": [
0.15440437197685242,
0.19411900639533997,
0.17806550860404968,
0.14696577191352844,
0.10594475269317627,
0.2205006331205368
],
"sources": 6
},
{
"entropy_mean": 1.5899518728256226,
"mean_weights": [
0.1636897623538971,
0.11789832264184952,
0.07030131667852402,
0.09238424897193909,
0.1399868130683899,
0.4157395362854004
],
"sources": 6
},
{
"entropy_mean": 1.2894535064697266,
"mean_weights": [
0.1212274506688118,
0.06944876164197922,
0.05726535990834236,
0.09096578508615494,
0.06335186958312988,
0.5977407693862915
],
"sources": 6
},
{
"entropy_mean": 1.6581614017486572,
"mean_weights": [
0.18132120370864868,
0.12711825966835022,
0.08560528606176376,
0.09748884290456772,
0.15072739124298096,
0.35773903131484985
],
"sources": 6
},
{
"entropy_mean": 1.3699336051940918,
"mean_weights": [
0.13604651391506195,
0.10151475667953491,
0.07715043425559998,
0.11300846189260483,
0.033717285841703415,
0.538562536239624
],
"sources": 6
},
{
"entropy_mean": 1.6926933526992798,
"mean_weights": [
0.19425281882286072,
0.14001017808914185,
0.09434761106967926,
0.10331346094608307,
0.14820793271064758,
0.3198680281639099
],
"sources": 6
},
{
"entropy_mean": 1.4148362874984741,
"mean_weights": [
0.16652247309684753,
0.10329679399728775,
0.08063912391662598,
0.1165856122970581,
0.033275946974754333,
0.4996800422668457
],
"sources": 6
},
{
"entropy_mean": 1.740808129310608,
"mean_weights": [
0.21459883451461792,
0.15640267729759216,
0.10570268332958221,
0.11097012460231781,
0.17001523077487946,
0.24231043457984924
],
"sources": 6
},
{
"entropy_mean": 1.3179612159729004,
"mean_weights": [
0.08068332821130753,
0.049232255667448044,
0.045073021203279495,
0.06111945956945419,
0.028317857533693314,
0.13192236423492432,
0.6036517024040222
],
"sources": 7
},
{
"entropy_mean": 1.5319744348526,
"mean_weights": [
0.12406684458255768,
0.07462044060230255,
0.05378449708223343,
0.05575074627995491,
0.07357081025838852,
0.10496053099632263,
0.5132461786270142
],
"sources": 7
},
{
"entropy_mean": 1.2461028099060059,
"mean_weights": [
0.08556011319160461,
0.0446893684566021,
0.04084867238998413,
0.05386578291654587,
0.025119207799434662,
0.11240880191326141,
0.6375080347061157
],
"sources": 7
},
{
"entropy_mean": 1.6323832273483276,
"mean_weights": [
0.1418324112892151,
0.08991365134716034,
0.06188817322254181,
0.05739111453294754,
0.09848876297473907,
0.0972195416688919,
0.45326635241508484
],
"sources": 7
},
{
"entropy_mean": 1.301480770111084,
"mean_weights": [
0.09513165056705475,
0.05029121786355972,
0.045042067766189575,
0.05902485549449921,
0.02615261822938919,
0.1109299436211586,
0.6134276390075684
],
"sources": 7
},
{
"entropy_mean": 1.629331111907959,
"mean_weights": [
0.14790503680706024,
0.08658000826835632,
0.06608416140079498,
0.06427294760942459,
0.0819322019815445,
0.097825787961483,
0.45539984107017517
],
"sources": 7
},
{
"entropy_mean": 1.338958501815796,
"mean_weights": [
0.10803166776895523,
0.0559723861515522,
0.047874726355075836,
0.060293328016996384,
0.029215168207883835,
0.10392352938652039,
0.5946891903877258
],
"sources": 7
},
{
"entropy_mean": 1.7611865997314453,
"mean_weights": [
0.1573912799358368,
0.10317619144916534,
0.0853322446346283,
0.08218386769294739,
0.09052872657775879,
0.11341015249490738,
0.3679775595664978
],
"sources": 7
},
{
"entropy_mean": 1.476489782333374,
"mean_weights": [
0.09070298075675964,
0.037138283252716064,
0.030013620853424072,
0.03533655032515526,
0.020730752497911453,
0.06278065592050552,
0.3836256265640259,
0.3396714925765991
],
"sources": 8
},
{
"entropy_mean": 1.7213938236236572,
"mean_weights": [
0.10584281384944916,
0.06887699663639069,
0.05207539349794388,
0.050678279250860214,
0.04768219590187073,
0.06809806823730469,
0.2275032103061676,
0.37924304604530334
],
"sources": 8
},
{
"entropy_mean": 1.5405172109603882,
"mean_weights": [
0.07428755611181259,
0.04491765797138214,
0.03515764698386192,
0.046252522617578506,
0.023427214473485947,
0.07749318331480026,
0.29666566848754883,
0.4017985761165619
],
"sources": 8
},
{
"entropy_mean": 1.8063910007476807,
"mean_weights": [
0.11433833837509155,
0.07374332845211029,
0.05932536721229553,
0.05389238893985748,
0.061260174959897995,
0.07349519431591034,
0.22022446990013123,
0.3437207341194153
],
"sources": 8
},
{
"entropy_mean": 1.4859801530838013,
"mean_weights": [
0.08387120068073273,
0.041406162083148956,
0.032444946467876434,
0.03686721250414848,
0.025052767246961594,
0.060124777257442474,
0.27774184942245483,
0.4424910843372345
],
"sources": 8
},
{
"entropy_mean": 1.7777256965637207,
"mean_weights": [
0.10586117208003998,
0.07087038457393646,
0.05850498005747795,
0.06360965967178345,
0.04672512412071228,
0.07485505938529968,
0.2385886013507843,
0.3409850001335144
],
"sources": 8
},
{
"entropy_mean": 1.6318418979644775,
"mean_weights": [
0.09244748204946518,
0.05762210488319397,
0.045099519193172455,
0.056236449629068375,
0.029373466968536377,
0.0723743885755539,
0.31761035323143005,
0.3292362093925476
],
"sources": 8
},
{
"entropy_mean": 1.8334553241729736,
"mean_weights": [
0.1157165914773941,
0.07161639630794525,
0.06577154994010925,
0.07425849884748459,
0.04881399869918823,
0.08197253942489624,
0.27155569195747375,
0.270294725894928
],
"sources": 8
},
{
"entropy_mean": 1.6227548122406006,
"mean_weights": [
0.0605507455766201,
0.040063511580228806,
0.032619886100292206,
0.03868011012673378,
0.02029048651456833,
0.04760073497891426,
0.19551873207092285,
0.11294309794902802,
0.45173269510269165
],
"sources": 9
},
{
"entropy_mean": 1.9787883758544922,
"mean_weights": [
0.0788881778717041,
0.08108963817358017,
0.057054273784160614,
0.08211588114500046,
0.026040803641080856,
0.10537256300449371,
0.24502354860305786,
0.12593510746955872,
0.1984800100326538
],
"sources": 9
},
{
"entropy_mean": 1.7590525150299072,
"mean_weights": [
0.07131639868021011,
0.044687941670417786,
0.033290669322013855,
0.043597716838121414,
0.02658817172050476,
0.06922741234302521,
0.23554196953773499,
0.14986532926559448,
0.3258844017982483
],
"sources": 9
},
{
"entropy_mean": 1.9736928939819336,
"mean_weights": [
0.09849467873573303,
0.068942591547966,
0.0599900484085083,
0.06335455179214478,
0.04671281576156616,
0.0713595598936081,
0.22560220956802368,
0.1557777225971222,
0.20976582169532776
],
"sources": 9
},
{
"entropy_mean": 1.809161901473999,
"mean_weights": [
0.08854075521230698,
0.05356752127408981,
0.04243364930152893,
0.04702243581414223,
0.02648795023560524,
0.05820769816637039,
0.18489578366279602,
0.21069815754890442,
0.2881460189819336
],
"sources": 9
},
{
"entropy_mean": 1.8544633388519287,
"mean_weights": [
0.07977832853794098,
0.05858254432678223,
0.053744811564683914,
0.07626348733901978,
0.031090255826711655,
0.0994817242026329,
0.3525030016899109,
0.1274903416633606,
0.12106549739837646
],
"sources": 9
},
{
"entropy_mean": 1.9456019401550293,
"mean_weights": [
0.08823960274457932,
0.06737187504768372,
0.05197303742170334,
0.0631948709487915,
0.02739548310637474,
0.08291902393102646,
0.2524128556251526,
0.186361163854599,
0.18013209104537964
],
"sources": 9
}
],
"layer_input_rms_by_sublayer": [
0.06279198825359344,
0.0596221424639225,
0.07690861076116562,
0.07634982466697693,
0.18058517575263977,
0.06968431174755096,
0.15882407128810883,
0.11288930475711823,
0.19892723858356476,
0.11885104328393936,
0.15647411346435547,
0.1495160460472107,
0.1533493995666504,
0.11554427444934845,
0.1527898609638214,
0.13194940984249115,
0.16355258226394653,
0.08884616196155548,
0.11789151281118393,
0.10897023230791092,
0.11456070095300674,
0.09843935817480087,
0.12148918211460114,
0.09431816637516022,
0.15094918012619019,
0.09025302529335022,
0.09854138642549515,
0.0895693451166153,
0.11213528364896774,
0.10807885229587555,
0.16254977881908417,
0.2204425185918808,
0.2323070764541626,
0.15380074083805084,
0.1390199512243271,
0.08548793941736221,
0.14812250435352325,
0.09071414917707443,
0.15333011746406555,
0.10860678553581238,
0.1781008094549179,
0.04808543995022774,
0.07593096792697906,
0.05631518363952637,
0.09691466391086578,
0.07238366454839706,
0.09368696808815002,
0.08572373539209366,
0.10724716633558273,
0.06546958535909653,
0.06538715958595276,
0.06323372572660446,
0.07412076741456985,
0.06869969516992569,
0.07655765861272812,
0.07729499042034149,
0.08420819044113159,
0.04753003641963005,
0.07230556756258011,
0.06484207510948181,
0.07723049819469452,
0.07835857570171356,
0.08422473073005676,
0.08203105628490448
],
"loss_nats": 1.3013269901275635,
"loss_scale": 1.0,
"output_weights": {
"entropy_mean": 1.596132755279541,
"mean_weights": [
0.09668220579624176,
0.03708431497216225,
0.034579962491989136,
0.026012204587459564,
0.0258483849465847,
0.037477150559425354,
0.0845583826303482,
0.3670029640197754,
0.29075440764427185
],
"sources": 9
},
"step": 8000,
"stream_state_rms_by_sublayer": [
0.09253507107496262,
0.15345558524131775,
0.30484533309936523,
0.3249167501926422,
0.33420246839523315,
0.3818626403808594,
0.38682517409324646,
0.40600064396858215,
0.12552575767040253,
0.1351035088300705,
0.14749252796173096,
0.15324510633945465,
0.1726136952638626,
0.18001793324947357,
0.1892337054014206,
0.20391276478767395,
0.031880609691143036,
0.04374876990914345,
0.06123264133930206,
0.08211024105548859,
0.10403046011924744,
0.1380448192358017,
0.15778414905071259,
0.2295975536108017,
0.053015924990177155,
0.08199761807918549,
0.09510950744152069,
0.13892096281051636,
0.1597633957862854,
0.21687114238739014,
0.7120755910873413,
0.7360114455223083,
0.04075472429394722,
0.057135771960020065,
0.07218439877033234,
0.08251863718032837,
0.11449843645095825,
0.12661415338516235,
0.16539782285690308,
0.19619159400463104,
0.0493597537279129,
0.06753352284431458,
0.07179203629493713,
0.095451220870018,
0.10295005142688751,
0.1274663656949997,
0.1323672980070114,
0.15628865361213684,
0.023035654798150063,
0.058436937630176544,
0.06294231116771698,
0.08794140070676804,
0.09175154566764832,
0.12247690558433533,
0.1252463310956955,
0.1640285700559616,
0.03424545377492905,
0.07537907361984253,
0.07851024717092514,
0.12290703505277634,
0.12469635903835297,
0.19157102704048157,
0.19299884140491486,
0.29249706864356995
]
}
],
"environment": {
"autocast": "cuda-bfloat16-forward-fp32-cross-entropy",
"compile": false,
"compute_capability": [
12,
0
],
"cublas_workspace_config": ":4096:8",
"cuda": "12.8",
"deterministic_algorithms": true,
"gpu": "NVIDIA GeForce RTX 5090",
"python": "3.10.14",
"torch": "2.11.0+cu128"
},
"evaluations": [
{
"bits_per_byte": 8.034013546791773,
"cross_entropy_nats": 5.5687538385391235,
"step": 0
},
{
"bits_per_byte": 5.132431212443787,
"cross_entropy_nats": 3.5575302243232727,
"step": 100
},
{
"bits_per_byte": 3.0911456537212687,
"cross_entropy_nats": 2.1426188945770264,
"step": 500
},
{
"bits_per_byte": 1.947918907793682,
"cross_entropy_nats": 1.3501944988965988,
"step": 2000
},
{
"bits_per_byte": 1.7911587423283797,
"cross_entropy_nats": 1.241536632180214,
"step": 4000
},
{
"bits_per_byte": 1.7093240655856785,
"cross_entropy_nats": 1.1848131567239761,
"step": 8000
}
],
"gradient_gate": null,
"hashes": {
"final_mixer_parameters": "91dda38f7c99ebfb81b2efc51836d81bcd8f26bf6deadbc08f763cb6225c959d",
"final_model_state": "bd2556388aeaa211b798c283c7cbd8ccd29edf166a2922fa13d172e8dfdc38d1",
"final_optimizer_state": "0b101eab3bc7d8d654be2ea335c86fc25563ce19912d721844ee4e639c569e77",
"final_public_parameters": "f4d3594a7bf1e78aff69c12541a837933d83661dec81d23a744ee66756f670a9",
"initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d",
"initial_public_parameter_elements": 18985152,
"initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670",
"initial_public_parameter_tensors": 227,
"initial_public_parameters": "bbf2cfc5c0019fb54bf6927274ab9b185b564a77b4b8ef2dc6f5d81575f835c2"
},
"manifest": {
"diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716",
"file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371",
"formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
"input_gate_tensor_hashes": {
"0": "add6249175ac3bb3fa911069fbe8f25d17d9ab05df34b66ee89996983522ce83",
"1": "619b5ee4f70e975312ead9adf5df4f9d8584ce2839369c749442e429923405cf",
"7999": "fe22994f7750c2f8323fdb5ac51c9be1e208733abaf6a337f5b1ae1df1cd2730"
},
"path": "experiments/k3/attnres_gradient/manifest.json",
"validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338"
},
"model": {
"attnres_aggregation_groups": 8,
"context": 256,
"d_ff": 768,
"d_head": 32,
"d_model": 192,
"heads": 6,
"layers": 32,
"parameters": {
"core": 18985152,
"embedding": 98304,
"mixer": 24960,
"total": 19010112
},
"sublayers": 64,
"sublayers_per_attnres_group": 8,
"transformer_blocks_per_attnres_group": 4,
"vocabulary": 256
},
"optimizer": {
"betas": [
0.9,
0.95
],
"epsilon": 1e-08,
"grad_clip": 1.0,
"min_lr": 3e-05,
"name": "AdamW",
"peak_lr": 0.0003,
"warmup_steps": 400,
"weight_decay_ndim_ge_2": 0.1
},
"protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
"run_kind": "formal",
"schema_version": 1,
"seed": 2026073002,
"steps": 8000,
"target_bytes_seen": 65536000,
"timing": {
"mean_ms": 109.03141246394478,
"measured_steps": 7980,
"median_ms": 108.93342149211094,
"p95_ms": 109.53918776649516,
"peak_allocated_bytes": 12815843328,
"peak_reserved_bytes": 13337886720,
"warmup_steps_excluded": 20
},
"training_history": [
{
"bits_per_byte": 8.018870646365954,
"learning_rate": 7.499999999999999e-07,
"loss_nats": 5.558257579803467,
"step": 1,
"unclipped_grad_norm": 26.834381103515625
},
{
"bits_per_byte": 7.214299345307752,
"learning_rate": 7.499999999999999e-06,
"loss_nats": 5.000571250915527,
"step": 10,
"unclipped_grad_norm": 12.740906715393066
},
{
"bits_per_byte": 6.684460697974439,
"learning_rate": 1.4999999999999999e-05,
"loss_nats": 4.633315086364746,
"step": 20,
"unclipped_grad_norm": 3.454355239868164
},
{
"bits_per_byte": 6.541223915307444,
"learning_rate": 2.2499999999999998e-05,
"loss_nats": 4.534030914306641,
"step": 30,
"unclipped_grad_norm": 3.060823440551758
},
{
"bits_per_byte": 6.411011034824987,
"learning_rate": 2.9999999999999997e-05,
"loss_nats": 4.443774223327637,
"step": 40,
"unclipped_grad_norm": 3.0002496242523193
},
{
"bits_per_byte": 6.085588902515397,
"learning_rate": 3.75e-05,
"loss_nats": 4.2182087898254395,
"step": 50,
"unclipped_grad_norm": 2.976402997970581
},
{
"bits_per_byte": 5.879418164452873,
"learning_rate": 4.4999999999999996e-05,
"loss_nats": 4.0753021240234375,
"step": 60,
"unclipped_grad_norm": 2.708517551422119
},
{
"bits_per_byte": 5.710178980836628,
"learning_rate": 5.2499999999999995e-05,
"loss_nats": 3.9579944610595703,
"step": 70,
"unclipped_grad_norm": 2.559852361679077
},
{
"bits_per_byte": 5.512089019607543,
"learning_rate": 5.9999999999999995e-05,
"loss_nats": 3.8206889629364014,
"step": 80,
"unclipped_grad_norm": 2.5563650131225586
},
{
"bits_per_byte": 5.341444737733749,
"learning_rate": 6.749999999999999e-05,
"loss_nats": 3.7024073600769043,
"step": 90,
"unclipped_grad_norm": 2.3861565589904785
},
{
"bits_per_byte": 5.169528816139475,
"learning_rate": 7.5e-05,
"loss_nats": 3.5832443237304688,
"step": 100,
"unclipped_grad_norm": 2.241894006729126
},
{
"bits_per_byte": 5.000991665704052,
"learning_rate": 8.249999999999999e-05,
"loss_nats": 3.466423273086548,
"step": 110,
"unclipped_grad_norm": 2.1930899620056152
},
{
"bits_per_byte": 4.733823840860015,
"learning_rate": 8.999999999999999e-05,
"loss_nats": 3.2812366485595703,
"step": 120,
"unclipped_grad_norm": 1.8853988647460938
},
{
"bits_per_byte": 4.585053344418857,
"learning_rate": 9.75e-05,
"loss_nats": 3.178116798400879,
"step": 130,
"unclipped_grad_norm": 1.6627076864242554
},
{
"bits_per_byte": 4.391709416422983,
"learning_rate": 0.00010499999999999999,
"loss_nats": 3.0441009998321533,
"step": 140,
"unclipped_grad_norm": 1.4586857557296753
},
{
"bits_per_byte": 4.260228335873064,
"learning_rate": 0.0001125,
"loss_nats": 2.952965259552002,
"step": 150,
"unclipped_grad_norm": 1.235063076019287
},
{
"bits_per_byte": 4.153197684831009,
"learning_rate": 0.00011999999999999999,
"loss_nats": 2.878777265548706,
"step": 160,
"unclipped_grad_norm": 1.0407519340515137
},
{
"bits_per_byte": 3.997309514401244,
"learning_rate": 0.00012749999999999998,
"loss_nats": 2.770723819732666,
"step": 170,
"unclipped_grad_norm": 0.9049285054206848
},
{
"bits_per_byte": 3.860879805055398,
"learning_rate": 0.00013499999999999997,
"loss_nats": 2.6761579513549805,
"step": 180,
"unclipped_grad_norm": 1.0858614444732666
},
{
"bits_per_byte": 3.870312021559318,
"learning_rate": 0.0001425,
"loss_nats": 2.6826958656311035,
"step": 190,
"unclipped_grad_norm": 0.6896197199821472
},
{
"bits_per_byte": 3.747811875037455,
"learning_rate": 0.00015,
"loss_nats": 2.597785234451294,
"step": 200,
"unclipped_grad_norm": 0.712465226650238
},
{
"bits_per_byte": 3.69472358243852,
"learning_rate": 0.0001575,
"loss_nats": 2.5609872341156006,
"step": 210,
"unclipped_grad_norm": 0.8807040452957153
},
{
"bits_per_byte": 3.662194783854328,
"learning_rate": 0.00016499999999999997,
"loss_nats": 2.538439989089966,
"step": 220,
"unclipped_grad_norm": 0.7631296515464783
},
{
"bits_per_byte": 3.6752317567203328,
"learning_rate": 0.0001725,
"loss_nats": 2.5474765300750732,
"step": 230,
"unclipped_grad_norm": 0.6869579553604126
},
{
"bits_per_byte": 3.6565805821995627,
"learning_rate": 0.00017999999999999998,
"loss_nats": 2.53454852104187,
"step": 240,
"unclipped_grad_norm": 0.781292736530304
},
{
"bits_per_byte": 3.5245367744160205,
"learning_rate": 0.0001875,
"loss_nats": 2.4430227279663086,
"step": 250,
"unclipped_grad_norm": 1.0127525329589844
},
{
"bits_per_byte": 3.524651658826797,
"learning_rate": 0.000195,
"loss_nats": 2.4431023597717285,
"step": 260,
"unclipped_grad_norm": 0.8615073561668396
},
{
"bits_per_byte": 3.485441678221887,
"learning_rate": 0.00020249999999999996,
"loss_nats": 2.415924072265625,
"step": 270,
"unclipped_grad_norm": 0.7731815576553345
},
{
"bits_per_byte": 3.4956767097400343,
"learning_rate": 0.00020999999999999998,
"loss_nats": 2.423018455505371,
"step": 280,
"unclipped_grad_norm": 0.9061474800109863
},
{
"bits_per_byte": 3.3938416545805445,
"learning_rate": 0.00021749999999999997,
"loss_nats": 2.3524317741394043,
"step": 290,
"unclipped_grad_norm": 1.0149996280670166
},
{
"bits_per_byte": 3.483576010425327,
"learning_rate": 0.000225,
"loss_nats": 2.414630889892578,
"step": 300,
"unclipped_grad_norm": 1.1414116621017456
},
{
"bits_per_byte": 3.381058528107214,
"learning_rate": 0.00023249999999999996,
"loss_nats": 2.343571186065674,
"step": 310,
"unclipped_grad_norm": 1.1472890377044678
},
{
"bits_per_byte": 3.3707543595634197,
"learning_rate": 0.00023999999999999998,
"loss_nats": 2.3364288806915283,
"step": 320,
"unclipped_grad_norm": 1.2828246355056763
},
{
"bits_per_byte": 3.3462052120139605,
"learning_rate": 0.0002475,
"loss_nats": 2.3194127082824707,
"step": 330,
"unclipped_grad_norm": 0.9300209879875183
},
{
"bits_per_byte": 3.3510090314178607,
"learning_rate": 0.00025499999999999996,
"loss_nats": 2.322742462158203,
"step": 340,
"unclipped_grad_norm": 0.7300398945808411
},
{
"bits_per_byte": 3.324897249502662,
"learning_rate": 0.0002625,
"loss_nats": 2.304643154144287,
"step": 350,
"unclipped_grad_norm": 0.96440589427948
},
{
"bits_per_byte": 3.301325651340707,
"learning_rate": 0.00026999999999999995,
"loss_nats": 2.288304567337036,
"step": 360,
"unclipped_grad_norm": 1.186806321144104
},
{
"bits_per_byte": 3.330423052074884,
"learning_rate": 0.00027749999999999997,
"loss_nats": 2.3084733486175537,
"step": 370,
"unclipped_grad_norm": 1.40411376953125
},
{
"bits_per_byte": 3.351273196769586,
"learning_rate": 0.000285,
"loss_nats": 2.322925567626953,
"step": 380,
"unclipped_grad_norm": 0.8571750521659851
},
{
"bits_per_byte": 3.2969108566930587,
"learning_rate": 0.0002925,
"loss_nats": 2.2852444648742676,
"step": 390,
"unclipped_grad_norm": 1.172652006149292
},
{
"bits_per_byte": 3.2316042286462263,
"learning_rate": 0.0003,
"loss_nats": 2.2399773597717285,
"step": 400,
"unclipped_grad_norm": 1.0324530601501465
},
{
"bits_per_byte": 3.191360632309631,
"learning_rate": 0.0002999988466112388,
"loss_nats": 2.212082624435425,
"step": 410,
"unclipped_grad_norm": 0.9553669691085815
},
{
"bits_per_byte": 3.2040835648551598,
"learning_rate": 0.00029999538646466355,
"loss_nats": 2.2209014892578125,
"step": 420,
"unclipped_grad_norm": 1.0212959051132202
},
{
"bits_per_byte": 3.1886670400318176,
"learning_rate": 0.0002999896196193985,
"loss_nats": 2.2102155685424805,
"step": 430,
"unclipped_grad_norm": 0.8366522789001465
},
{
"bits_per_byte": 3.2100664973133752,
"learning_rate": 0.00029998154617398326,
"loss_nats": 2.225048542022705,
"step": 440,
"unclipped_grad_norm": 1.1677532196044922
},
{
"bits_per_byte": 3.1886219805772913,
"learning_rate": 0.0002999711662663707,
"loss_nats": 2.210184335708618,
"step": 450,
"unclipped_grad_norm": 1.3463479280471802
},
{
"bits_per_byte": 3.136037597145384,
"learning_rate": 0.0002999584800739247,
"loss_nats": 2.1737356185913086,
"step": 460,
"unclipped_grad_norm": 1.704139232635498
},
{
"bits_per_byte": 3.1589835224235,
"learning_rate": 0.0002999434878134173,
"loss_nats": 2.189640522003174,
"step": 470,
"unclipped_grad_norm": 1.1849488019943237
},
{
"bits_per_byte": 3.1210654754918714,
"learning_rate": 0.000299926189741025,
"loss_nats": 2.163357734680176,
"step": 480,
"unclipped_grad_norm": 0.8924318552017212
},
{
"bits_per_byte": 3.0823876092087312,
"learning_rate": 0.000299906586152324,
"loss_nats": 2.1365482807159424,
"step": 490,
"unclipped_grad_norm": 1.3279240131378174
},
{
"bits_per_byte": 3.12389562199447,
"learning_rate": 0.0002998846773822856,
"loss_nats": 2.1653194427490234,
"step": 500,
"unclipped_grad_norm": 1.3776742219924927
},
{
"bits_per_byte": 3.102860080002138,
"learning_rate": 0.0002998604638052702,
"loss_nats": 2.1507387161254883,
"step": 510,
"unclipped_grad_norm": 1.061196208000183
},
{
"bits_per_byte": 2.961672642602824,
"learning_rate": 0.0002998339458350211,
"loss_nats": 2.05287504196167,
"step": 520,
"unclipped_grad_norm": 1.0864008665084839
},
{
"bits_per_byte": 3.0521231342057993,
"learning_rate": 0.0002998051239246572,
"loss_nats": 2.115570545196533,
"step": 530,
"unclipped_grad_norm": 0.9000556468963623
},
{
"bits_per_byte": 2.992112883874938,
"learning_rate": 0.00029977399856666537,
"loss_nats": 2.073974609375,
"step": 540,
"unclipped_grad_norm": 0.8655318021774292
},
{
"bits_per_byte": 2.9456885750316344,
"learning_rate": 0.0002997405702928922,
"loss_nats": 2.0417957305908203,
"step": 550,
"unclipped_grad_norm": 0.8011268973350525
},
{
"bits_per_byte": 2.9526635034200264,
"learning_rate": 0.0002997048396745345,
"loss_nats": 2.046630382537842,
"step": 560,
"unclipped_grad_norm": 1.2049078941345215
},
{
"bits_per_byte": 2.943776471919342,
"learning_rate": 0.0002996668073221302,
"loss_nats": 2.0404703617095947,
"step": 570,
"unclipped_grad_norm": 0.9911091923713684
},
{
"bits_per_byte": 2.836173462615215,
"learning_rate": 0.000299626473885547,
"loss_nats": 1.9658856391906738,
"step": 580,
"unclipped_grad_norm": 0.9621384143829346
},
{
"bits_per_byte": 2.898327304706431,
"learning_rate": 0.00029958384005397226,
"loss_nats": 2.008967399597168,
"step": 590,
"unclipped_grad_norm": 1.0092018842697144
},
{
"bits_per_byte": 2.807946982025008,
"learning_rate": 0.0002995389065559004,
"loss_nats": 1.9463205337524414,
"step": 600,
"unclipped_grad_norm": 1.041255235671997
},
{
"bits_per_byte": 2.8662821213364653,
"learning_rate": 0.000299491674159121,
"loss_nats": 1.98675537109375,
"step": 610,
"unclipped_grad_norm": 1.0585267543792725
},
{
"bits_per_byte": 2.7525066546928727,
"learning_rate": 0.00029944214367070535,
"loss_nats": 1.9078922271728516,
"step": 620,
"unclipped_grad_norm": 0.9587937593460083
},
{
"bits_per_byte": 2.7676626258001575,
"learning_rate": 0.00029939031593699284,
"loss_nats": 1.9183975458145142,
"step": 630,
"unclipped_grad_norm": 0.9836012721061707
},
{
"bits_per_byte": 2.7750938241612886,
"learning_rate": 0.00029933619184357654,
"loss_nats": 1.9235484600067139,
"step": 640,
"unclipped_grad_norm": 0.9689151644706726
},
{
"bits_per_byte": 2.6921524390602856,
"learning_rate": 0.00029927977231528774,
"loss_nats": 1.8660578727722168,
"step": 650,
"unclipped_grad_norm": 1.0390667915344238
},
{
"bits_per_byte": 2.6851900654054064,
"learning_rate": 0.00029922105831618057,
"loss_nats": 1.8612319231033325,
"step": 660,
"unclipped_grad_norm": 1.6513670682907104
},
{
"bits_per_byte": 2.6846809967588525,
"learning_rate": 0.0002991600508495153,
"loss_nats": 1.8608790636062622,
"step": 670,
"unclipped_grad_norm": 0.9175130724906921
},
{
"bits_per_byte": 2.7078205745018398,
"learning_rate": 0.00029909675095774114,
"loss_nats": 1.8769181966781616,
"step": 680,
"unclipped_grad_norm": 1.0189234018325806
},
{
"bits_per_byte": 2.5875252455640383,
"learning_rate": 0.0002990311597224787,
"loss_nats": 1.793535828590393,
"step": 690,
"unclipped_grad_norm": 0.9444974064826965
},
{
"bits_per_byte": 2.5413828162854992,
"learning_rate": 0.00029896327826450103,
"loss_nats": 1.761552333831787,
"step": 700,
"unclipped_grad_norm": 1.1904406547546387
},
{
"bits_per_byte": 2.623998466245768,
"learning_rate": 0.0002988931077437152,
"loss_nats": 1.818817138671875,
"step": 710,
"unclipped_grad_norm": 1.1858620643615723
},
{
"bits_per_byte": 2.683836561943116,
"learning_rate": 0.00029882064935914164,
"loss_nats": 1.8602937459945679,
"step": 720,
"unclipped_grad_norm": 0.9852609634399414
},
{
"bits_per_byte": 2.4849188482154285,
"learning_rate": 0.0002987459043488943,
"loss_nats": 1.722414493560791,
"step": 730,
"unclipped_grad_norm": 1.1975579261779785
},
{
"bits_per_byte": 2.5244201674309004,
"learning_rate": 0.0002986688739901591,
"loss_nats": 1.7497947216033936,
"step": 740,
"unclipped_grad_norm": 0.9646193385124207
},
{
"bits_per_byte": 2.5296962511940695,
"learning_rate": 0.0002985895595991723,
"loss_nats": 1.7534518241882324,
"step": 750,
"unclipped_grad_norm": 1.191988468170166
},
{
"bits_per_byte": 2.4988650933565446,
"learning_rate": 0.00029850796253119824,
"loss_nats": 1.7320812940597534,
"step": 760,
"unclipped_grad_norm": 1.092863917350769
},
{
"bits_per_byte": 2.4838112799438723,
"learning_rate": 0.00029842408418050544,
"loss_nats": 1.721646785736084,
"step": 770,
"unclipped_grad_norm": 1.0348618030548096
},
{
"bits_per_byte": 2.4223032327066463,
"learning_rate": 0.0002983379259803436,
"loss_nats": 1.679012656211853,
"step": 780,
"unclipped_grad_norm": 1.034103274345398
},
{
"bits_per_byte": 2.485370818621896,
"learning_rate": 0.0002982494894029186,
"loss_nats": 1.7227277755737305,
"step": 790,
"unclipped_grad_norm": 1.060265302658081
},
{
"bits_per_byte": 2.4233009040643014,
"learning_rate": 0.0002981587759593675,
"loss_nats": 1.679704189300537,
"step": 800,
"unclipped_grad_norm": 1.3169445991516113
},
{
"bits_per_byte": 2.4599308009409677,
"learning_rate": 0.00029806578719973283,
"loss_nats": 1.7050940990447998,
"step": 810,
"unclipped_grad_norm": 0.9821087121963501
},
{
"bits_per_byte": 2.425854846429614,
"learning_rate": 0.00029797052471293596,
"loss_nats": 1.6814744472503662,
"step": 820,
"unclipped_grad_norm": 1.0924170017242432
},
{
"bits_per_byte": 2.4277851797028984,
"learning_rate": 0.0002978729901267499,
"loss_nats": 1.6828124523162842,
"step": 830,
"unclipped_grad_norm": 1.055789828300476
},
{
"bits_per_byte": 2.3982261775338056,
"learning_rate": 0.0002977731851077716,
"loss_nats": 1.6623237133026123,
"step": 840,
"unclipped_grad_norm": 1.230089545249939
},
{
"bits_per_byte": 2.339956047664374,
"learning_rate": 0.0002976711113613935,
"loss_nats": 1.621933937072754,
"step": 850,
"unclipped_grad_norm": 0.94926518201828
},
{
"bits_per_byte": 2.3803994878412724,
"learning_rate": 0.0002975667706317743,
"loss_nats": 1.6499671936035156,
"step": 860,
"unclipped_grad_norm": 1.1705771684646606
},
{
"bits_per_byte": 2.3436123988217177,
"learning_rate": 0.00029746016470180915,
"loss_nats": 1.6244683265686035,
"step": 870,
"unclipped_grad_norm": 1.1496235132217407
},
{
"bits_per_byte": 2.3796778486382517,
"learning_rate": 0.00029735129539309926,
"loss_nats": 1.6494669914245605,
"step": 880,
"unclipped_grad_norm": 1.1764546632766724
},
{
"bits_per_byte": 2.357840007579645,
"learning_rate": 0.0002972401645659207,
"loss_nats": 1.634330153465271,
"step": 890,
"unclipped_grad_norm": 0.9926145672798157
},
{
"bits_per_byte": 2.2929997964819524,
"learning_rate": 0.00029712677411919283,
"loss_nats": 1.5893863439559937,
"step": 900,
"unclipped_grad_norm": 1.009945034980774
},
{
"bits_per_byte": 2.3411444477818364,
"learning_rate": 0.0002970111259904454,
"loss_nats": 1.6227576732635498,
"step": 910,
"unclipped_grad_norm": 1.2830723524093628
},
{
"bits_per_byte": 2.2736926801307917,
"learning_rate": 0.0002968932221557859,
"loss_nats": 1.5760036706924438,
"step": 920,
"unclipped_grad_norm": 1.0900828838348389
},
{
"bits_per_byte": 2.3038844064724087,
"learning_rate": 0.0002967730646298655,
"loss_nats": 1.596930980682373,
"step": 930,
"unclipped_grad_norm": 1.1090145111083984
},
{
"bits_per_byte": 2.2389674911124313,
"learning_rate": 0.0002966506554658449,
"loss_nats": 1.551934003829956,
"step": 940,
"unclipped_grad_norm": 0.9897664189338684
},
{
"bits_per_byte": 2.2434025797128814,
"learning_rate": 0.00029652599675535876,
"loss_nats": 1.5550081729888916,
"step": 950,
"unclipped_grad_norm": 1.0138952732086182
},
{
"bits_per_byte": 2.2236732459538526,
"learning_rate": 0.0002963990906284808,
"loss_nats": 1.5413328409194946,
"step": 960,
"unclipped_grad_norm": 1.0875126123428345
},
{
"bits_per_byte": 2.2667958318658914,
"learning_rate": 0.0002962699392536863,
"loss_nats": 1.5712231397628784,
"step": 970,
"unclipped_grad_norm": 1.245374321937561
},
{
"bits_per_byte": 2.2344890628839638,
"learning_rate": 0.0002961385448378161,
"loss_nats": 1.5488297939300537,
"step": 980,
"unclipped_grad_norm": 1.1757959127426147
},
{
"bits_per_byte": 2.1613678871472266,
"learning_rate": 0.00029600490962603817,
"loss_nats": 1.4981460571289062,
"step": 990,
"unclipped_grad_norm": 1.066755771636963
},
{
"bits_per_byte": 2.207448574654106,
"learning_rate": 0.00029586903590180956,
"loss_nats": 1.5300867557525635,
"step": 1000,
"unclipped_grad_norm": 0.9658553004264832
},
{
"bits_per_byte": 2.213320750285166,
"learning_rate": 0.00029573092598683735,
"loss_nats": 1.5341570377349854,
"step": 1010,
"unclipped_grad_norm": 1.1267611980438232
},
{
"bits_per_byte": 2.2186436133293697,
"learning_rate": 0.00029559058224103895,
"loss_nats": 1.537846565246582,
"step": 1020,
"unclipped_grad_norm": 0.9853240251541138
},
{
"bits_per_byte": 2.1280228589020402,
"learning_rate": 0.0002954480070625017,
"loss_nats": 1.4750330448150635,
"step": 1030,
"unclipped_grad_norm": 0.972481369972229
},
{
"bits_per_byte": 2.2322469250646653,
"learning_rate": 0.00029530320288744205,
"loss_nats": 1.5472756624221802,
"step": 1040,
"unclipped_grad_norm": 1.0531573295593262
},
{
"bits_per_byte": 2.179971250491057,
"learning_rate": 0.0002951561721901638,
"loss_nats": 1.5110409259796143,
"step": 1050,
"unclipped_grad_norm": 0.9985654950141907
},
{
"bits_per_byte": 2.1476089311129005,
"learning_rate": 0.00029500691748301594,
"loss_nats": 1.4886090755462646,
"step": 1060,
"unclipped_grad_norm": 1.100019931793213
},
{
"bits_per_byte": 2.2640807417567204,
"learning_rate": 0.00029485544131634963,
"loss_nats": 1.5693411827087402,
"step": 1070,
"unclipped_grad_norm": 1.0959177017211914
},
{
"bits_per_byte": 2.1797610876917024,
"learning_rate": 0.0002947017462784745,
"loss_nats": 1.5108952522277832,
"step": 1080,
"unclipped_grad_norm": 1.088409423828125
},
{
"bits_per_byte": 2.1080023585151006,
"learning_rate": 0.00029454583499561476,
"loss_nats": 1.461155891418457,
"step": 1090,
"unclipped_grad_norm": 1.0188393592834473
},
{
"bits_per_byte": 2.1918825969071642,
"learning_rate": 0.00029438771013186413,
"loss_nats": 1.5192972421646118,
"step": 1100,
"unclipped_grad_norm": 0.9501848816871643
},
{
"bits_per_byte": 2.205643788715952,
"learning_rate": 0.0002942273743891401,
"loss_nats": 1.5288357734680176,
"step": 1110,
"unclipped_grad_norm": 0.9797388911247253
},
{
"bits_per_byte": 2.0930500148664373,
"learning_rate": 0.00029406483050713824,
"loss_nats": 1.4507917165756226,
"step": 1120,
"unclipped_grad_norm": 0.8856902122497559
},
{
"bits_per_byte": 2.095190166973774,
"learning_rate": 0.00029390008126328504,
"loss_nats": 1.4522751569747925,
"step": 1130,
"unclipped_grad_norm": 0.9447507262229919
},
{
"bits_per_byte": 2.116739593126884,
"learning_rate": 0.00029373312947269045,
"loss_nats": 1.4672120809555054,
"step": 1140,
"unclipped_grad_norm": 1.0488460063934326
},
{
"bits_per_byte": 2.1023439573190634,
"learning_rate": 0.00029356397798809995,
"loss_nats": 1.4572337865829468,
"step": 1150,
"unclipped_grad_norm": 0.9838997721672058
},
{
"bits_per_byte": 2.0562952585852448,
"learning_rate": 0.0002933926296998457,
"loss_nats": 1.425315260887146,
"step": 1160,
"unclipped_grad_norm": 0.9282211065292358
},
{
"bits_per_byte": 2.160992449020393,
"learning_rate": 0.0002932190875357972,
"loss_nats": 1.497885823249817,
"step": 1170,
"unclipped_grad_norm": 1.0253955125808716
},
{
"bits_per_byte": 2.0716469459492117,
"learning_rate": 0.00029304335446131126,
"loss_nats": 1.4359562397003174,
"step": 1180,
"unclipped_grad_norm": 0.9352066516876221
},
{
"bits_per_byte": 2.0893013090255805,
"learning_rate": 0.00029286543347918116,
"loss_nats": 1.4481933116912842,
"step": 1190,
"unclipped_grad_norm": 1.0826658010482788
},
{
"bits_per_byte": 2.0752424152481495,
"learning_rate": 0.0002926853276295856,
"loss_nats": 1.438448429107666,
"step": 1200,
"unclipped_grad_norm": 0.9942638874053955
},
{
"bits_per_byte": 2.2049758081000004,
"learning_rate": 0.0002925030399900368,
"loss_nats": 1.5283727645874023,
"step": 1210,
"unclipped_grad_norm": 1.1482313871383667
},
{
"bits_per_byte": 2.0506430487646385,
"learning_rate": 0.00029231857367532726,
"loss_nats": 1.4213974475860596,
"step": 1220,
"unclipped_grad_norm": 0.9142818450927734
},
{
"bits_per_byte": 2.00269480165204,
"learning_rate": 0.0002921319318374776,
"loss_nats": 1.3881622552871704,
"step": 1230,
"unclipped_grad_norm": 0.9387761354446411
},
{
"bits_per_byte": 2.0166881700394925,
"learning_rate": 0.00029194311766568174,
"loss_nats": 1.3978617191314697,
"step": 1240,
"unclipped_grad_norm": 1.0545530319213867
},
{
"bits_per_byte": 2.1787518934964396,
"learning_rate": 0.000291752134386253,
"loss_nats": 1.5101957321166992,
"step": 1250,
"unclipped_grad_norm": 1.1504697799682617
},
{
"bits_per_byte": 2.1047604855463367,
"learning_rate": 0.00029155898526256856,
"loss_nats": 1.4589087963104248,
"step": 1260,
"unclipped_grad_norm": 0.9371413588523865
},
{
"bits_per_byte": 1.9885089846782722,
"learning_rate": 0.0002913636735950141,
"loss_nats": 1.3783293962478638,
"step": 1270,
"unclipped_grad_norm": 0.9821643233299255
},
{
"bits_per_byte": 2.1346452228561636,
"learning_rate": 0.0002911662027209272,
"loss_nats": 1.4796233177185059,
"step": 1280,
"unclipped_grad_norm": 0.9382128715515137
},
{
"bits_per_byte": 2.022505311222276,
"learning_rate": 0.00029096657601454013,
"loss_nats": 1.4018938541412354,
"step": 1290,
"unclipped_grad_norm": 0.9164487719535828
},
{
"bits_per_byte": 2.050535903573151,
"learning_rate": 0.00029076479688692266,
"loss_nats": 1.4213231801986694,
"step": 1300,
"unclipped_grad_norm": 0.9040083885192871
},
{
"bits_per_byte": 2.128253659619498,
"learning_rate": 0.00029056086878592323,
"loss_nats": 1.4751930236816406,
"step": 1310,
"unclipped_grad_norm": 1.045804738998413
},
{
"bits_per_byte": 2.0627867437420613,
"learning_rate": 0.00029035479519611045,
"loss_nats": 1.4298148155212402,
"step": 1320,
"unclipped_grad_norm": 1.0330690145492554
},
{
"bits_per_byte": 1.954123977343318,
"learning_rate": 0.0002901465796387134,
"loss_nats": 1.3544955253601074,
"step": 1330,
"unclipped_grad_norm": 0.9727330803871155
},
{
"bits_per_byte": 1.9791288789175636,
"learning_rate": 0.00028993622567156154,
"loss_nats": 1.3718276023864746,
"step": 1340,
"unclipped_grad_norm": 0.966886043548584
},
{
"bits_per_byte": 2.1481475807754027,
"learning_rate": 0.0002897237368890237,
"loss_nats": 1.4889824390411377,
"step": 1350,
"unclipped_grad_norm": 0.9399362802505493
},
{
"bits_per_byte": 1.9898208683390537,
"learning_rate": 0.0002895091169219469,
"loss_nats": 1.3792387247085571,
"step": 1360,
"unclipped_grad_norm": 0.9223746061325073
},
{
"bits_per_byte": 2.0666878262115826,
"learning_rate": 0.00028929236943759425,
"loss_nats": 1.4325188398361206,
"step": 1370,
"unclipped_grad_norm": 0.9086402654647827
},
{
"bits_per_byte": 2.05495087020345,
"learning_rate": 0.00028907349813958225,
"loss_nats": 1.4243834018707275,
"step": 1380,
"unclipped_grad_norm": 0.954355776309967
},
{
"bits_per_byte": 2.067550663170961,
"learning_rate": 0.0002888525067678175,
"loss_nats": 1.4331169128417969,
"step": 1390,
"unclipped_grad_norm": 0.8977138996124268
},
{
"bits_per_byte": 2.0721085473841274,
"learning_rate": 0.00028862939909843273,
"loss_nats": 1.4362761974334717,
"step": 1400,
"unclipped_grad_norm": 0.8833977580070496
},
{
"bits_per_byte": 1.9799110560136877,
"learning_rate": 0.0002884041789437225,
"loss_nats": 1.3723697662353516,
"step": 1410,
"unclipped_grad_norm": 0.9529625177383423
},
{
"bits_per_byte": 2.038817521691312,
"learning_rate": 0.0002881768501520778,
"loss_nats": 1.4132006168365479,
"step": 1420,
"unclipped_grad_norm": 1.083083152770996
},
{
"bits_per_byte": 2.0292872750764,
"learning_rate": 0.0002879474166079205,
"loss_nats": 1.4065947532653809,
"step": 1430,
"unclipped_grad_norm": 0.9355389475822449
},
{
"bits_per_byte": 1.987968787171912,
"learning_rate": 0.00028771588223163694,
"loss_nats": 1.3779549598693848,
"step": 1440,
"unclipped_grad_norm": 0.8836045265197754
},
{
"bits_per_byte": 2.0423230440638482,
"learning_rate": 0.00028748225097951054,
"loss_nats": 1.4156304597854614,
"step": 1450,
"unclipped_grad_norm": 0.924534022808075
},
{
"bits_per_byte": 1.9816687187055055,
"learning_rate": 0.000287246526843655,
"loss_nats": 1.3735880851745605,
"step": 1460,
"unclipped_grad_norm": 0.8437807559967041
},
{
"bits_per_byte": 1.9797782854072217,
"learning_rate": 0.00028700871385194533,
"loss_nats": 1.3722777366638184,
"step": 1470,
"unclipped_grad_norm": 0.9143778085708618
},
{
"bits_per_byte": 1.8767507025466492,
"learning_rate": 0.00028676881606794945,
"loss_nats": 1.3008644580841064,
"step": 1480,
"unclipped_grad_norm": 0.8930306434631348
},
{
"bits_per_byte": 2.001770738868954,
"learning_rate": 0.00028652683759085854,
"loss_nats": 1.387521743774414,
"step": 1490,
"unclipped_grad_norm": 0.8246955871582031
},
{
"bits_per_byte": 1.9125255016832254,
"learning_rate": 0.0002862827825554171,
"loss_nats": 1.3256616592407227,
"step": 1500,
"unclipped_grad_norm": 0.9061107635498047
},
{
"bits_per_byte": 1.973825621895558,
"learning_rate": 0.00028603665513185245,
"loss_nats": 1.3681516647338867,
"step": 1510,
"unclipped_grad_norm": 0.9889123439788818
},
{
"bits_per_byte": 1.980869687309597,
"learning_rate": 0.000285788459525803,
"loss_nats": 1.3730342388153076,
"step": 1520,
"unclipped_grad_norm": 0.9045401811599731
},
{
"bits_per_byte": 1.9378121108395843,
"learning_rate": 0.00028553819997824707,
"loss_nats": 1.343189001083374,
"step": 1530,
"unclipped_grad_norm": 0.8896906971931458
},
{
"bits_per_byte": 2.017761685746178,
"learning_rate": 0.00028528588076542964,
"loss_nats": 1.3986058235168457,
"step": 1540,
"unclipped_grad_norm": 0.9600386619567871
},
{
"bits_per_byte": 1.9800400430018348,
"learning_rate": 0.00028503150619878996,
"loss_nats": 1.3724591732025146,
"step": 1550,
"unclipped_grad_norm": 0.8903042078018188
},
{
"bits_per_byte": 2.037820882229562,
"learning_rate": 0.0002847750806248876,
"loss_nats": 1.412509799003601,
"step": 1560,
"unclipped_grad_norm": 0.9084417819976807
},
{
"bits_per_byte": 2.015308525214271,
"learning_rate": 0.000284516608425328,
"loss_nats": 1.3969054222106934,
"step": 1570,
"unclipped_grad_norm": 0.9122006893157959
},
{
"bits_per_byte": 1.985575476602505,
"learning_rate": 0.000284256094016688,
"loss_nats": 1.376296043395996,
"step": 1580,
"unclipped_grad_norm": 0.8903481960296631
},
{
"bits_per_byte": 1.996281568601366,
"learning_rate": 0.0002839935418504399,
"loss_nats": 1.3837169408798218,
"step": 1590,
"unclipped_grad_norm": 0.933029294013977
},
{
"bits_per_byte": 2.002402775110875,
"learning_rate": 0.000283728956412876,
"loss_nats": 1.3879598379135132,
"step": 1600,
"unclipped_grad_norm": 0.9777830243110657
},
{
"bits_per_byte": 2.0119596790366705,
"learning_rate": 0.00028346234222503136,
"loss_nats": 1.3945841789245605,
"step": 1610,
"unclipped_grad_norm": 0.9627588391304016
},
{
"bits_per_byte": 2.0159194075901357,
"learning_rate": 0.00028319370384260675,
"loss_nats": 1.3973288536071777,
"step": 1620,
"unclipped_grad_norm": 0.8731178641319275
},
{
"bits_per_byte": 1.8905791395719242,
"learning_rate": 0.00028292304585589105,
"loss_nats": 1.3104496002197266,
"step": 1630,
"unclipped_grad_norm": 0.8998029828071594
},
{
"bits_per_byte": 2.0235616286638756,
"learning_rate": 0.0002826503728896824,
"loss_nats": 1.4026260375976562,
"step": 1640,
"unclipped_grad_norm": 0.9714035391807556
},
{
"bits_per_byte": 1.9588359580116574,
"learning_rate": 0.0002823756896032095,
"loss_nats": 1.3577616214752197,
"step": 1650,
"unclipped_grad_norm": 0.9218164682388306
},
{
"bits_per_byte": 1.936817019221692,
"learning_rate": 0.0002820990006900518,
"loss_nats": 1.3424992561340332,
"step": 1660,
"unclipped_grad_norm": 0.8974215984344482
},
{
"bits_per_byte": 2.096671109580354,
"learning_rate": 0.00028182031087805947,
"loss_nats": 1.4533016681671143,
"step": 1670,
"unclipped_grad_norm": 0.9378093481063843
},
{
"bits_per_byte": 2.0150522710644854,
"learning_rate": 0.0002815396249292725,
"loss_nats": 1.3967278003692627,
"step": 1680,
"unclipped_grad_norm": 0.8720370531082153
},
{
"bits_per_byte": 1.8838673446393523,
"learning_rate": 0.0002812569476398391,
"loss_nats": 1.3057973384857178,
"step": 1690,
"unclipped_grad_norm": 0.8577803373336792
},
{
"bits_per_byte": 1.927341291107104,
"learning_rate": 0.00028097228383993446,
"loss_nats": 1.3359311819076538,
"step": 1700,
"unclipped_grad_norm": 0.9562329053878784
},
{
"bits_per_byte": 1.9962937793695772,
"learning_rate": 0.0002806856383936774,
"loss_nats": 1.3837254047393799,
"step": 1710,
"unclipped_grad_norm": 0.9462636113166809
},
{
"bits_per_byte": 1.995081645646296,
"learning_rate": 0.00028039701619904774,
"loss_nats": 1.382885217666626,
"step": 1720,
"unclipped_grad_norm": 0.8408851027488708
},
{
"bits_per_byte": 1.894888164889292,
"learning_rate": 0.0002801064221878024,
"loss_nats": 1.3134363889694214,
"step": 1730,
"unclipped_grad_norm": 0.8187386989593506
},
{
"bits_per_byte": 1.9475287866480313,
"learning_rate": 0.00027981386132539134,
"loss_nats": 1.349924087524414,
"step": 1740,
"unclipped_grad_norm": 0.9181671142578125
},
{
"bits_per_byte": 2.0484275682562236,
"learning_rate": 0.0002795193386108724,
"loss_nats": 1.4198617935180664,
"step": 1750,
"unclipped_grad_norm": 0.9259082674980164
},
{
"bits_per_byte": 1.9089950418263137,
"learning_rate": 0.00027922285907682625,
"loss_nats": 1.3232145309448242,
"step": 1760,
"unclipped_grad_norm": 0.9000674486160278
},
{
"bits_per_byte": 1.9653874651136254,
"learning_rate": 0.00027892442778927005,
"loss_nats": 1.3623027801513672,
"step": 1770,
"unclipped_grad_norm": 0.8686579465866089
},
{
"bits_per_byte": 1.9032847018697159,
"learning_rate": 0.00027862404984757116,
"loss_nats": 1.3192564249038696,
"step": 1780,
"unclipped_grad_norm": 0.8219738006591797
},
{
"bits_per_byte": 1.8754542973244455,
"learning_rate": 0.0002783217303843597,
"loss_nats": 1.2999658584594727,
"step": 1790,
"unclipped_grad_norm": 0.8401536345481873
},
{
"bits_per_byte": 1.9695277754504965,
"learning_rate": 0.00027801747456544134,
"loss_nats": 1.3651726245880127,
"step": 1800,
"unclipped_grad_norm": 0.9855192303657532
},
{
"bits_per_byte": 1.952977025044714,
"learning_rate": 0.00027771128758970843,
"loss_nats": 1.3537005186080933,
"step": 1810,
"unclipped_grad_norm": 0.9152796268463135
},
{
"bits_per_byte": 2.0236675699768067,
"learning_rate": 0.00027740317468905165,
"loss_nats": 1.4026994705200195,
"step": 1820,
"unclipped_grad_norm": 0.9160932898521423
},
{
"bits_per_byte": 1.863515605666851,
"learning_rate": 0.0002770931411282704,
"loss_nats": 1.2916905879974365,
"step": 1830,
"unclipped_grad_norm": 0.8749093413352966
},
{
"bits_per_byte": 1.9532654399502112,
"learning_rate": 0.0002767811922049827,
"loss_nats": 1.35390043258667,
"step": 1840,
"unclipped_grad_norm": 0.8691527843475342
},
{
"bits_per_byte": 1.8425976998077092,
"learning_rate": 0.00027646733324953503,
"loss_nats": 1.277191400527954,
"step": 1850,
"unclipped_grad_norm": 0.918012797832489
},
{
"bits_per_byte": 1.9936586611930571,
"learning_rate": 0.00027615156962491097,
"loss_nats": 1.3818988800048828,
"step": 1860,
"unclipped_grad_norm": 0.9178117513656616
},
{
"bits_per_byte": 1.995530520365048,
"learning_rate": 0.0002758339067266396,
"loss_nats": 1.3831963539123535,
"step": 1870,
"unclipped_grad_norm": 0.8769021034240723
},
{
"bits_per_byte": 1.8636951555543517,
"learning_rate": 0.0002755143499827035,
"loss_nats": 1.2918150424957275,
"step": 1880,
"unclipped_grad_norm": 0.8614026308059692
},
{
"bits_per_byte": 1.9426209177403593,
"learning_rate": 0.0002751929048534456,
"loss_nats": 1.3465222120285034,
"step": 1890,
"unclipped_grad_norm": 0.8975904583930969
},
{
"bits_per_byte": 1.861964838104021,
"learning_rate": 0.0002748695768314763,
"loss_nats": 1.2906156778335571,
"step": 1900,
"unclipped_grad_norm": 0.8269307613372803
},
{
"bits_per_byte": 1.8502344174366214,
"learning_rate": 0.00027454437144157947,
"loss_nats": 1.282484769821167,
"step": 1910,
"unclipped_grad_norm": 0.8237605690956116
},
{
"bits_per_byte": 1.9149965484108225,
"learning_rate": 0.00027421729424061787,
"loss_nats": 1.3273744583129883,
"step": 1920,
"unclipped_grad_norm": 0.8641052842140198
},
{
"bits_per_byte": 1.8801597426520518,
"learning_rate": 0.0002738883508174385,
"loss_nats": 1.303227424621582,
"step": 1930,
"unclipped_grad_norm": 0.8907292485237122
},
{
"bits_per_byte": 1.8629336163763313,
"learning_rate": 0.0002735575467927768,
"loss_nats": 1.2912871837615967,
"step": 1940,
"unclipped_grad_norm": 0.8837904930114746
},
{
"bits_per_byte": 1.923144226495448,
"learning_rate": 0.0002732248878191609,
"loss_nats": 1.3330219984054565,
"step": 1950,
"unclipped_grad_norm": 1.0997384786605835
},
{
"bits_per_byte": 1.88083511852199,
"learning_rate": 0.0002728903795808148,
"loss_nats": 1.303695559501648,
"step": 1960,
"unclipped_grad_norm": 0.8385953307151794
},
{
"bits_per_byte": 1.8359603733629608,
"learning_rate": 0.0002725540277935614,
"loss_nats": 1.2725907564163208,
"step": 1970,
"unclipped_grad_norm": 0.839948832988739
},
{
"bits_per_byte": 1.7988822896981462,
"learning_rate": 0.00027221583820472456,
"loss_nats": 1.2468901872634888,
"step": 1980,
"unclipped_grad_norm": 0.7658324837684631
},
{
"bits_per_byte": 1.9198664091526536,
"learning_rate": 0.0002718758165930314,
"loss_nats": 1.3307499885559082,
"step": 1990,
"unclipped_grad_norm": 0.8840731382369995
},
{
"bits_per_byte": 1.863330208369221,
"learning_rate": 0.0002715339687685131,
"loss_nats": 1.2915620803833008,
"step": 2000,
"unclipped_grad_norm": 0.8192090392112732
},
{
"bits_per_byte": 1.9331472534175815,
"learning_rate": 0.00027119030057240576,
"loss_nats": 1.3399555683135986,
"step": 2010,
"unclipped_grad_norm": 0.8343153595924377
},
{
"bits_per_byte": 1.9117808168049895,
"learning_rate": 0.0002708448178770506,
"loss_nats": 1.3251454830169678,
"step": 2020,
"unclipped_grad_norm": 0.913158118724823
},
{
"bits_per_byte": 1.8970498148279866,
"learning_rate": 0.00027049752658579376,
"loss_nats": 1.3149347305297852,
"step": 2030,
"unclipped_grad_norm": 0.8877862691879272
},
{
"bits_per_byte": 1.9115138997308505,
"learning_rate": 0.00027014843263288514,
"loss_nats": 1.324960470199585,
"step": 2040,
"unclipped_grad_norm": 0.8749900460243225
},
{
"bits_per_byte": 1.8657040848990802,
"learning_rate": 0.00026979754198337724,
"loss_nats": 1.2932075262069702,
"step": 2050,
"unclipped_grad_norm": 0.9185486435890198
},
{
"bits_per_byte": 1.7675556498434857,
"learning_rate": 0.00026944486063302305,
"loss_nats": 1.225176215171814,
"step": 2060,
"unclipped_grad_norm": 0.7904725670814514
},
{
"bits_per_byte": 1.8617049723185672,
"learning_rate": 0.0002690903946081738,
"loss_nats": 1.290435552597046,
"step": 2070,
"unclipped_grad_norm": 0.8348630666732788
},
{
"bits_per_byte": 1.973646415973359,
"learning_rate": 0.00026873414996567596,
"loss_nats": 1.3680274486541748,
"step": 2080,
"unclipped_grad_norm": 0.8117388486862183
},
{
"bits_per_byte": 1.8279640400110946,
"learning_rate": 0.00026837613279276736,
"loss_nats": 1.2670481204986572,
"step": 2090,
"unclipped_grad_norm": 0.7758682370185852
},
{
"bits_per_byte": 1.8962239541385433,
"learning_rate": 0.0002680163492069739,
"loss_nats": 1.3143622875213623,
"step": 2100,
"unclipped_grad_norm": 0.8137099146842957
},
{
"bits_per_byte": 1.8350517890184526,
"learning_rate": 0.00026765480535600414,
"loss_nats": 1.271960973739624,
"step": 2110,
"unclipped_grad_norm": 0.9251447319984436
},
{
"bits_per_byte": 1.870564486595114,
"learning_rate": 0.0002672915074176452,
"loss_nats": 1.2965764999389648,
"step": 2120,
"unclipped_grad_norm": 0.9138389825820923
},
{
"bits_per_byte": 1.8882444750864615,
"learning_rate": 0.00026692646159965624,
"loss_nats": 1.3088313341140747,
"step": 2130,
"unclipped_grad_norm": 0.8262670040130615
},
{
"bits_per_byte": 1.7726164113277747,
"learning_rate": 0.0002665596741396633,
"loss_nats": 1.2286840677261353,
"step": 2140,
"unclipped_grad_norm": 0.7359873056411743
},
{
"bits_per_byte": 1.8268258588276844,
"learning_rate": 0.000266191151305052,
"loss_nats": 1.2662591934204102,
"step": 2150,
"unclipped_grad_norm": 0.8173519372940063
},
{
"bits_per_byte": 1.8637190611428216,
"learning_rate": 0.00026582089939286075,
"loss_nats": 1.291831612586975,
"step": 2160,
"unclipped_grad_norm": 0.8174240589141846
},
{
"bits_per_byte": 1.8528119214251022,
"learning_rate": 0.0002654489247296732,
"loss_nats": 1.2842713594436646,
"step": 2170,
"unclipped_grad_norm": 0.8957990407943726
},
{
"bits_per_byte": 1.782856774308099,
"learning_rate": 0.00026507523367151007,
"loss_nats": 1.2357821464538574,
"step": 2180,
"unclipped_grad_norm": 0.8048707246780396
},
{
"bits_per_byte": 1.8923490130319534,
"learning_rate": 0.00026469983260372037,
"loss_nats": 1.3116763830184937,
"step": 2190,
"unclipped_grad_norm": 0.8952655792236328
},
{
"bits_per_byte": 1.9442205283760343,
"learning_rate": 0.0002643227279408727,
"loss_nats": 1.3476309776306152,
"step": 2200,
"unclipped_grad_norm": 0.8895230889320374
},
{
"bits_per_byte": 1.9876000563684622,
"learning_rate": 0.0002639439261266453,
"loss_nats": 1.377699375152588,
"step": 2210,
"unclipped_grad_norm": 0.8686981797218323
},
{
"bits_per_byte": 1.852814845130167,
"learning_rate": 0.000263563433633716,
"loss_nats": 1.284273386001587,
"step": 2220,
"unclipped_grad_norm": 0.8053512573242188
},
{
"bits_per_byte": 1.859576858996791,
"learning_rate": 0.00026318125696365156,
"loss_nats": 1.2889604568481445,
"step": 2230,
"unclipped_grad_norm": 0.8349927663803101
},
{
"bits_per_byte": 1.8206912376714082,
"learning_rate": 0.000262797402646797,
"loss_nats": 1.2620069980621338,
"step": 2240,
"unclipped_grad_norm": 0.8471757769584656
},
{
"bits_per_byte": 1.8270817690121683,
"learning_rate": 0.00026241187724216334,
"loss_nats": 1.2664365768432617,
"step": 2250,
"unclipped_grad_norm": 0.8009039163589478
},
{
"bits_per_byte": 1.8453999851208678,
"learning_rate": 0.0002620246873373162,
"loss_nats": 1.2791337966918945,
"step": 2260,
"unclipped_grad_norm": 0.8003636002540588
},
{
"bits_per_byte": 1.9009572606555896,
"learning_rate": 0.00026163583954826264,
"loss_nats": 1.317643165588379,
"step": 2270,
"unclipped_grad_norm": 0.8468067646026611
},
{
"bits_per_byte": 1.7920475271702074,
"learning_rate": 0.0002612453405193386,
"loss_nats": 1.2421526908874512,
"step": 2280,
"unclipped_grad_norm": 0.9071624875068665
},
{
"bits_per_byte": 1.8281408381761814,
"learning_rate": 0.000260853196923095,
"loss_nats": 1.2671706676483154,
"step": 2290,
"unclipped_grad_norm": 0.822325587272644
},
{
"bits_per_byte": 1.8827026781277094,
"learning_rate": 0.0002604594154601839,
"loss_nats": 1.3049900531768799,
"step": 2300,
"unclipped_grad_norm": 0.8915430307388306
},
{
"bits_per_byte": 1.831042185496237,
"learning_rate": 0.0002600640028592441,
"loss_nats": 1.269181728363037,
"step": 2310,
"unclipped_grad_norm": 0.8500391840934753
},
{
"bits_per_byte": 1.8431096921593277,
"learning_rate": 0.0002596669658767859,
"loss_nats": 1.2775462865829468,
"step": 2320,
"unclipped_grad_norm": 0.8417274355888367
},
{
"bits_per_byte": 1.7906957435344257,
"learning_rate": 0.000259268311297076,
"loss_nats": 1.241215705871582,
"step": 2330,
"unclipped_grad_norm": 0.777140736579895
},
{
"bits_per_byte": 1.825728265549878,
"learning_rate": 0.00025886804593202114,
"loss_nats": 1.265498399734497,
"step": 2340,
"unclipped_grad_norm": 0.8567083477973938
},
{
"bits_per_byte": 1.7348746466132805,
"learning_rate": 0.00025846617662105193,
"loss_nats": 1.2025234699249268,
"step": 2350,
"unclipped_grad_norm": 0.7593752145767212
},
{
"bits_per_byte": 1.8440781264663364,
"learning_rate": 0.00025806271023100637,
"loss_nats": 1.2782175540924072,
"step": 2360,
"unclipped_grad_norm": 0.8551052212715149
},
{
"bits_per_byte": 1.871989878805465,
"learning_rate": 0.00025765765365601165,
"loss_nats": 1.2975645065307617,
"step": 2370,
"unclipped_grad_norm": 0.8494746685028076
},
{
"bits_per_byte": 1.7757177744707837,
"learning_rate": 0.00025725101381736707,
"loss_nats": 1.2308337688446045,
"step": 2380,
"unclipped_grad_norm": 0.7841326594352722
},
{
"bits_per_byte": 1.7735628318554726,
"learning_rate": 0.00025684279766342567,
"loss_nats": 1.2293400764465332,
"step": 2390,
"unclipped_grad_norm": 0.8951802849769592
},
{
"bits_per_byte": 1.8680005692360513,
"learning_rate": 0.00025643301216947504,
"loss_nats": 1.2947993278503418,
"step": 2400,
"unclipped_grad_norm": 0.8339796662330627
},
{
"bits_per_byte": 1.8459044102358484,
"learning_rate": 0.00025602166433761866,
"loss_nats": 1.279483437538147,
"step": 2410,
"unclipped_grad_norm": 0.9356110692024231
},
{
"bits_per_byte": 1.931278833898608,
"learning_rate": 0.0002556087611966561,
"loss_nats": 1.338660478591919,
"step": 2420,
"unclipped_grad_norm": 0.8573734760284424
},
{
"bits_per_byte": 1.825236739133712,
"learning_rate": 0.00025519430980196277,
"loss_nats": 1.265157699584961,
"step": 2430,
"unclipped_grad_norm": 0.8783476948738098
},
{
"bits_per_byte": 1.7732133631089195,
"learning_rate": 0.0002547783172353695,
"loss_nats": 1.229097843170166,
"step": 2440,
"unclipped_grad_norm": 0.8393950462341309
},
{
"bits_per_byte": 1.8696945983470499,
"learning_rate": 0.0002543607906050415,
"loss_nats": 1.295973539352417,
"step": 2450,
"unclipped_grad_norm": 0.8328628540039062
},
{
"bits_per_byte": 1.8721511985319743,
"learning_rate": 0.0002539417370453568,
"loss_nats": 1.2976763248443604,
"step": 2460,
"unclipped_grad_norm": 0.8645251989364624
},
{
"bits_per_byte": 1.8203225068679583,
"learning_rate": 0.00025352116371678464,
"loss_nats": 1.261751413345337,
"step": 2470,
"unclipped_grad_norm": 0.8000954985618591
},
{
"bits_per_byte": 1.8206716316492098,
"learning_rate": 0.0002530990778057627,
"loss_nats": 1.261993408203125,
"step": 2480,
"unclipped_grad_norm": 0.7867690920829773
},
{
"bits_per_byte": 1.7380601092725614,
"learning_rate": 0.00025267548652457475,
"loss_nats": 1.2047314643859863,
"step": 2490,
"unclipped_grad_norm": 0.7984139919281006
},
{
"bits_per_byte": 1.822108202731867,
"learning_rate": 0.00025225039711122696,
"loss_nats": 1.2629891633987427,
"step": 2500,
"unclipped_grad_norm": 0.7798662781715393
},
{
"bits_per_byte": 1.7878630172920644,
"learning_rate": 0.00025182381682932464,
"loss_nats": 1.2392522096633911,
"step": 2510,
"unclipped_grad_norm": 0.8921790719032288
},
{
"bits_per_byte": 1.7912486677569497,
"learning_rate": 0.0002513957529679477,
"loss_nats": 1.2415989637374878,
"step": 2520,
"unclipped_grad_norm": 0.8250295519828796
},
{
"bits_per_byte": 1.7871405181757898,
"learning_rate": 0.00025096621284152656,
"loss_nats": 1.2387514114379883,
"step": 2530,
"unclipped_grad_norm": 0.7922425270080566
},
{
"bits_per_byte": 1.7956409326773348,
"learning_rate": 0.0002505352037897168,
"loss_nats": 1.2446434497833252,
"step": 2540,
"unclipped_grad_norm": 0.8244069218635559
},
{
"bits_per_byte": 1.833951787983534,
"learning_rate": 0.0002501027331772739,
"loss_nats": 1.2711985111236572,
"step": 2550,
"unclipped_grad_norm": 0.8060020804405212
},
{
"bits_per_byte": 1.7851159384098327,
"learning_rate": 0.00024966880839392746,
"loss_nats": 1.2373480796813965,
"step": 2560,
"unclipped_grad_norm": 0.7821792960166931
},
{
"bits_per_byte": 1.768823849910948,
"learning_rate": 0.0002492334368542548,
"loss_nats": 1.2260552644729614,
"step": 2570,
"unclipped_grad_norm": 0.7924907207489014
},
{
"bits_per_byte": 1.8169237857216074,
"learning_rate": 0.00024879662599755416,
"loss_nats": 1.2593955993652344,
"step": 2580,
"unclipped_grad_norm": 0.813592255115509
},
{
"bits_per_byte": 1.7917042497990852,
"learning_rate": 0.00024835838328771803,
"loss_nats": 1.2419147491455078,
"step": 2590,
"unclipped_grad_norm": 0.8239418268203735
},
{
"bits_per_byte": 1.8340940176357976,
"learning_rate": 0.0002479187162131051,
"loss_nats": 1.2712970972061157,
"step": 2600,
"unclipped_grad_norm": 0.7961696982383728
},
{
"bits_per_byte": 1.7288738279593754,
"learning_rate": 0.00024747763228641264,
"loss_nats": 1.198364019393921,
"step": 2610,
"unclipped_grad_norm": 0.7893867492675781
},
{
"bits_per_byte": 1.7710386424887592,
"learning_rate": 0.00024703513904454795,
"loss_nats": 1.2275904417037964,
"step": 2620,
"unclipped_grad_norm": 0.7791327238082886
},
{
"bits_per_byte": 1.783225161146247,
"learning_rate": 0.00024659124404849956,
"loss_nats": 1.2360374927520752,
"step": 2630,
"unclipped_grad_norm": 0.7614359259605408
},
{
"bits_per_byte": 1.8200944578729141,
"learning_rate": 0.00024614595488320833,
"loss_nats": 1.2615933418273926,
"step": 2640,
"unclipped_grad_norm": 0.831463098526001
},
{
"bits_per_byte": 1.7566091260987187,
"learning_rate": 0.00024569927915743745,
"loss_nats": 1.2175886631011963,
"step": 2650,
"unclipped_grad_norm": 0.7739477157592773
},
{
"bits_per_byte": 1.7470520501902724,
"learning_rate": 0.0002452512245036427,
"loss_nats": 1.2109642028808594,
"step": 2660,
"unclipped_grad_norm": 0.7929992079734802
},
{
"bits_per_byte": 1.775358846678433,
"learning_rate": 0.00024480179857784177,
"loss_nats": 1.230584979057312,
"step": 2670,
"unclipped_grad_norm": 0.7438913583755493
},
{
"bits_per_byte": 1.7678870604116983,
"learning_rate": 0.00024435100905948387,
"loss_nats": 1.2254059314727783,
"step": 2680,
"unclipped_grad_norm": 0.8285056352615356
},
{
"bits_per_byte": 1.8436208245976922,
"learning_rate": 0.00024389886365131793,
"loss_nats": 1.2779005765914917,
"step": 2690,
"unclipped_grad_norm": 0.7684076428413391
},
{
"bits_per_byte": 1.825501592416041,
"learning_rate": 0.00024344537007926163,
"loss_nats": 1.2653412818908691,
"step": 2700,
"unclipped_grad_norm": 0.8987715840339661
},
{
"bits_per_byte": 1.7152053347994027,
"learning_rate": 0.00024299053609226878,
"loss_nats": 1.188889741897583,
"step": 2710,
"unclipped_grad_norm": 0.8146651983261108
},
{
"bits_per_byte": 1.8198911743795942,
"learning_rate": 0.00024253436946219733,
"loss_nats": 1.2614524364471436,
"step": 2720,
"unclipped_grad_norm": 0.8013731241226196
},
{
"bits_per_byte": 1.7448410412307802,
"learning_rate": 0.0002420768779836764,
"loss_nats": 1.2094316482543945,
"step": 2730,
"unclipped_grad_norm": 0.7481213212013245
},
{
"bits_per_byte": 1.7681415947349752,
"learning_rate": 0.00024161806947397305,
"loss_nats": 1.2255823612213135,
"step": 2740,
"unclipped_grad_norm": 0.8155582547187805
},
{
"bits_per_byte": 1.781660806953999,
"learning_rate": 0.00024115795177285886,
"loss_nats": 1.2349531650543213,
"step": 2750,
"unclipped_grad_norm": 0.816902756690979
},
{
"bits_per_byte": 1.869134278870539,
"learning_rate": 0.0002406965327424758,
"loss_nats": 1.2955851554870605,
"step": 2760,
"unclipped_grad_norm": 0.8422117233276367
},
{
"bits_per_byte": 1.7766440730283308,
"learning_rate": 0.00024023382026720197,
"loss_nats": 1.231475830078125,
"step": 2770,
"unclipped_grad_norm": 0.8072130084037781
},
{
"bits_per_byte": 1.778421169759696,
"learning_rate": 0.00023976982225351698,
"loss_nats": 1.2327076196670532,
"step": 2780,
"unclipped_grad_norm": 0.8882554173469543
},
{
"bits_per_byte": 1.7671284449387423,
"learning_rate": 0.0002393045466298666,
"loss_nats": 1.2248800992965698,
"step": 2790,
"unclipped_grad_norm": 0.8406802415847778
},
{
"bits_per_byte": 1.719355792112675,
"learning_rate": 0.00023883800134652763,
"loss_nats": 1.191766619682312,
"step": 2800,
"unclipped_grad_norm": 0.8179776072502136
},
{
"bits_per_byte": 1.7902867687906738,
"learning_rate": 0.0002383701943754715,
"loss_nats": 1.2409322261810303,
"step": 2810,
"unclipped_grad_norm": 0.8046079874038696
},
{
"bits_per_byte": 1.7916516231079211,
"learning_rate": 0.00023790113371022884,
"loss_nats": 1.2418782711029053,
"step": 2820,
"unclipped_grad_norm": 0.7889978885650635
},
{
"bits_per_byte": 1.717342391219024,
"learning_rate": 0.0002374308273657521,
"loss_nats": 1.190371036529541,
"step": 2830,
"unclipped_grad_norm": 0.7270697355270386
},
{
"bits_per_byte": 1.8021411889493455,
"learning_rate": 0.00023695928337827911,
"loss_nats": 1.2491490840911865,
"step": 2840,
"unclipped_grad_norm": 0.8529788255691528
},
{
"bits_per_byte": 1.7264863648000979,
"learning_rate": 0.00023648650980519564,
"loss_nats": 1.196709156036377,
"step": 2850,
"unclipped_grad_norm": 0.7487845420837402
},
{
"bits_per_byte": 1.698717702026487,
"learning_rate": 0.00023601251472489757,
"loss_nats": 1.1774613857269287,
"step": 2860,
"unclipped_grad_norm": 0.8107967972755432
},
{
"bits_per_byte": 1.8323354950307253,
"learning_rate": 0.00023553730623665305,
"loss_nats": 1.270078182220459,
"step": 2870,
"unclipped_grad_norm": 0.822874128818512
},
{
"bits_per_byte": 1.7418315168233318,
"learning_rate": 0.00023506089246046403,
"loss_nats": 1.2073456048965454,
"step": 2880,
"unclipped_grad_norm": 0.7772940993309021
},
{
"bits_per_byte": 1.7973201712803593,
"learning_rate": 0.00023458328153692736,
"loss_nats": 1.245807409286499,
"step": 2890,
"unclipped_grad_norm": 0.7877682447433472
},
{
"bits_per_byte": 1.7076477291898875,
"learning_rate": 0.0002341044816270961,
"loss_nats": 1.1836512088775635,
"step": 2900,
"unclipped_grad_norm": 0.8104777932167053
},
{
"bits_per_byte": 1.7337485042154308,
"learning_rate": 0.00023362450091233953,
"loss_nats": 1.2017428874969482,
"step": 2910,
"unclipped_grad_norm": 0.8031578063964844
},
{
"bits_per_byte": 1.8229534974608574,
"learning_rate": 0.00023314334759420388,
"loss_nats": 1.2635750770568848,
"step": 2920,
"unclipped_grad_norm": 0.7966766953468323
},
{
"bits_per_byte": 1.766033431400699,
"learning_rate": 0.0002326610298942717,
"loss_nats": 1.22412109375,
"step": 2930,
"unclipped_grad_norm": 0.8636581301689148
},
{
"bits_per_byte": 1.8135838826418518,
"learning_rate": 0.00023217755605402175,
"loss_nats": 1.2570805549621582,
"step": 2940,
"unclipped_grad_norm": 0.8673250675201416
},
{
"bits_per_byte": 1.7613176671140407,
"learning_rate": 0.00023169293433468802,
"loss_nats": 1.2208523750305176,
"step": 2950,
"unclipped_grad_norm": 0.7721947431564331
},
{
"bits_per_byte": 1.7305283010606756,
"learning_rate": 0.00023120717301711857,
"loss_nats": 1.1995108127593994,
"step": 2960,
"unclipped_grad_norm": 0.7505601644515991
},
{
"bits_per_byte": 1.8081494028572378,
"learning_rate": 0.00023072028040163408,
"loss_nats": 1.253313660621643,
"step": 2970,
"unclipped_grad_norm": 0.7939547896385193
},
{
"bits_per_byte": 1.72656255311443,
"learning_rate": 0.00023023226480788593,
"loss_nats": 1.196761965751648,
"step": 2980,
"unclipped_grad_norm": 0.7610272765159607
},
{
"bits_per_byte": 1.8584056571444152,
"learning_rate": 0.00022974313457471416,
"loss_nats": 1.2881486415863037,
"step": 2990,
"unclipped_grad_norm": 0.833854615688324
},
{
"bits_per_byte": 1.7959412143857414,
"learning_rate": 0.0002292528980600049,
"loss_nats": 1.2448515892028809,
"step": 3000,
"unclipped_grad_norm": 0.7871666550636292
},
{
"bits_per_byte": 1.7543190051198296,
"learning_rate": 0.00022876156364054758,
"loss_nats": 1.216001272201538,
"step": 3010,
"unclipped_grad_norm": 0.7730440497398376
},
{
"bits_per_byte": 1.7304921847039942,
"learning_rate": 0.0002282691397118918,
"loss_nats": 1.1994857788085938,
"step": 3020,
"unclipped_grad_norm": 0.8199000954627991
},
{
"bits_per_byte": 1.7578830015936595,
"learning_rate": 0.00022777563468820376,
"loss_nats": 1.218471646308899,
"step": 3030,
"unclipped_grad_norm": 0.9139093160629272
},
{
"bits_per_byte": 1.7370483353375354,
"learning_rate": 0.00022728105700212282,
"loss_nats": 1.204030156135559,
"step": 3040,
"unclipped_grad_norm": 0.7832452058792114
},
{
"bits_per_byte": 1.745498014957076,
"learning_rate": 0.00022678541510461698,
"loss_nats": 1.2098870277404785,
"step": 3050,
"unclipped_grad_norm": 0.7990915179252625
},
{
"bits_per_byte": 1.7370017280391516,
"learning_rate": 0.00022628871746483876,
"loss_nats": 1.2039978504180908,
"step": 3060,
"unclipped_grad_norm": 0.8140543103218079
},
{
"bits_per_byte": 1.7324813360438736,
"learning_rate": 0.0002257909725699804,
"loss_nats": 1.200864553451538,
"step": 3070,
"unclipped_grad_norm": 0.8290132284164429
},
{
"bits_per_byte": 1.73554830265671,
"learning_rate": 0.00022529218892512886,
"loss_nats": 1.2029904127120972,
"step": 3080,
"unclipped_grad_norm": 0.8363073468208313
},
{
"bits_per_byte": 1.7064868462965634,
"learning_rate": 0.00022479237505312034,
"loss_nats": 1.1828465461730957,
"step": 3090,
"unclipped_grad_norm": 0.7568578124046326
},
{
"bits_per_byte": 1.8015361539836103,
"learning_rate": 0.00022429153949439496,
"loss_nats": 1.2487297058105469,
"step": 3100,
"unclipped_grad_norm": 0.8538148999214172
},
{
"bits_per_byte": 1.7730212584879057,
"learning_rate": 0.00022378969080685053,
"loss_nats": 1.2289646863937378,
"step": 3110,
"unclipped_grad_norm": 0.8123647570610046
},
{
"bits_per_byte": 1.664611306585931,
"learning_rate": 0.0002232868375656965,
"loss_nats": 1.1538206338882446,
"step": 3120,
"unclipped_grad_norm": 0.814990758895874
},
{
"bits_per_byte": 1.7603762340832176,
"learning_rate": 0.00022278298836330733,
"loss_nats": 1.2201998233795166,
"step": 3130,
"unclipped_grad_norm": 0.8308927416801453
},
{
"bits_per_byte": 1.713619138810495,
"learning_rate": 0.00022227815180907572,
"loss_nats": 1.1877902746200562,
"step": 3140,
"unclipped_grad_norm": 0.7710813879966736
},
{
"bits_per_byte": 1.7093383186478688,
"learning_rate": 0.00022177233652926546,
"loss_nats": 1.1848230361938477,
"step": 3150,
"unclipped_grad_norm": 0.7859835624694824
},
{
"bits_per_byte": 1.7563532159142348,
"learning_rate": 0.00022126555116686416,
"loss_nats": 1.2174112796783447,
"step": 3160,
"unclipped_grad_norm": 0.811743974685669
},
{
"bits_per_byte": 1.8161149513145995,
"learning_rate": 0.00022075780438143532,
"loss_nats": 1.258834958076477,
"step": 3170,
"unclipped_grad_norm": 0.8695415258407593
},
{
"bits_per_byte": 1.6889931150161004,
"learning_rate": 0.00022024910484897065,
"loss_nats": 1.1707208156585693,
"step": 3180,
"unclipped_grad_norm": 0.8201394081115723
},
{
"bits_per_byte": 1.7704115937437135,
"learning_rate": 0.00021973946126174158,
"loss_nats": 1.2271558046340942,
"step": 3190,
"unclipped_grad_norm": 0.8031932711601257
},
{
"bits_per_byte": 1.6638831320421776,
"learning_rate": 0.00021922888232815086,
"loss_nats": 1.1533159017562866,
"step": 3200,
"unclipped_grad_norm": 0.7531365156173706
},
{
"bits_per_byte": 1.7080410535124106,
"learning_rate": 0.0002187173767725837,
"loss_nats": 1.1839238405227661,
"step": 3210,
"unclipped_grad_norm": 0.7989531755447388
},
{
"bits_per_byte": 1.7142219380017691,
"learning_rate": 0.0002182049533352588,
"loss_nats": 1.1882081031799316,
"step": 3220,
"unclipped_grad_norm": 0.8516196012496948
},
{
"bits_per_byte": 1.687988048404458,
"learning_rate": 0.00021769162077207884,
"loss_nats": 1.1700241565704346,
"step": 3230,
"unclipped_grad_norm": 0.8453141450881958
},
{
"bits_per_byte": 1.725164850110868,
"learning_rate": 0.00021717738785448102,
"loss_nats": 1.1957931518554688,
"step": 3240,
"unclipped_grad_norm": 0.8131914734840393
},
{
"bits_per_byte": 1.7438406181507111,
"learning_rate": 0.00021666226336928708,
"loss_nats": 1.2087382078170776,
"step": 3250,
"unclipped_grad_norm": 0.8390238285064697
},
{
"bits_per_byte": 1.8351064795014271,
"learning_rate": 0.0002161462561185532,
"loss_nats": 1.2719988822937012,
"step": 3260,
"unclipped_grad_norm": 0.8244672417640686
},
{
"bits_per_byte": 1.6914478233918653,
"learning_rate": 0.00021562937491941957,
"loss_nats": 1.1724222898483276,
"step": 3270,
"unclipped_grad_norm": 0.8076629042625427
},
{
"bits_per_byte": 1.6199020087370317,
"learning_rate": 0.00021511162860395975,
"loss_nats": 1.1228305101394653,
"step": 3280,
"unclipped_grad_norm": 0.7831076979637146
},
{
"bits_per_byte": 1.6792207168287738,
"learning_rate": 0.0002145930260190298,
"loss_nats": 1.1639471054077148,
"step": 3290,
"unclipped_grad_norm": 0.8022177219390869
},
{
"bits_per_byte": 1.7567814527148833,
"learning_rate": 0.00021407357602611698,
"loss_nats": 1.2177081108093262,
"step": 3300,
"unclipped_grad_norm": 0.8066262602806091
},
{
"bits_per_byte": 1.745768887632185,
"learning_rate": 0.0002135532875011885,
"loss_nats": 1.210074782371521,
"step": 3310,
"unclipped_grad_norm": 0.8170367479324341
},
{
"bits_per_byte": 1.7488066171943748,
"learning_rate": 0.0002130321693345397,
"loss_nats": 1.2121803760528564,
"step": 3320,
"unclipped_grad_norm": 0.8205949664115906
},
{
"bits_per_byte": 1.6972405430382262,
"learning_rate": 0.00021251023043064224,
"loss_nats": 1.176437497138977,
"step": 3330,
"unclipped_grad_norm": 0.7750418782234192
},
{
"bits_per_byte": 1.7083963696690934,
"learning_rate": 0.00021198747970799188,
"loss_nats": 1.184170126914978,
"step": 3340,
"unclipped_grad_norm": 0.7907351851463318
},
{
"bits_per_byte": 1.7221589373390878,
"learning_rate": 0.00021146392609895618,
"loss_nats": 1.1937096118927002,
"step": 3350,
"unclipped_grad_norm": 0.7935147285461426
},
{
"bits_per_byte": 1.6939794080125659,
"learning_rate": 0.00021093957854962178,
"loss_nats": 1.1741770505905151,
"step": 3360,
"unclipped_grad_norm": 0.7715696096420288
},
{
"bits_per_byte": 1.6536716621471985,
"learning_rate": 0.00021041444601964157,
"loss_nats": 1.146237850189209,
"step": 3370,
"unclipped_grad_norm": 0.7847044467926025
},
{
"bits_per_byte": 1.7430156173745224,
"learning_rate": 0.00020988853748208164,
"loss_nats": 1.2081663608551025,
"step": 3380,
"unclipped_grad_norm": 0.9182605743408203
},
{
"bits_per_byte": 1.7282608817917002,
"learning_rate": 0.00020936186192326782,
"loss_nats": 1.197939157485962,
"step": 3390,
"unclipped_grad_norm": 0.8387140035629272
},
{
"bits_per_byte": 1.7168453613580306,
"learning_rate": 0.00020883442834263225,
"loss_nats": 1.1900265216827393,
"step": 3400,
"unclipped_grad_norm": 0.827458918094635
},
{
"bits_per_byte": 1.744452876387783,
"learning_rate": 0.0002083062457525596,
"loss_nats": 1.2091625928878784,
"step": 3410,
"unclipped_grad_norm": 0.8461803197860718
},
{
"bits_per_byte": 1.7169175940713928,
"learning_rate": 0.00020777732317823306,
"loss_nats": 1.1900765895843506,
"step": 3420,
"unclipped_grad_norm": 0.7754979133605957
},
{
"bits_per_byte": 1.665198971303929,
"learning_rate": 0.00020724766965748005,
"loss_nats": 1.1542279720306396,
"step": 3430,
"unclipped_grad_norm": 0.784263551235199
},
{
"bits_per_byte": 1.6807204055442975,
"learning_rate": 0.00020671729424061788,
"loss_nats": 1.1649866104125977,
"step": 3440,
"unclipped_grad_norm": 0.768701434135437
},
{
"bits_per_byte": 1.674807642007634,
"learning_rate": 0.00020618620599029906,
"loss_nats": 1.1608881950378418,
"step": 3450,
"unclipped_grad_norm": 0.8573304414749146
},
{
"bits_per_byte": 1.6607244986764313,
"learning_rate": 0.00020565441398135649,
"loss_nats": 1.151126503944397,
"step": 3460,
"unclipped_grad_norm": 0.7546505331993103
},
{
"bits_per_byte": 1.7168016777647113,
"learning_rate": 0.00020512192730064827,
"loss_nats": 1.1899962425231934,
"step": 3470,
"unclipped_grad_norm": 0.8346678614616394
},
{
"bits_per_byte": 1.7794081781929978,
"learning_rate": 0.00020458875504690273,
"loss_nats": 1.2333917617797852,
"step": 3480,
"unclipped_grad_norm": 0.8080501556396484
},
{
"bits_per_byte": 1.7098234817059528,
"learning_rate": 0.00020405490633056245,
"loss_nats": 1.1851593255996704,
"step": 3490,
"unclipped_grad_norm": 0.7828541398048401
},
{
"bits_per_byte": 1.6618220919542377,
"learning_rate": 0.00020352039027362917,
"loss_nats": 1.15188729763031,
"step": 3500,
"unclipped_grad_norm": 0.8354499936103821
},
{
"bits_per_byte": 1.670962969847596,
"learning_rate": 0.0002029852160095075,
"loss_nats": 1.158223271369934,
"step": 3510,
"unclipped_grad_norm": 0.7484581470489502
},
{
"bits_per_byte": 1.7077951183216769,
"learning_rate": 0.000202449392682849,
"loss_nats": 1.1837533712387085,
"step": 3520,
"unclipped_grad_norm": 0.8215554356575012
},
{
"bits_per_byte": 1.6640239858332344,
"learning_rate": 0.0002019129294493959,
"loss_nats": 1.1534135341644287,
"step": 3530,
"unclipped_grad_norm": 0.7679988741874695
},
{
"bits_per_byte": 1.6934656958344385,
"learning_rate": 0.00020137583547582478,
"loss_nats": 1.173820972442627,
"step": 3540,
"unclipped_grad_norm": 0.849611759185791
},
{
"bits_per_byte": 1.7380399873024104,
"learning_rate": 0.00020083811993958973,
"loss_nats": 1.2047175168991089,
"step": 3550,
"unclipped_grad_norm": 0.8348926305770874
},
{
"bits_per_byte": 1.6724150193688307,
"learning_rate": 0.00020029979202876563,
"loss_nats": 1.1592297554016113,
"step": 3560,
"unclipped_grad_norm": 0.82209312915802
},
{
"bits_per_byte": 1.707797526078789,
"learning_rate": 0.00019976086094189118,
"loss_nats": 1.1837550401687622,
"step": 3570,
"unclipped_grad_norm": 0.814924955368042
},
{
"bits_per_byte": 1.725196322935976,
"learning_rate": 0.00019922133588781175,
"loss_nats": 1.1958149671554565,
"step": 3580,
"unclipped_grad_norm": 0.8165888786315918
},
{
"bits_per_byte": 1.7086591591596116,
"learning_rate": 0.0001986812260855219,
"loss_nats": 1.1843522787094116,
"step": 3590,
"unclipped_grad_norm": 0.8172020316123962
},
{
"bits_per_byte": 1.7535423314685332,
"learning_rate": 0.00019814054076400787,
"loss_nats": 1.2154629230499268,
"step": 3600,
"unclipped_grad_norm": 0.8490144610404968
},
{
"bits_per_byte": 1.7320164669385913,
"learning_rate": 0.00019759928916209004,
"loss_nats": 1.2005423307418823,
"step": 3610,
"unclipped_grad_norm": 0.8558303117752075
},
{
"bits_per_byte": 1.671523289324107,
"learning_rate": 0.00019705748052826493,
"loss_nats": 1.1586116552352905,
"step": 3620,
"unclipped_grad_norm": 0.7700185179710388
},
{
"bits_per_byte": 1.6783824733884685,
"learning_rate": 0.00019651512412054723,
"loss_nats": 1.1633660793304443,
"step": 3630,
"unclipped_grad_norm": 0.8705776333808899
},
{
"bits_per_byte": 1.6085100499265301,
"learning_rate": 0.00019597222920631144,
"loss_nats": 1.1149342060089111,
"step": 3640,
"unclipped_grad_norm": 0.7363900542259216
},
{
"bits_per_byte": 1.69346002040696,
"learning_rate": 0.0001954288050621338,
"loss_nats": 1.1738170385360718,
"step": 3650,
"unclipped_grad_norm": 0.7923496961593628
},
{
"bits_per_byte": 1.6919579239343243,
"learning_rate": 0.00019488486097363363,
"loss_nats": 1.1727758646011353,
"step": 3660,
"unclipped_grad_norm": 0.8016297221183777
},
{
"bits_per_byte": 1.7219477426438283,
"learning_rate": 0.00019434040623531452,
"loss_nats": 1.1935632228851318,
"step": 3670,
"unclipped_grad_norm": 0.8946071863174438
},
{
"bits_per_byte": 1.6611216066172736,
"learning_rate": 0.0001937954501504058,
"loss_nats": 1.1514017581939697,
"step": 3680,
"unclipped_grad_norm": 0.7832621932029724
},
{
"bits_per_byte": 1.7299588665036687,
"learning_rate": 0.00019325000203070333,
"loss_nats": 1.1991161108016968,
"step": 3690,
"unclipped_grad_norm": 0.8317174911499023
},
{
"bits_per_byte": 1.6188276331170917,
"learning_rate": 0.0001927040711964106,
"loss_nats": 1.1220858097076416,
"step": 3700,
"unclipped_grad_norm": 0.7848891615867615
},
{
"bits_per_byte": 1.69280218676741,
"learning_rate": 0.00019215766697597923,
"loss_nats": 1.17336106300354,
"step": 3710,
"unclipped_grad_norm": 0.7887690663337708
},
{
"bits_per_byte": 1.7133850704226707,
"learning_rate": 0.00019161079870594978,
"loss_nats": 1.1876280307769775,
"step": 3720,
"unclipped_grad_norm": 0.8315525650978088
},
{
"bits_per_byte": 1.6847540866376338,
"learning_rate": 0.0001910634757307922,
"loss_nats": 1.1677825450897217,
"step": 3730,
"unclipped_grad_norm": 0.8175208568572998
},
{
"bits_per_byte": 1.6646517225088835,
"learning_rate": 0.0001905157074027459,
"loss_nats": 1.153848648071289,
"step": 3740,
"unclipped_grad_norm": 0.795343816280365
},
{
"bits_per_byte": 1.6976003307438312,
"learning_rate": 0.00018996750308166037,
"loss_nats": 1.1766868829727173,
"step": 3750,
"unclipped_grad_norm": 0.8138338327407837
},
{
"bits_per_byte": 1.7200370153927425,
"learning_rate": 0.00018941887213483482,
"loss_nats": 1.1922388076782227,
"step": 3760,
"unclipped_grad_norm": 0.8444334268569946
},
{
"bits_per_byte": 1.7215936303657018,
"learning_rate": 0.00018886982393685845,
"loss_nats": 1.1933177709579468,
"step": 3770,
"unclipped_grad_norm": 0.7583279609680176
},
{
"bits_per_byte": 1.7717356881727062,
"learning_rate": 0.00018832036786945008,
"loss_nats": 1.2280735969543457,
"step": 3780,
"unclipped_grad_norm": 0.8467698693275452
},
{
"bits_per_byte": 1.6571842358084203,
"learning_rate": 0.0001877705133212979,
"loss_nats": 1.1486725807189941,
"step": 3790,
"unclipped_grad_norm": 0.769040584564209
},
{
"bits_per_byte": 1.6766210270783317,
"learning_rate": 0.00018722026968789906,
"loss_nats": 1.1621451377868652,
"step": 3800,
"unclipped_grad_norm": 0.7948989272117615
},
{
"bits_per_byte": 1.6280214816669156,
"learning_rate": 0.00018666964637139907,
"loss_nats": 1.1284584999084473,
"step": 3810,
"unclipped_grad_norm": 0.8209450840950012
},
{
"bits_per_byte": 1.6241315780697008,
"learning_rate": 0.00018611865278043115,
"loss_nats": 1.1257622241973877,
"step": 3820,
"unclipped_grad_norm": 0.81658935546875
},
{
"bits_per_byte": 1.6981439679032087,
"learning_rate": 0.00018556729832995555,
"loss_nats": 1.1770637035369873,
"step": 3830,
"unclipped_grad_norm": 0.8478888273239136
},
{
"bits_per_byte": 1.649875145129402,
"learning_rate": 0.00018501559244109864,
"loss_nats": 1.1436063051223755,
"step": 3840,
"unclipped_grad_norm": 0.8229350447654724
},
{
"bits_per_byte": 1.621423539249215,
"learning_rate": 0.00018446354454099178,
"loss_nats": 1.123885154724121,
"step": 3850,
"unclipped_grad_norm": 0.8153557181358337
},
{
"bits_per_byte": 1.6024271955481633,
"learning_rate": 0.00018391116406261048,
"loss_nats": 1.1107178926467896,
"step": 3860,
"unclipped_grad_norm": 0.794911801815033
},
{
"bits_per_byte": 1.5664059452729007,
"learning_rate": 0.00018335846044461305,
"loss_nats": 1.085749864578247,
"step": 3870,
"unclipped_grad_norm": 0.75998854637146
},
{
"bits_per_byte": 1.6739675067581694,
"learning_rate": 0.0001828054431311793,
"loss_nats": 1.1603058576583862,
"step": 3880,
"unclipped_grad_norm": 0.8571500182151794
},
{
"bits_per_byte": 1.6411153807903556,
"learning_rate": 0.00018225212157184934,
"loss_nats": 1.137534499168396,
"step": 3890,
"unclipped_grad_norm": 0.8254691958427429
},
{
"bits_per_byte": 1.6427378651185953,
"learning_rate": 0.00018169850522136192,
"loss_nats": 1.138659119606018,
"step": 3900,
"unclipped_grad_norm": 0.8312620520591736
},
{
"bits_per_byte": 1.6573687731927962,
"learning_rate": 0.000181144603539493,
"loss_nats": 1.1488004922866821,
"step": 3910,
"unclipped_grad_norm": 0.8051927089691162
},
{
"bits_per_byte": 1.6875147521492837,
"learning_rate": 0.00018059042599089412,
"loss_nats": 1.1696960926055908,
"step": 3920,
"unclipped_grad_norm": 0.8500935435295105
},
{
"bits_per_byte": 1.685968628118027,
"learning_rate": 0.00018003598204493048,
"loss_nats": 1.1686244010925293,
"step": 3930,
"unclipped_grad_norm": 0.8186569213867188
},
{
"bits_per_byte": 1.6142946863883005,
"learning_rate": 0.00017948128117551936,
"loss_nats": 1.1189438104629517,
"step": 3940,
"unclipped_grad_norm": 0.7923241853713989
},
{
"bits_per_byte": 1.7195771337843353,
"learning_rate": 0.00017892633286096816,
"loss_nats": 1.1919200420379639,
"step": 3950,
"unclipped_grad_norm": 0.8632136583328247
},
{
"bits_per_byte": 1.7081789835984027,
"learning_rate": 0.00017837114658381247,
"loss_nats": 1.1840194463729858,
"step": 3960,
"unclipped_grad_norm": 0.8128528594970703
},
{
"bits_per_byte": 1.6418629893736563,
"learning_rate": 0.00017781573183065386,
"loss_nats": 1.1380527019500732,
"step": 3970,
"unclipped_grad_norm": 0.8211596608161926
},
{
"bits_per_byte": 1.6416170541829225,
"learning_rate": 0.0001772600980919981,
"loss_nats": 1.1378822326660156,
"step": 3980,
"unclipped_grad_norm": 0.8518010973930359
},
{
"bits_per_byte": 1.672920992327669,
"learning_rate": 0.0001767042548620927,
"loss_nats": 1.1595804691314697,
"step": 3990,
"unclipped_grad_norm": 0.8055341243743896
},
{
"bits_per_byte": 1.6017095119461129,
"learning_rate": 0.00017614821163876486,
"loss_nats": 1.1102204322814941,
"step": 4000,
"unclipped_grad_norm": 0.7798590064048767
},
{
"bits_per_byte": 1.6685314791296977,
"learning_rate": 0.00017559197792325906,
"loss_nats": 1.1565378904342651,
"step": 4010,
"unclipped_grad_norm": 0.8833235502243042
},
{
"bits_per_byte": 1.6832612772281446,
"learning_rate": 0.0001750355632200748,
"loss_nats": 1.166747808456421,
"step": 4020,
"unclipped_grad_norm": 0.8783014416694641
},
{
"bits_per_byte": 1.6355704881438877,
"learning_rate": 0.00017447897703680418,
"loss_nats": 1.1336910724639893,
"step": 4030,
"unclipped_grad_norm": 0.831271231174469
},
{
"bits_per_byte": 1.6221185211413516,
"learning_rate": 0.0001739222288839694,
"loss_nats": 1.1243668794631958,
"step": 4040,
"unclipped_grad_norm": 0.828468382358551
},
{
"bits_per_byte": 1.6444031731269,
"learning_rate": 0.00017336532827486026,
"loss_nats": 1.1398134231567383,
"step": 4050,
"unclipped_grad_norm": 0.8378818035125732
},
{
"bits_per_byte": 1.6213299806871455,
"learning_rate": 0.00017280828472537163,
"loss_nats": 1.1238203048706055,
"step": 4060,
"unclipped_grad_norm": 0.808568000793457
},
{
"bits_per_byte": 1.749336667724334,
"learning_rate": 0.0001722511077538408,
"loss_nats": 1.212547779083252,
"step": 4070,
"unclipped_grad_norm": 0.8772934675216675
},
{
"bits_per_byte": 1.5885019322904526,
"learning_rate": 0.00017169380688088497,
"loss_nats": 1.1010656356811523,
"step": 4080,
"unclipped_grad_norm": 0.8058938980102539
},
{
"bits_per_byte": 1.6013793052564562,
"learning_rate": 0.0001711363916292384,
"loss_nats": 1.1099915504455566,
"step": 4090,
"unclipped_grad_norm": 0.8040578365325928
},
{
"bits_per_byte": 1.737544161319973,
"learning_rate": 0.00017057887152358974,
"loss_nats": 1.204373836517334,
"step": 4100,
"unclipped_grad_norm": 0.8918697834014893
},
{
"bits_per_byte": 1.6258001537483717,
"learning_rate": 0.0001700212560904194,
"loss_nats": 1.1269187927246094,
"step": 4110,
"unclipped_grad_norm": 0.8254836797714233
},
{
"bits_per_byte": 1.6466504704257243,
"learning_rate": 0.00016946355485783656,
"loss_nats": 1.1413711309432983,
"step": 4120,
"unclipped_grad_norm": 0.8797640800476074
},
{
"bits_per_byte": 1.7012891866221878,
"learning_rate": 0.0001689057773554166,
"loss_nats": 1.179243803024292,
"step": 4130,
"unclipped_grad_norm": 0.8626418709754944
},
{
"bits_per_byte": 1.6871897049391529,
"learning_rate": 0.00016834793311403806,
"loss_nats": 1.1694707870483398,
"step": 4140,
"unclipped_grad_norm": 0.8501971364021301
},
{
"bits_per_byte": 1.629858600343432,
"learning_rate": 0.00016779003166571981,
"loss_nats": 1.1297318935394287,
"step": 4150,
"unclipped_grad_norm": 0.8614740371704102
},
{
"bits_per_byte": 1.6422026551091102,
"learning_rate": 0.00016723208254345838,
"loss_nats": 1.138288140296936,
"step": 4160,
"unclipped_grad_norm": 0.8621925115585327
},
{
"bits_per_byte": 1.6599294228814925,
"learning_rate": 0.00016667409528106481,
"loss_nats": 1.1505753993988037,
"step": 4170,
"unclipped_grad_norm": 0.8789071440696716
},
{
"bits_per_byte": 1.6412854716320588,
"learning_rate": 0.00016611607941300184,
"loss_nats": 1.1376523971557617,
"step": 4180,
"unclipped_grad_norm": 0.8332215547561646
},
{
"bits_per_byte": 1.6753517951149641,
"learning_rate": 0.0001655580444742211,
"loss_nats": 1.1612653732299805,
"step": 4190,
"unclipped_grad_norm": 0.8483413457870483
},
{
"bits_per_byte": 1.6524258198243482,
"learning_rate": 0.00016499999999999997,
"loss_nats": 1.1453742980957031,
"step": 4200,
"unclipped_grad_norm": 0.8514559268951416
},
{
"bits_per_byte": 1.6764456047744516,
"learning_rate": 0.0001644419555257789,
"loss_nats": 1.1620235443115234,
"step": 4210,
"unclipped_grad_norm": 0.7994921803474426
},
{
"bits_per_byte": 1.6553684429806108,
"learning_rate": 0.0001638839205869981,
"loss_nats": 1.147413969039917,
"step": 4220,
"unclipped_grad_norm": 0.8137004375457764
},
{
"bits_per_byte": 1.654246428166382,
"learning_rate": 0.0001633259047189352,
"loss_nats": 1.1466362476348877,
"step": 4230,
"unclipped_grad_norm": 0.8624707460403442
},
{
"bits_per_byte": 1.670982919835096,
"learning_rate": 0.0001627679174565416,
"loss_nats": 1.158237099647522,
"step": 4240,
"unclipped_grad_norm": 0.83209228515625
},
{
"bits_per_byte": 1.605450134602379,
"learning_rate": 0.00016220996833428016,
"loss_nats": 1.1128132343292236,
"step": 4250,
"unclipped_grad_norm": 0.8165753483772278
},
{
"bits_per_byte": 1.64798711958823,
"learning_rate": 0.00016165206688596191,
"loss_nats": 1.142297625541687,
"step": 4260,
"unclipped_grad_norm": 0.8390073180198669
},
{
"bits_per_byte": 1.5630770490828,
"learning_rate": 0.00016109422264458335,
"loss_nats": 1.0834424495697021,
"step": 4270,
"unclipped_grad_norm": 0.7599806189537048
},
{
"bits_per_byte": 1.634252757072977,
"learning_rate": 0.00016053644514216336,
"loss_nats": 1.1327776908874512,
"step": 4280,
"unclipped_grad_norm": 0.8349722027778625
},
{
"bits_per_byte": 1.622665597953746,
"learning_rate": 0.00015997874390958058,
"loss_nats": 1.1247460842132568,
"step": 4290,
"unclipped_grad_norm": 0.8544122576713562
},
{
"bits_per_byte": 1.5775992641216556,
"learning_rate": 0.00015942112847641018,
"loss_nats": 1.0935084819793701,
"step": 4300,
"unclipped_grad_norm": 0.8515903353691101
},
{
"bits_per_byte": 1.5682213941354084,
"learning_rate": 0.00015886360837076156,
"loss_nats": 1.0870082378387451,
"step": 4310,
"unclipped_grad_norm": 0.8175749182701111
},
{
"bits_per_byte": 1.709660270170284,
"learning_rate": 0.00015830619311911495,
"loss_nats": 1.1850461959838867,
"step": 4320,
"unclipped_grad_norm": 0.9185000061988831
},
{
"bits_per_byte": 1.6398645509706304,
"learning_rate": 0.00015774889224615915,
"loss_nats": 1.1366674900054932,
"step": 4330,
"unclipped_grad_norm": 0.8850803971290588
},
{
"bits_per_byte": 1.5869262272432476,
"learning_rate": 0.00015719171527462834,
"loss_nats": 1.099973440170288,
"step": 4340,
"unclipped_grad_norm": 0.8476384878158569
},
{
"bits_per_byte": 1.5594358323987327,
"learning_rate": 0.0001566346717251397,
"loss_nats": 1.080918550491333,
"step": 4350,
"unclipped_grad_norm": 0.8173791170120239
},
{
"bits_per_byte": 1.6383724294917448,
"learning_rate": 0.00015607777111603056,
"loss_nats": 1.1356332302093506,
"step": 4360,
"unclipped_grad_norm": 0.8429195880889893
},
{
"bits_per_byte": 1.6201880158854165,
"learning_rate": 0.00015552102296319576,
"loss_nats": 1.1230287551879883,
"step": 4370,
"unclipped_grad_norm": 0.8339970707893372
},
{
"bits_per_byte": 1.5560377991829855,
"learning_rate": 0.00015496443677992516,
"loss_nats": 1.0785632133483887,
"step": 4380,
"unclipped_grad_norm": 0.7866214513778687
},
{
"bits_per_byte": 1.7055524645544258,
"learning_rate": 0.0001544080220767409,
"loss_nats": 1.1821988821029663,
"step": 4390,
"unclipped_grad_norm": 0.8178918361663818
},
{
"bits_per_byte": 1.6491636529027824,
"learning_rate": 0.00015385178836123511,
"loss_nats": 1.143113136291504,
"step": 4400,
"unclipped_grad_norm": 0.8444706797599792
},
{
"bits_per_byte": 1.6592194784987309,
"learning_rate": 0.00015329574513790726,
"loss_nats": 1.150083303451538,
"step": 4410,
"unclipped_grad_norm": 0.8914594054222107
},
{
"bits_per_byte": 1.676016508060549,
"learning_rate": 0.00015273990190800186,
"loss_nats": 1.1617261171340942,
"step": 4420,
"unclipped_grad_norm": 0.894481897354126
},
{
"bits_per_byte": 1.6150391992838857,
"learning_rate": 0.0001521842681693461,
"loss_nats": 1.119459867477417,
"step": 4430,
"unclipped_grad_norm": 0.8697190880775452
},
{
"bits_per_byte": 1.63267068866769,
"learning_rate": 0.00015162885341618752,
"loss_nats": 1.1316810846328735,
"step": 4440,
"unclipped_grad_norm": 0.8085072636604309
},
{
"bits_per_byte": 1.6597337066248108,
"learning_rate": 0.00015107366713903178,
"loss_nats": 1.150439739227295,
"step": 4450,
"unclipped_grad_norm": 0.8258711099624634
},
{
"bits_per_byte": 1.6374223973283786,
"learning_rate": 0.0001505187188244806,
"loss_nats": 1.134974718093872,
"step": 4460,
"unclipped_grad_norm": 0.8935436010360718
},
{
"bits_per_byte": 1.6156965169754833,
"learning_rate": 0.00014996401795506952,
"loss_nats": 1.11991548538208,
"step": 4470,
"unclipped_grad_norm": 0.8515243530273438
},
{
"bits_per_byte": 1.663532975365021,
"learning_rate": 0.00014940957400910585,
"loss_nats": 1.1530731916427612,
"step": 4480,
"unclipped_grad_norm": 0.8462327718734741
},
{
"bits_per_byte": 1.6896239473794652,
"learning_rate": 0.00014885539646050699,
"loss_nats": 1.1711580753326416,
"step": 4490,
"unclipped_grad_norm": 0.8624916672706604
},
{
"bits_per_byte": 1.6220770733224936,
"learning_rate": 0.00014830149477863805,
"loss_nats": 1.124338150024414,
"step": 4500,
"unclipped_grad_norm": 0.830740213394165
},
{
"bits_per_byte": 1.6091536090060587,
"learning_rate": 0.00014774787842815066,
"loss_nats": 1.1153802871704102,
"step": 4510,
"unclipped_grad_norm": 0.8425999879837036
},
{
"bits_per_byte": 1.6507618876772505,
"learning_rate": 0.00014719455686882066,
"loss_nats": 1.1442209482192993,
"step": 4520,
"unclipped_grad_norm": 0.8468135595321655
},
{
"bits_per_byte": 1.6238777316770272,
"learning_rate": 0.00014664153955538695,
"loss_nats": 1.1255862712860107,
"step": 4530,
"unclipped_grad_norm": 0.8532826900482178
},
{
"bits_per_byte": 1.6469891042652731,
"learning_rate": 0.00014608883593738946,
"loss_nats": 1.1416058540344238,
"step": 4540,
"unclipped_grad_norm": 0.8736191987991333
},
{
"bits_per_byte": 1.6559469926181134,
"learning_rate": 0.00014553645545900817,
"loss_nats": 1.1478149890899658,
"step": 4550,
"unclipped_grad_norm": 0.8442186713218689
},
{
"bits_per_byte": 1.6446332859137545,
"learning_rate": 0.0001449844075589013,
"loss_nats": 1.1399729251861572,
"step": 4560,
"unclipped_grad_norm": 0.9025750756263733
},
{
"bits_per_byte": 1.637351540476223,
"learning_rate": 0.0001444327016700444,
"loss_nats": 1.1349256038665771,
"step": 4570,
"unclipped_grad_norm": 0.8963979482650757
},
{
"bits_per_byte": 1.6074537324849307,
"learning_rate": 0.0001438813472195688,
"loss_nats": 1.1142020225524902,
"step": 4580,
"unclipped_grad_norm": 0.8524222373962402
},
{
"bits_per_byte": 1.592079171428399,
"learning_rate": 0.00014333035362860093,
"loss_nats": 1.1035451889038086,
"step": 4590,
"unclipped_grad_norm": 0.8329559564590454
},
{
"bits_per_byte": 1.593515914493707,
"learning_rate": 0.00014277973031210088,
"loss_nats": 1.1045410633087158,
"step": 4600,
"unclipped_grad_norm": 0.856568455696106
},
{
"bits_per_byte": 1.6099590037600493,
"learning_rate": 0.00014222948667870207,
"loss_nats": 1.1159385442733765,
"step": 4610,
"unclipped_grad_norm": 0.8378997445106506
},
{
"bits_per_byte": 1.6641679353120065,
"learning_rate": 0.00014167963213054991,
"loss_nats": 1.1535133123397827,
"step": 4620,
"unclipped_grad_norm": 0.8407614827156067
},
{
"bits_per_byte": 1.64286238055782,
"learning_rate": 0.00014113017606314152,
"loss_nats": 1.1387454271316528,
"step": 4630,
"unclipped_grad_norm": 0.9177886247634888
},
{
"bits_per_byte": 1.6574955244064822,
"learning_rate": 0.00014058112786516518,
"loss_nats": 1.148888349533081,
"step": 4640,
"unclipped_grad_norm": 0.8678322434425354
},
{
"bits_per_byte": 1.6864685816840852,
"learning_rate": 0.0001400324969183396,
"loss_nats": 1.1689709424972534,
"step": 4650,
"unclipped_grad_norm": 0.8395454287528992
},
{
"bits_per_byte": 1.6055636431519484,
"learning_rate": 0.00013948429259725406,
"loss_nats": 1.1128919124603271,
"step": 4660,
"unclipped_grad_norm": 0.8130927681922913
},
{
"bits_per_byte": 1.6111651180905502,
"learning_rate": 0.00013893652426920776,
"loss_nats": 1.116774559020996,
"step": 4670,
"unclipped_grad_norm": 0.8343866467475891
},
{
"bits_per_byte": 1.6120890088909852,
"learning_rate": 0.00013838920129405016,
"loss_nats": 1.117414951324463,
"step": 4680,
"unclipped_grad_norm": 0.9091700911521912
},
{
"bits_per_byte": 1.598018764258597,
"learning_rate": 0.0001378423330240207,
"loss_nats": 1.1076622009277344,
"step": 4690,
"unclipped_grad_norm": 0.8087259531021118
},
{
"bits_per_byte": 1.6257145063882419,
"learning_rate": 0.00013729592880358936,
"loss_nats": 1.126859426498413,
"step": 4700,
"unclipped_grad_norm": 0.8473510146141052
},
{
"bits_per_byte": 1.6037934257266175,
"learning_rate": 0.00013674999796929659,
"loss_nats": 1.111664891242981,
"step": 4710,
"unclipped_grad_norm": 0.8299785256385803
},
{
"bits_per_byte": 1.6360324335441052,
"learning_rate": 0.00013620454984959417,
"loss_nats": 1.1340112686157227,
"step": 4720,
"unclipped_grad_norm": 0.8852627873420715
},
{
"bits_per_byte": 1.6613310814860245,
"learning_rate": 0.00013565959376468542,
"loss_nats": 1.1515469551086426,
"step": 4730,
"unclipped_grad_norm": 0.8318192958831787
},
{
"bits_per_byte": 1.6077376758415052,
"learning_rate": 0.00013511513902636634,
"loss_nats": 1.1143988370895386,
"step": 4740,
"unclipped_grad_norm": 0.9029622673988342
},
{
"bits_per_byte": 1.5803602735986073,
"learning_rate": 0.0001345711949378662,
"loss_nats": 1.0954222679138184,
"step": 4750,
"unclipped_grad_norm": 0.8787683248519897
},
{
"bits_per_byte": 1.5545300272828713,
"learning_rate": 0.00013402777079368853,
"loss_nats": 1.077518105506897,
"step": 4760,
"unclipped_grad_norm": 0.8357442021369934
},
{
"bits_per_byte": 1.5563697256991507,
"learning_rate": 0.0001334848758794528,
"loss_nats": 1.0787932872772217,
"step": 4770,
"unclipped_grad_norm": 0.8196295499801636
},
{
"bits_per_byte": 1.593790398804484,
"learning_rate": 0.00013294251947173504,
"loss_nats": 1.1047313213348389,
"step": 4780,
"unclipped_grad_norm": 0.9009830951690674
},
{
"bits_per_byte": 1.5906680537780697,
"learning_rate": 0.00013240071083790996,
"loss_nats": 1.1025670766830444,
"step": 4790,
"unclipped_grad_norm": 0.8538159728050232
},
{
"bits_per_byte": 1.62577985979557,
"learning_rate": 0.0001318594592359921,
"loss_nats": 1.1269047260284424,
"step": 4800,
"unclipped_grad_norm": 0.9271508455276489
},
{
"bits_per_byte": 1.5952574108163438,
"learning_rate": 0.0001313187739144781,
"loss_nats": 1.105748176574707,
"step": 4810,
"unclipped_grad_norm": 0.8505629301071167
},
{
"bits_per_byte": 1.5898409892100704,
"learning_rate": 0.0001307786641121882,
"loss_nats": 1.1019937992095947,
"step": 4820,
"unclipped_grad_norm": 0.8469997048377991
},
{
"bits_per_byte": 1.5923758415011373,
"learning_rate": 0.00013023913905810876,
"loss_nats": 1.1037508249282837,
"step": 4830,
"unclipped_grad_norm": 0.8425012826919556
},
{
"bits_per_byte": 1.5390662072305417,
"learning_rate": 0.00012970020797123432,
"loss_nats": 1.0667994022369385,
"step": 4840,
"unclipped_grad_norm": 0.8439065217971802
},
{
"bits_per_byte": 1.6086853002477592,
"learning_rate": 0.00012916188006041027,
"loss_nats": 1.1150556802749634,
"step": 4850,
"unclipped_grad_norm": 0.8472049236297607
},
{
"bits_per_byte": 1.5891265732783864,
"learning_rate": 0.00012862416452417516,
"loss_nats": 1.1014986038208008,
"step": 4860,
"unclipped_grad_norm": 0.8380921483039856
},
{
"bits_per_byte": 1.603959560967351,
"learning_rate": 0.00012808707055060405,
"loss_nats": 1.111780047416687,
"step": 4870,
"unclipped_grad_norm": 0.8809474110603333
},
{
"bits_per_byte": 1.5692639529649388,
"learning_rate": 0.000127550607317151,
"loss_nats": 1.087730884552002,
"step": 4880,
"unclipped_grad_norm": 0.7983198761940002
},
{
"bits_per_byte": 1.525049621185223,
"learning_rate": 0.00012701478399049248,
"loss_nats": 1.0570838451385498,
"step": 4890,
"unclipped_grad_norm": 0.874304473400116
},
{
"bits_per_byte": 1.5072996357549742,
"learning_rate": 0.0001264796097263708,
"loss_nats": 1.0447804927825928,
"step": 4900,
"unclipped_grad_norm": 0.8955963253974915
},
{
"bits_per_byte": 1.5786693401753242,
"learning_rate": 0.0001259450936694375,
"loss_nats": 1.094250202178955,
"step": 4910,
"unclipped_grad_norm": 0.8447177410125732
},
{
"bits_per_byte": 1.585648052182035,
"learning_rate": 0.00012541124495309727,
"loss_nats": 1.0990874767303467,
"step": 4920,
"unclipped_grad_norm": 0.8577683568000793
},
{
"bits_per_byte": 1.5737733380705616,
"learning_rate": 0.00012487807269935167,
"loss_nats": 1.0908565521240234,
"step": 4930,
"unclipped_grad_norm": 0.8645251393318176
},
{
"bits_per_byte": 1.6512401714293001,
"learning_rate": 0.00012434558601864349,
"loss_nats": 1.14455246925354,
"step": 4940,
"unclipped_grad_norm": 0.9303934574127197
},
{
"bits_per_byte": 1.5141001737353912,
"learning_rate": 0.0001238137940097009,
"loss_nats": 1.0494942665100098,
"step": 4950,
"unclipped_grad_norm": 0.8813614249229431
},
{
"bits_per_byte": 1.592528046147151,
"learning_rate": 0.0001232827057593821,
"loss_nats": 1.1038563251495361,
"step": 4960,
"unclipped_grad_norm": 0.915719747543335
},
{
"bits_per_byte": 1.6794828183886887,
"learning_rate": 0.00012275233034251992,
"loss_nats": 1.1641287803649902,
"step": 4970,
"unclipped_grad_norm": 0.9402682781219482
},
{
"bits_per_byte": 1.543928500735735,
"learning_rate": 0.0001222226768217669,
"loss_nats": 1.0701696872711182,
"step": 4980,
"unclipped_grad_norm": 0.9095892310142517
},
{
"bits_per_byte": 1.6067152389821258,
"learning_rate": 0.00012169375424744034,
"loss_nats": 1.1136901378631592,
"step": 4990,
"unclipped_grad_norm": 0.934682309627533
},
{
"bits_per_byte": 1.6283919322968743,
"learning_rate": 0.00012116557165736772,
"loss_nats": 1.1287152767181396,
"step": 5000,
"unclipped_grad_norm": 0.8894299268722534
},
{
"bits_per_byte": 1.5848010376265358,
"learning_rate": 0.00012063813807673214,
"loss_nats": 1.098500370979309,
"step": 5010,
"unclipped_grad_norm": 0.8762696385383606
},
{
"bits_per_byte": 1.5972207647585936,
"learning_rate": 0.00012011146251791832,
"loss_nats": 1.1071090698242188,
"step": 5020,
"unclipped_grad_norm": 0.87384033203125
},
{
"bits_per_byte": 1.5759778116893213,
"learning_rate": 0.0001195855539803584,
"loss_nats": 1.0923845767974854,
"step": 5030,
"unclipped_grad_norm": 0.875623881816864
},
{
"bits_per_byte": 1.5723771829108575,
"learning_rate": 0.00011906042145037817,
"loss_nats": 1.0898888111114502,
"step": 5040,
"unclipped_grad_norm": 0.8759424686431885
},
{
"bits_per_byte": 1.6142312247901323,
"learning_rate": 0.0001185360739010438,
"loss_nats": 1.1188998222351074,
"step": 5050,
"unclipped_grad_norm": 0.886820912361145
},
{
"bits_per_byte": 1.5584469321562717,
"learning_rate": 0.00011801252029200808,
"loss_nats": 1.080233097076416,
"step": 5060,
"unclipped_grad_norm": 0.880901038646698
},
{
"bits_per_byte": 1.6299523308881525,
"learning_rate": 0.00011748976956935777,
"loss_nats": 1.1297968626022339,
"step": 5070,
"unclipped_grad_norm": 0.9343949556350708
},
{
"bits_per_byte": 1.651922254622622,
"learning_rate": 0.00011696783066546024,
"loss_nats": 1.1450252532958984,
"step": 5080,
"unclipped_grad_norm": 0.9880313277244568
},
{
"bits_per_byte": 1.6199948793684975,
"learning_rate": 0.00011644671249881147,
"loss_nats": 1.1228948831558228,
"step": 5090,
"unclipped_grad_norm": 1.0230180025100708
},
{
"bits_per_byte": 1.5480533326340287,
"learning_rate": 0.00011592642397388293,
"loss_nats": 1.073028802871704,
"step": 5100,
"unclipped_grad_norm": 0.9155668020248413
},
{
"bits_per_byte": 1.5785750936826513,
"learning_rate": 0.00011540697398097015,
"loss_nats": 1.0941848754882812,
"step": 5110,
"unclipped_grad_norm": 0.8749560713768005
},
{
"bits_per_byte": 1.553970911684916,
"learning_rate": 0.00011488837139604016,
"loss_nats": 1.0771305561065674,
"step": 5120,
"unclipped_grad_norm": 0.9114576578140259
},
{
"bits_per_byte": 1.6072428817549729,
"learning_rate": 0.00011437062508058039,
"loss_nats": 1.114055871963501,
"step": 5130,
"unclipped_grad_norm": 0.89686119556427
},
{
"bits_per_byte": 1.5469013928385495,
"learning_rate": 0.00011385374388144673,
"loss_nats": 1.072230339050293,
"step": 5140,
"unclipped_grad_norm": 0.9089201092720032
},
{
"bits_per_byte": 1.5509646549478342,
"learning_rate": 0.00011333773663071286,
"loss_nats": 1.0750467777252197,
"step": 5150,
"unclipped_grad_norm": 0.9353929162025452
},
{
"bits_per_byte": 1.5786652125917033,
"learning_rate": 0.00011282261214551897,
"loss_nats": 1.0942473411560059,
"step": 5160,
"unclipped_grad_norm": 0.9100342988967896
},
{
"bits_per_byte": 1.5893422395225683,
"learning_rate": 0.00011230837922792112,
"loss_nats": 1.1016480922698975,
"step": 5170,
"unclipped_grad_norm": 0.8825995922088623
},
{
"bits_per_byte": 1.611850984902191,
"learning_rate": 0.0001117950466647412,
"loss_nats": 1.1172499656677246,
"step": 5180,
"unclipped_grad_norm": 0.8858869671821594
},
{
"bits_per_byte": 1.5765690880429875,
"learning_rate": 0.00011128262322741627,
"loss_nats": 1.092794418334961,
"step": 5190,
"unclipped_grad_norm": 0.893822431564331
},
{
"bits_per_byte": 1.5180597303062058,
"learning_rate": 0.00011077111767184913,
"loss_nats": 1.0522388219833374,
"step": 5200,
"unclipped_grad_norm": 0.9365838766098022
},
{
"bits_per_byte": 1.6136103674205176,
"learning_rate": 0.00011026053873825837,
"loss_nats": 1.118469476699829,
"step": 5210,
"unclipped_grad_norm": 0.9187290668487549
},
{
"bits_per_byte": 1.578151672396227,
"learning_rate": 0.0001097508951510293,
"loss_nats": 1.0938913822174072,
"step": 5220,
"unclipped_grad_norm": 0.9324937462806702
},
{
"bits_per_byte": 1.6149896682804372,
"learning_rate": 0.0001092421956185646,
"loss_nats": 1.1194255352020264,
"step": 5230,
"unclipped_grad_norm": 0.9387596249580383
},
{
"bits_per_byte": 1.5729148006774545,
"learning_rate": 0.00010873444883313581,
"loss_nats": 1.090261459350586,
"step": 5240,
"unclipped_grad_norm": 0.8688122630119324
},
{
"bits_per_byte": 1.5356076361216906,
"learning_rate": 0.00010822766347073447,
"loss_nats": 1.0644021034240723,
"step": 5250,
"unclipped_grad_norm": 0.890831708908081
},
{
"bits_per_byte": 1.5141386978491844,
"learning_rate": 0.00010772184819092424,
"loss_nats": 1.0495209693908691,
"step": 5260,
"unclipped_grad_norm": 0.8596166968345642
},
{
"bits_per_byte": 1.6116535488190005,
"learning_rate": 0.0001072170116366926,
"loss_nats": 1.1171131134033203,
"step": 5270,
"unclipped_grad_norm": 0.92216956615448
},
{
"bits_per_byte": 1.5994467362087108,
"learning_rate": 0.00010671316243430345,
"loss_nats": 1.1086519956588745,
"step": 5280,
"unclipped_grad_norm": 0.9085080623626709
},
{
"bits_per_byte": 1.5769201046333985,
"learning_rate": 0.00010621030919314944,
"loss_nats": 1.093037724494934,
"step": 5290,
"unclipped_grad_norm": 0.8931360244750977
},
{
"bits_per_byte": 1.537008606795619,
"learning_rate": 0.000105708460505605,
"loss_nats": 1.065373182296753,
"step": 5300,
"unclipped_grad_norm": 0.8922452330589294
},
{
"bits_per_byte": 1.5906943671236518,
"learning_rate": 0.00010520762494687964,
"loss_nats": 1.1025853157043457,
"step": 5310,
"unclipped_grad_norm": 0.887484073638916
},
{
"bits_per_byte": 1.5623235930893702,
"learning_rate": 0.00010470781107487111,
"loss_nats": 1.0829201936721802,
"step": 5320,
"unclipped_grad_norm": 0.9026566743850708
},
{
"bits_per_byte": 1.6297041599229574,
"learning_rate": 0.00010420902743001955,
"loss_nats": 1.129624843597412,
"step": 5330,
"unclipped_grad_norm": 0.9325461387634277
},
{
"bits_per_byte": 1.5387120949524151,
"learning_rate": 0.00010371128253516117,
"loss_nats": 1.0665539503097534,
"step": 5340,
"unclipped_grad_norm": 0.8923580050468445
},
{
"bits_per_byte": 1.5882112816104945,
"learning_rate": 0.000103214584895383,
"loss_nats": 1.1008641719818115,
"step": 5350,
"unclipped_grad_norm": 0.9464812278747559
},
{
"bits_per_byte": 1.4963152758270173,
"learning_rate": 0.00010271894299787715,
"loss_nats": 1.037166714668274,
"step": 5360,
"unclipped_grad_norm": 0.8723886609077454
},
{
"bits_per_byte": 1.5320489711100373,
"learning_rate": 0.00010222436531179622,
"loss_nats": 1.0619354248046875,
"step": 5370,
"unclipped_grad_norm": 0.9227972626686096
},
{
"bits_per_byte": 1.5933907111238788,
"learning_rate": 0.0001017308602881082,
"loss_nats": 1.1044542789459229,
"step": 5380,
"unclipped_grad_norm": 0.9233055710792542
},
{
"bits_per_byte": 1.5140970780476757,
"learning_rate": 0.00010123843635945242,
"loss_nats": 1.0494921207427979,
"step": 5390,
"unclipped_grad_norm": 0.8707529902458191
},
{
"bits_per_byte": 1.5792117734561455,
"learning_rate": 0.00010074710193999505,
"loss_nats": 1.0946261882781982,
"step": 5400,
"unclipped_grad_norm": 0.9207195043563843
},
{
"bits_per_byte": 1.5488073045754112,
"learning_rate": 0.0001002568654252858,
"loss_nats": 1.0735514163970947,
"step": 5410,
"unclipped_grad_norm": 0.8852476477622986
},
{
"bits_per_byte": 1.5115954184082252,
"learning_rate": 9.976773519211404e-05,
"loss_nats": 1.0477581024169922,
"step": 5420,
"unclipped_grad_norm": 0.9259113073348999
},
{
"bits_per_byte": 1.5742989170515984,
"learning_rate": 9.92797195983659e-05,
"loss_nats": 1.0912208557128906,
"step": 5430,
"unclipped_grad_norm": 0.9123548865318298
},
{
"bits_per_byte": 1.5198012266288425,
"learning_rate": 9.879282698288142e-05,
"loss_nats": 1.0534459352493286,
"step": 5440,
"unclipped_grad_norm": 0.8873674273490906
},
{
"bits_per_byte": 1.5575636292614403,
"learning_rate": 9.830706566531197e-05,
"loss_nats": 1.0796208381652832,
"step": 5450,
"unclipped_grad_norm": 0.851129949092865
},
{
"bits_per_byte": 1.5371788696199733,
"learning_rate": 9.782244394597826e-05,
"loss_nats": 1.0654911994934082,
"step": 5460,
"unclipped_grad_norm": 0.8905500173568726
},
{
"bits_per_byte": 1.5449696837040585,
"learning_rate": 9.733897010572829e-05,
"loss_nats": 1.0708913803100586,
"step": 5470,
"unclipped_grad_norm": 0.8742530345916748
},
{
"bits_per_byte": 1.5184021477640735,
"learning_rate": 9.685665240579612e-05,
"loss_nats": 1.052476167678833,
"step": 5480,
"unclipped_grad_norm": 0.9149489998817444
},
{
"bits_per_byte": 1.6054506505503316,
"learning_rate": 9.637549908766041e-05,
"loss_nats": 1.1128135919570923,
"step": 5490,
"unclipped_grad_norm": 1.0173310041427612
},
{
"bits_per_byte": 1.5652256283400292,
"learning_rate": 9.589551837290389e-05,
"loss_nats": 1.08493173122406,
"step": 5500,
"unclipped_grad_norm": 0.9233388304710388
},
{
"bits_per_byte": 1.5242242764437322,
"learning_rate": 9.541671846307257e-05,
"loss_nats": 1.0565117597579956,
"step": 5510,
"unclipped_grad_norm": 0.8623188734054565
},
{
"bits_per_byte": 1.4956853033769069,
"learning_rate": 9.493910753953597e-05,
"loss_nats": 1.0367300510406494,
"step": 5520,
"unclipped_grad_norm": 0.8464505672454834
},
{
"bits_per_byte": 1.501370017918526,
"learning_rate": 9.446269376334689e-05,
"loss_nats": 1.040670394897461,
"step": 5530,
"unclipped_grad_norm": 0.9552764296531677
},
{
"bits_per_byte": 1.6303358521995766,
"learning_rate": 9.39874852751024e-05,
"loss_nats": 1.1300626993179321,
"step": 5540,
"unclipped_grad_norm": 0.930858314037323
},
{
"bits_per_byte": 1.5750513411491232,
"learning_rate": 9.351349019480433e-05,
"loss_nats": 1.0917423963546753,
"step": 5550,
"unclipped_grad_norm": 0.9512235522270203
},
{
"bits_per_byte": 1.6036597952068972,
"learning_rate": 9.304071662172086e-05,
"loss_nats": 1.111572265625,
"step": 5560,
"unclipped_grad_norm": 0.9797965884208679
},
{
"bits_per_byte": 1.5364850916063928,
"learning_rate": 9.256917263424787e-05,
"loss_nats": 1.0650103092193604,
"step": 5570,
"unclipped_grad_norm": 0.8889045715332031
},
{
"bits_per_byte": 1.5818145588943031,
"learning_rate": 9.209886628977114e-05,
"loss_nats": 1.0964303016662598,
"step": 5580,
"unclipped_grad_norm": 0.91837477684021
},
{
"bits_per_byte": 1.571553042047923,
"learning_rate": 9.162980562452843e-05,
"loss_nats": 1.0893175601959229,
"step": 5590,
"unclipped_grad_norm": 0.9484079480171204
},
{
"bits_per_byte": 1.5589711352761015,
"learning_rate": 9.116199865347238e-05,
"loss_nats": 1.0805964469909668,
"step": 5600,
"unclipped_grad_norm": 0.9200659394264221
},
{
"bits_per_byte": 1.4898029807694462,
"learning_rate": 9.069545337013333e-05,
"loss_nats": 1.032652735710144,
"step": 5610,
"unclipped_grad_norm": 0.898281455039978
},
{
"bits_per_byte": 1.5617056594248202,
"learning_rate": 9.0230177746483e-05,
"loss_nats": 1.0824918746948242,
"step": 5620,
"unclipped_grad_norm": 0.9515117406845093
},
{
"bits_per_byte": 1.481016903084818,
"learning_rate": 8.976617973279797e-05,
"loss_nats": 1.0265626907348633,
"step": 5630,
"unclipped_grad_norm": 0.8537728786468506
},
{
"bits_per_byte": 1.471526900410208,
"learning_rate": 8.930346725752419e-05,
"loss_nats": 1.0199847221374512,
"step": 5640,
"unclipped_grad_norm": 0.851723313331604
},
{
"bits_per_byte": 1.5525594500692852,
"learning_rate": 8.88420482271411e-05,
"loss_nats": 1.0761522054672241,
"step": 5650,
"unclipped_grad_norm": 0.9591006636619568
},
{
"bits_per_byte": 1.4925528113740916,
"learning_rate": 8.838193052602691e-05,
"loss_nats": 1.0345587730407715,
"step": 5660,
"unclipped_grad_norm": 0.9846746325492859
},
{
"bits_per_byte": 1.5646704683430441,
"learning_rate": 8.792312201632352e-05,
"loss_nats": 1.0845469236373901,
"step": 5670,
"unclipped_grad_norm": 0.9337268471717834
},
{
"bits_per_byte": 1.5783896963850212,
"learning_rate": 8.746563053780261e-05,
"loss_nats": 1.0940563678741455,
"step": 5680,
"unclipped_grad_norm": 1.0177373886108398
},
{
"bits_per_byte": 1.5898387534356093,
"learning_rate": 8.700946390773114e-05,
"loss_nats": 1.1019922494888306,
"step": 5690,
"unclipped_grad_norm": 0.9374083876609802
},
{
"bits_per_byte": 1.5923674143512452,
"learning_rate": 8.655462992073833e-05,
"loss_nats": 1.1037449836730957,
"step": 5700,
"unclipped_grad_norm": 0.9426813721656799
},
{
"bits_per_byte": 1.5582474322812707,
"learning_rate": 8.610113634868199e-05,
"loss_nats": 1.080094814300537,
"step": 5710,
"unclipped_grad_norm": 0.968785285949707
},
{
"bits_per_byte": 1.5819518010496918,
"learning_rate": 8.564899094051613e-05,
"loss_nats": 1.0965254306793213,
"step": 5720,
"unclipped_grad_norm": 0.9166642427444458
},
{
"bits_per_byte": 1.5197234904706525,
"learning_rate": 8.519820142215816e-05,
"loss_nats": 1.0533920526504517,
"step": 5730,
"unclipped_grad_norm": 0.9203759431838989
},
{
"bits_per_byte": 1.6015581672133536,
"learning_rate": 8.474877549635728e-05,
"loss_nats": 1.1101155281066895,
"step": 5740,
"unclipped_grad_norm": 0.934572160243988
},
{
"bits_per_byte": 1.4943809869527633,
"learning_rate": 8.430072084256247e-05,
"loss_nats": 1.0358259677886963,
"step": 5750,
"unclipped_grad_norm": 0.8915490508079529
},
{
"bits_per_byte": 1.50218005620409,
"learning_rate": 8.385404511679161e-05,
"loss_nats": 1.0412318706512451,
"step": 5760,
"unclipped_grad_norm": 0.9438080787658691
},
{
"bits_per_byte": 1.5178872317073904,
"learning_rate": 8.340875595150034e-05,
"loss_nats": 1.052119255065918,
"step": 5770,
"unclipped_grad_norm": 0.9708113670349121
},
{
"bits_per_byte": 1.6077987296825615,
"learning_rate": 8.296486095545203e-05,
"loss_nats": 1.114441156387329,
"step": 5780,
"unclipped_grad_norm": 1.0216169357299805
},
{
"bits_per_byte": 1.454466909375221,
"learning_rate": 8.252236771358729e-05,
"loss_nats": 1.0081596374511719,
"step": 5790,
"unclipped_grad_norm": 0.8829967975616455
},
{
"bits_per_byte": 1.5674770532224742,
"learning_rate": 8.208128378689484e-05,
"loss_nats": 1.0864923000335693,
"step": 5800,
"unclipped_grad_norm": 0.9498817920684814
},
{
"bits_per_byte": 1.4688064788388602,
"learning_rate": 8.16416167122819e-05,
"loss_nats": 1.018099069595337,
"step": 5810,
"unclipped_grad_norm": 0.9071933031082153
},
{
"bits_per_byte": 1.4812057400354652,
"learning_rate": 8.120337400244578e-05,
"loss_nats": 1.02669358253479,
"step": 5820,
"unclipped_grad_norm": 0.9959194660186768
},
{
"bits_per_byte": 1.5459386339590195,
"learning_rate": 8.076656314574521e-05,
"loss_nats": 1.0715630054473877,
"step": 5830,
"unclipped_grad_norm": 0.9805806875228882
},
{
"bits_per_byte": 1.5386066695874363,
"learning_rate": 8.03311916060725e-05,
"loss_nats": 1.0664808750152588,
"step": 5840,
"unclipped_grad_norm": 0.9813235998153687
},
{
"bits_per_byte": 1.4904921152514534,
"learning_rate": 7.989726682272606e-05,
"loss_nats": 1.033130407333374,
"step": 5850,
"unclipped_grad_norm": 0.8884730339050293
},
{
"bits_per_byte": 1.505957139182339,
"learning_rate": 7.946479621028318e-05,
"loss_nats": 1.0438499450683594,
"step": 5860,
"unclipped_grad_norm": 0.9569032192230225
},
{
"bits_per_byte": 1.5707541826346652,
"learning_rate": 7.903378715847343e-05,
"loss_nats": 1.0887638330459595,
"step": 5870,
"unclipped_grad_norm": 0.9548988342285156
},
{
"bits_per_byte": 1.5056532458382643,
"learning_rate": 7.860424703205224e-05,
"loss_nats": 1.0436393022537231,
"step": 5880,
"unclipped_grad_norm": 0.9100088477134705
},
{
"bits_per_byte": 1.5621944341185723,
"learning_rate": 7.817618317067534e-05,
"loss_nats": 1.0828306674957275,
"step": 5890,
"unclipped_grad_norm": 1.0087047815322876
},
{
"bits_per_byte": 1.564086587243365,
"learning_rate": 7.774960288877295e-05,
"loss_nats": 1.0841422080993652,
"step": 5900,
"unclipped_grad_norm": 0.9735300540924072
},
{
"bits_per_byte": 1.5205851235514751,
"learning_rate": 7.732451347542522e-05,
"loss_nats": 1.053989291191101,
"step": 5910,
"unclipped_grad_norm": 0.928986668586731
},
{
"bits_per_byte": 1.4827504882055151,
"learning_rate": 7.690092219423722e-05,
"loss_nats": 1.0277643203735352,
"step": 5920,
"unclipped_grad_norm": 0.9661484956741333
},
{
"bits_per_byte": 1.5936927126587939,
"learning_rate": 7.647883628321536e-05,
"loss_nats": 1.104663610458374,
"step": 5930,
"unclipped_grad_norm": 0.9932063817977905
},
{
"bits_per_byte": 1.5284586612906255,
"learning_rate": 7.605826295464316e-05,
"loss_nats": 1.0594468116760254,
"step": 5940,
"unclipped_grad_norm": 1.0194250345230103
},
{
"bits_per_byte": 1.5388254315193337,
"learning_rate": 7.563920939495851e-05,
"loss_nats": 1.0666325092315674,
"step": 5950,
"unclipped_grad_norm": 0.9446147084236145
},
{
"bits_per_byte": 1.4754302186541906,
"learning_rate": 7.522168276463047e-05,
"loss_nats": 1.0226902961730957,
"step": 5960,
"unclipped_grad_norm": 0.9241304993629456
},
{
"bits_per_byte": 1.5277755462013984,
"learning_rate": 7.480569019803719e-05,
"loss_nats": 1.0589733123779297,
"step": 5970,
"unclipped_grad_norm": 0.9580891728401184
},
{
"bits_per_byte": 1.5012076662961116,
"learning_rate": 7.439123880334388e-05,
"loss_nats": 1.040557861328125,
"step": 5980,
"unclipped_grad_norm": 0.954113245010376
},
{
"bits_per_byte": 1.4896813890352862,
"learning_rate": 7.39783356623813e-05,
"loss_nats": 1.0325684547424316,
"step": 5990,
"unclipped_grad_norm": 0.8945097923278809
},
{
"bits_per_byte": 1.5812009247960246,
"learning_rate": 7.356698783052497e-05,
"loss_nats": 1.0960049629211426,
"step": 6000,
"unclipped_grad_norm": 1.0032626390457153
},
{
"bits_per_byte": 1.556720054358958,
"learning_rate": 7.315720233657431e-05,
"loss_nats": 1.0790361166000366,
"step": 6010,
"unclipped_grad_norm": 0.9715147614479065
},
{
"bits_per_byte": 1.5090167105411882,
"learning_rate": 7.274898618263286e-05,
"loss_nats": 1.0459706783294678,
"step": 6020,
"unclipped_grad_norm": 0.9192096590995789
},
{
"bits_per_byte": 1.4969094758857484,
"learning_rate": 7.234234634398831e-05,
"loss_nats": 1.0375785827636719,
"step": 6030,
"unclipped_grad_norm": 0.9740291833877563
},
{
"bits_per_byte": 1.5263979651679873,
"learning_rate": 7.193728976899362e-05,
"loss_nats": 1.058018445968628,
"step": 6040,
"unclipped_grad_norm": 0.9185873866081238
},
{
"bits_per_byte": 1.5301623214300724,
"learning_rate": 7.153382337894797e-05,
"loss_nats": 1.0606276988983154,
"step": 6050,
"unclipped_grad_norm": 0.9192250370979309
},
{
"bits_per_byte": 1.5237477125181913,
"learning_rate": 7.113195406797887e-05,
"loss_nats": 1.0561814308166504,
"step": 6060,
"unclipped_grad_norm": 0.9670439958572388
},
{
"bits_per_byte": 1.4068365721927425,
"learning_rate": 7.073168870292397e-05,
"loss_nats": 0.9751448035240173,
"step": 6070,
"unclipped_grad_norm": 0.917002260684967
},
{
"bits_per_byte": 1.4967151354902735,
"learning_rate": 7.033303412321404e-05,
"loss_nats": 1.0374438762664795,
"step": 6080,
"unclipped_grad_norm": 0.9365084171295166
},
{
"bits_per_byte": 1.4969400887976019,
"learning_rate": 6.993599714075586e-05,
"loss_nats": 1.037599802017212,
"step": 6090,
"unclipped_grad_norm": 0.9915803074836731
},
{
"bits_per_byte": 1.5907201645212812,
"learning_rate": 6.954058453981609e-05,
"loss_nats": 1.1026031970977783,
"step": 6100,
"unclipped_grad_norm": 1.0093804597854614
},
{
"bits_per_byte": 1.5313995646203795,
"learning_rate": 6.914680307690504e-05,
"loss_nats": 1.0614852905273438,
"step": 6110,
"unclipped_grad_norm": 1.0098049640655518
},
{
"bits_per_byte": 1.4730523865233613,
"learning_rate": 6.875465948066139e-05,
"loss_nats": 1.0210421085357666,
"step": 6120,
"unclipped_grad_norm": 0.9365134239196777
},
{
"bits_per_byte": 1.5512489422697104,
"learning_rate": 6.836416045173733e-05,
"loss_nats": 1.0752438306808472,
"step": 6130,
"unclipped_grad_norm": 0.9237281680107117
},
{
"bits_per_byte": 1.497698360305256,
"learning_rate": 6.797531266268377e-05,
"loss_nats": 1.0381253957748413,
"step": 6140,
"unclipped_grad_norm": 0.9138990044593811
},
{
"bits_per_byte": 1.4838755987074594,
"learning_rate": 6.758812275783663e-05,
"loss_nats": 1.0285441875457764,
"step": 6150,
"unclipped_grad_norm": 0.9429982900619507
},
{
"bits_per_byte": 1.551023816979731,
"learning_rate": 6.720259735320298e-05,
"loss_nats": 1.0750877857208252,
"step": 6160,
"unclipped_grad_norm": 1.0046567916870117
},
{
"bits_per_byte": 1.4928049379402566,
"learning_rate": 6.681874303634842e-05,
"loss_nats": 1.034733533859253,
"step": 6170,
"unclipped_grad_norm": 0.964278519153595
},
{
"bits_per_byte": 1.4789379768011883,
"learning_rate": 6.643656636628402e-05,
"loss_nats": 1.0251216888427734,
"step": 6180,
"unclipped_grad_norm": 0.9606515169143677
},
{
"bits_per_byte": 1.501445346319604,
"learning_rate": 6.605607387335466e-05,
"loss_nats": 1.0407226085662842,
"step": 6190,
"unclipped_grad_norm": 1.0252128839492798
},
{
"bits_per_byte": 1.4915814533620186,
"learning_rate": 6.567727205912724e-05,
"loss_nats": 1.0338854789733887,
"step": 6200,
"unclipped_grad_norm": 0.8806238770484924
},
{
"bits_per_byte": 1.5186611536362729,
"learning_rate": 6.530016739627959e-05,
"loss_nats": 1.0526556968688965,
"step": 6210,
"unclipped_grad_norm": 0.989686906337738
},
{
"bits_per_byte": 1.539449556559315,
"learning_rate": 6.492476632848993e-05,
"loss_nats": 1.0670651197433472,
"step": 6220,
"unclipped_grad_norm": 0.9960981011390686
},
{
"bits_per_byte": 1.4808485320696232,
"learning_rate": 6.455107527032678e-05,
"loss_nats": 1.026445984840393,
"step": 6230,
"unclipped_grad_norm": 0.9705929160118103
},
{
"bits_per_byte": 1.552468127281677,
"learning_rate": 6.417910060713926e-05,
"loss_nats": 1.0760889053344727,
"step": 6240,
"unclipped_grad_norm": 1.0799938440322876
},
{
"bits_per_byte": 1.4757860507588256,
"learning_rate": 6.380884869494799e-05,
"loss_nats": 1.0229369401931763,
"step": 6250,
"unclipped_grad_norm": 0.9792200922966003
},
{
"bits_per_byte": 1.445374530589406,
"learning_rate": 6.34403258603367e-05,
"loss_nats": 1.0018572807312012,
"step": 6260,
"unclipped_grad_norm": 0.9422674179077148
},
{
"bits_per_byte": 1.5013631386124917,
"learning_rate": 6.30735384003437e-05,
"loss_nats": 1.040665626525879,
"step": 6270,
"unclipped_grad_norm": 0.9675420522689819
},
{
"bits_per_byte": 1.5119997496204038,
"learning_rate": 6.270849258235484e-05,
"loss_nats": 1.048038363456726,
"step": 6280,
"unclipped_grad_norm": 0.9240654110908508
},
{
"bits_per_byte": 1.5072342823476461,
"learning_rate": 6.234519464399581e-05,
"loss_nats": 1.0447351932525635,
"step": 6290,
"unclipped_grad_norm": 0.9724128842353821
},
{
"bits_per_byte": 1.4760084243263911,
"learning_rate": 6.198365079302614e-05,
"loss_nats": 1.0230910778045654,
"step": 6300,
"unclipped_grad_norm": 0.9724760055541992
},
{
"bits_per_byte": 1.539534687971492,
"learning_rate": 6.16238672072326e-05,
"loss_nats": 1.0671241283416748,
"step": 6310,
"unclipped_grad_norm": 1.0172570943832397
},
{
"bits_per_byte": 1.498671266161187,
"learning_rate": 6.126585003432406e-05,
"loss_nats": 1.03879976272583,
"step": 6320,
"unclipped_grad_norm": 0.9810369610786438
},
{
"bits_per_byte": 1.5103010769778318,
"learning_rate": 6.090960539182614e-05,
"loss_nats": 1.046860933303833,
"step": 6330,
"unclipped_grad_norm": 0.9735086560249329
},
{
"bits_per_byte": 1.4197871237766881,
"learning_rate": 6.0555139366976935e-05,
"loss_nats": 0.9841214418411255,
"step": 6340,
"unclipped_grad_norm": 0.9049187302589417
},
{
"bits_per_byte": 1.5345103868091858,
"learning_rate": 6.020245801662274e-05,
"loss_nats": 1.0636415481567383,
"step": 6350,
"unclipped_grad_norm": 0.9579975008964539
},
{
"bits_per_byte": 1.5509206273892133,
"learning_rate": 5.985156736711483e-05,
"loss_nats": 1.0750162601470947,
"step": 6360,
"unclipped_grad_norm": 0.989890456199646
},
{
"bits_per_byte": 1.4729708667468522,
"learning_rate": 5.950247341420618e-05,
"loss_nats": 1.0209856033325195,
"step": 6370,
"unclipped_grad_norm": 1.021422266960144
},
{
"bits_per_byte": 1.4625091620948676,
"learning_rate": 5.915518212294935e-05,
"loss_nats": 1.0137341022491455,
"step": 6380,
"unclipped_grad_norm": 1.0111291408538818
},
{
"bits_per_byte": 1.576993369242666,
"learning_rate": 5.880969942759425e-05,
"loss_nats": 1.0930885076522827,
"step": 6390,
"unclipped_grad_norm": 1.0084502696990967
},
{
"bits_per_byte": 1.5426105976821733,
"learning_rate": 5.846603123148688e-05,
"loss_nats": 1.0692561864852905,
"step": 6400,
"unclipped_grad_norm": 1.060890793800354
},
{
"bits_per_byte": 1.542634847235945,
"learning_rate": 5.812418340696858e-05,
"loss_nats": 1.0692729949951172,
"step": 6410,
"unclipped_grad_norm": 1.0786292552947998
},
{
"bits_per_byte": 1.4677484415707522,
"learning_rate": 5.778416179527541e-05,
"loss_nats": 1.0173656940460205,
"step": 6420,
"unclipped_grad_norm": 1.0185354948043823
},
{
"bits_per_byte": 1.5249625979638863,
"learning_rate": 5.7445972206438635e-05,
"loss_nats": 1.057023525238037,
"step": 6430,
"unclipped_grad_norm": 0.9899823069572449
},
{
"bits_per_byte": 1.447091949340922,
"learning_rate": 5.710962041918516e-05,
"loss_nats": 1.0030477046966553,
"step": 6440,
"unclipped_grad_norm": 0.9853183031082153
},
{
"bits_per_byte": 1.4600725118974434,
"learning_rate": 5.677511218083908e-05,
"loss_nats": 1.01204514503479,
"step": 6450,
"unclipped_grad_norm": 0.9329874515533447
},
{
"bits_per_byte": 1.4881399085356029,
"learning_rate": 5.644245320722316e-05,
"loss_nats": 1.031499981880188,
"step": 6460,
"unclipped_grad_norm": 0.9553210735321045
},
{
"bits_per_byte": 1.5093300629310604,
"learning_rate": 5.61116491825615e-05,
"loss_nats": 1.0461878776550293,
"step": 6470,
"unclipped_grad_norm": 1.0880736112594604
},
{
"bits_per_byte": 1.4620568477230984,
"learning_rate": 5.5782705759382104e-05,
"loss_nats": 1.013420581817627,
"step": 6480,
"unclipped_grad_norm": 0.978254497051239
},
{
"bits_per_byte": 1.5301041912940807,
"learning_rate": 5.5455628558420546e-05,
"loss_nats": 1.0605874061584473,
"step": 6490,
"unclipped_grad_norm": 1.0214275121688843
},
{
"bits_per_byte": 1.5703515712489953,
"learning_rate": 5.513042316852368e-05,
"loss_nats": 1.088484764099121,
"step": 6500,
"unclipped_grad_norm": 1.024044156074524
},
{
"bits_per_byte": 1.5418132861127734,
"learning_rate": 5.480709514655443e-05,
"loss_nats": 1.068703532218933,
"step": 6510,
"unclipped_grad_norm": 1.0472792387008667
},
{
"bits_per_byte": 1.5326505664227554,
"learning_rate": 5.448565001729653e-05,
"loss_nats": 1.0623524188995361,
"step": 6520,
"unclipped_grad_norm": 0.9873340129852295
},
{
"bits_per_byte": 1.493115194642413,
"learning_rate": 5.41660932733604e-05,
"loss_nats": 1.0349485874176025,
"step": 6530,
"unclipped_grad_norm": 0.9528921246528625
},
{
"bits_per_byte": 1.4717057623671055,
"learning_rate": 5.384843037508902e-05,
"loss_nats": 1.020108699798584,
"step": 6540,
"unclipped_grad_norm": 0.9596993327140808
},
{
"bits_per_byte": 1.532975785615537,
"learning_rate": 5.353266675046498e-05,
"loss_nats": 1.0625778436660767,
"step": 6550,
"unclipped_grad_norm": 0.9932572245597839
},
{
"bits_per_byte": 1.5382425823155597,
"learning_rate": 5.321880779501729e-05,
"loss_nats": 1.0662285089492798,
"step": 6560,
"unclipped_grad_norm": 1.0132641792297363
},
{
"bits_per_byte": 1.5160394501065202,
"learning_rate": 5.290685887172962e-05,
"loss_nats": 1.0508384704589844,
"step": 6570,
"unclipped_grad_norm": 0.9644629955291748
},
{
"bits_per_byte": 1.503854823258192,
"learning_rate": 5.2596825310948295e-05,
"loss_nats": 1.0423927307128906,
"step": 6580,
"unclipped_grad_norm": 0.9727261662483215
},
{
"bits_per_byte": 1.5231672710715294,
"learning_rate": 5.228871241029154e-05,
"loss_nats": 1.0557790994644165,
"step": 6590,
"unclipped_grad_norm": 1.029261589050293
},
{
"bits_per_byte": 1.415425385776831,
"learning_rate": 5.198252543455865e-05,
"loss_nats": 0.9810981154441833,
"step": 6600,
"unclipped_grad_norm": 0.9430753588676453
},
{
"bits_per_byte": 1.4673490978554489,
"learning_rate": 5.167826961564027e-05,
"loss_nats": 1.0170888900756836,
"step": 6610,
"unclipped_grad_norm": 0.9699459671974182
},
{
"bits_per_byte": 1.5770974187464382,
"learning_rate": 5.137595015242884e-05,
"loss_nats": 1.093160629272461,
"step": 6620,
"unclipped_grad_norm": 1.053364872932434
},
{
"bits_per_byte": 1.520643081704816,
"learning_rate": 5.1075572210729925e-05,
"loss_nats": 1.0540294647216797,
"step": 6630,
"unclipped_grad_norm": 1.0116336345672607
},
{
"bits_per_byte": 1.4843385760035823,
"learning_rate": 5.07771409231737e-05,
"loss_nats": 1.028865098953247,
"step": 6640,
"unclipped_grad_norm": 0.9483272433280945
},
{
"bits_per_byte": 1.481005552229861,
"learning_rate": 5.048066138912756e-05,
"loss_nats": 1.026554822921753,
"step": 6650,
"unclipped_grad_norm": 0.9847526550292969
},
{
"bits_per_byte": 1.5181830418668742,
"learning_rate": 5.018613867460863e-05,
"loss_nats": 1.0523242950439453,
"step": 6660,
"unclipped_grad_norm": 1.0222764015197754
},
{
"bits_per_byte": 1.483456133022005,
"learning_rate": 4.989357781219755e-05,
"loss_nats": 1.028253436088562,
"step": 6670,
"unclipped_grad_norm": 0.9609436392784119
},
{
"bits_per_byte": 1.5070292790278177,
"learning_rate": 4.960298380095222e-05,
"loss_nats": 1.044593095779419,
"step": 6680,
"unclipped_grad_norm": 1.0481340885162354
},
{
"bits_per_byte": 1.4999456576040804,
"learning_rate": 4.931436160632256e-05,
"loss_nats": 1.0396831035614014,
"step": 6690,
"unclipped_grad_norm": 1.0560411214828491
},
{
"bits_per_byte": 1.5028304945896531,
"learning_rate": 4.9027716160065485e-05,
"loss_nats": 1.0416827201843262,
"step": 6700,
"unclipped_grad_norm": 1.0250314474105835
},
{
"bits_per_byte": 1.5313854620430087,
"learning_rate": 4.874305236016084e-05,
"loss_nats": 1.0614755153656006,
"step": 6710,
"unclipped_grad_norm": 0.9873788356781006
},
{
"bits_per_byte": 1.5848861690387128,
"learning_rate": 4.846037507072752e-05,
"loss_nats": 1.0985593795776367,
"step": 6720,
"unclipped_grad_norm": 1.065596103668213
},
{
"bits_per_byte": 1.4535899698384716,
"learning_rate": 4.817968912194049e-05,
"loss_nats": 1.0075517892837524,
"step": 6730,
"unclipped_grad_norm": 0.9767746329307556
},
{
"bits_per_byte": 1.4828168735087481,
"learning_rate": 4.790099930994813e-05,
"loss_nats": 1.0278103351593018,
"step": 6740,
"unclipped_grad_norm": 0.9589064717292786
},
{
"bits_per_byte": 1.4599156637198565,
"learning_rate": 4.762431039679047e-05,
"loss_nats": 1.0119364261627197,
"step": 6750,
"unclipped_grad_norm": 0.9927419424057007
},
{
"bits_per_byte": 1.4484303183299363,
"learning_rate": 4.734962711031755e-05,
"loss_nats": 1.0039753913879395,
"step": 6760,
"unclipped_grad_norm": 0.9825301766395569
},
{
"bits_per_byte": 1.469467236183475,
"learning_rate": 4.707695414410892e-05,
"loss_nats": 1.018557071685791,
"step": 6770,
"unclipped_grad_norm": 1.0425026416778564
},
{
"bits_per_byte": 1.4605650702095145,
"learning_rate": 4.680629615739322e-05,
"loss_nats": 1.0123865604400635,
"step": 6780,
"unclipped_grad_norm": 1.0209197998046875
},
{
"bits_per_byte": 1.494792713418929,
"learning_rate": 4.653765777496863e-05,
"loss_nats": 1.0361113548278809,
"step": 6790,
"unclipped_grad_norm": 0.9975507855415344
},
{
"bits_per_byte": 1.4806505800384802,
"learning_rate": 4.6271043587123986e-05,
"loss_nats": 1.0263087749481201,
"step": 6800,
"unclipped_grad_norm": 1.0428087711334229
},
{
"bits_per_byte": 1.4777557680591573,
"learning_rate": 4.600645814956006e-05,
"loss_nats": 1.0243022441864014,
"step": 6810,
"unclipped_grad_norm": 0.9905012249946594
},
{
"bits_per_byte": 1.4914748241184836,
"learning_rate": 4.574390598331204e-05,
"loss_nats": 1.0338115692138672,
"step": 6820,
"unclipped_grad_norm": 1.0114696025848389
},
{
"bits_per_byte": 1.4751165222990168,
"learning_rate": 4.548339157467199e-05,
"loss_nats": 1.022472858428955,
"step": 6830,
"unclipped_grad_norm": 0.9909161329269409
},
{
"bits_per_byte": 1.462379831141419,
"learning_rate": 4.52249193751124e-05,
"loss_nats": 1.0136444568634033,
"step": 6840,
"unclipped_grad_norm": 1.0066962242126465
},
{
"bits_per_byte": 1.542950607382929,
"learning_rate": 4.4968493801209985e-05,
"loss_nats": 1.0694918632507324,
"step": 6850,
"unclipped_grad_norm": 1.0904512405395508
},
{
"bits_per_byte": 1.5094851912821385,
"learning_rate": 4.471411923457034e-05,
"loss_nats": 1.046295404434204,
"step": 6860,
"unclipped_grad_norm": 1.0577117204666138
},
{
"bits_per_byte": 1.4431473552607326,
"learning_rate": 4.446180002175291e-05,
"loss_nats": 1.0003135204315186,
"step": 6870,
"unclipped_grad_norm": 0.931652843952179
},
{
"bits_per_byte": 1.4721159409894133,
"learning_rate": 4.421154047419693e-05,
"loss_nats": 1.0203930139541626,
"step": 6880,
"unclipped_grad_norm": 1.0439471006393433
},
{
"bits_per_byte": 1.4601951355275085,
"learning_rate": 4.396334486814753e-05,
"loss_nats": 1.0121301412582397,
"step": 6890,
"unclipped_grad_norm": 0.9945755004882812
},
{
"bits_per_byte": 1.5078214311176918,
"learning_rate": 4.371721744458285e-05,
"loss_nats": 1.0451421737670898,
"step": 6900,
"unclipped_grad_norm": 1.0577389001846313
},
{
"bits_per_byte": 1.51248938422741,
"learning_rate": 4.347316240914145e-05,
"loss_nats": 1.0483777523040771,
"step": 6910,
"unclipped_grad_norm": 1.042457938194275
},
{
"bits_per_byte": 1.4546720846777004,
"learning_rate": 4.3231183932050546e-05,
"loss_nats": 1.008301854133606,
"step": 6920,
"unclipped_grad_norm": 1.0035841464996338
},
{
"bits_per_byte": 1.4388924184870613,
"learning_rate": 4.299128614805464e-05,
"loss_nats": 0.9973642230033875,
"step": 6930,
"unclipped_grad_norm": 1.0526496171951294
},
{
"bits_per_byte": 1.5114688391771902,
"learning_rate": 4.2753473156344965e-05,
"loss_nats": 1.0476703643798828,
"step": 6940,
"unclipped_grad_norm": 1.0117003917694092
},
{
"bits_per_byte": 1.4256293744264978,
"learning_rate": 4.251774902048942e-05,
"loss_nats": 0.9881709814071655,
"step": 6950,
"unclipped_grad_norm": 1.020249843597412
},
{
"bits_per_byte": 1.4392803253560824,
"learning_rate": 4.228411776836306e-05,
"loss_nats": 0.9976330995559692,
"step": 6960,
"unclipped_grad_norm": 1.007938265800476
},
{
"bits_per_byte": 1.425250496646647,
"learning_rate": 4.2052583392079427e-05,
"loss_nats": 0.9879083633422852,
"step": 6970,
"unclipped_grad_norm": 1.0789695978164673
},
{
"bits_per_byte": 1.5310960152416067,
"learning_rate": 4.182314984792214e-05,
"loss_nats": 1.0612748861312866,
"step": 6980,
"unclipped_grad_norm": 1.0816211700439453
},
{
"bits_per_byte": 1.4834984407541174,
"learning_rate": 4.159582105627748e-05,
"loss_nats": 1.0282827615737915,
"step": 6990,
"unclipped_grad_norm": 1.0542293787002563
},
{
"bits_per_byte": 1.494992041311279,
"learning_rate": 4.137060090156724e-05,
"loss_nats": 1.0362495183944702,
"step": 7000,
"unclipped_grad_norm": 1.040958046913147
},
{
"bits_per_byte": 1.4039343649594325,
"learning_rate": 4.114749323218249e-05,
"loss_nats": 0.9731331467628479,
"step": 7010,
"unclipped_grad_norm": 1.0020650625228882
},
{
"bits_per_byte": 1.4906596263533938,
"learning_rate": 4.0926501860417706e-05,
"loss_nats": 1.0332465171813965,
"step": 7020,
"unclipped_grad_norm": 1.0739529132843018
},
{
"bits_per_byte": 1.4642052549976767,
"learning_rate": 4.070763056240571e-05,
"loss_nats": 1.0149097442626953,
"step": 7030,
"unclipped_grad_norm": 1.0350828170776367
},
{
"bits_per_byte": 1.4574465088014192,
"learning_rate": 4.049088307805306e-05,
"loss_nats": 1.0102249383926392,
"step": 7040,
"unclipped_grad_norm": 1.0089728832244873
},
{
"bits_per_byte": 1.4534154074478458,
"learning_rate": 4.027626311097629e-05,
"loss_nats": 1.0074307918548584,
"step": 7050,
"unclipped_grad_norm": 1.0608088970184326
},
{
"bits_per_byte": 1.4416395833606184,
"learning_rate": 4.006377432843846e-05,
"loss_nats": 0.9992684125900269,
"step": 7060,
"unclipped_grad_norm": 0.9864199757575989
},
{
"bits_per_byte": 1.4906718371216052,
"learning_rate": 3.985342036128656e-05,
"loss_nats": 1.0332549810409546,
"step": 7070,
"unclipped_grad_norm": 1.0150113105773926
},
{
"bits_per_byte": 1.4931912109740944,
"learning_rate": 3.964520480388956e-05,
"loss_nats": 1.035001277923584,
"step": 7080,
"unclipped_grad_norm": 1.0090219974517822
},
{
"bits_per_byte": 1.4509221749582877,
"learning_rate": 3.943913121407678e-05,
"loss_nats": 1.0057026147842407,
"step": 7090,
"unclipped_grad_norm": 1.0420514345169067
},
{
"bits_per_byte": 1.4389093587781714,
"learning_rate": 3.923520311307735e-05,
"loss_nats": 0.9973759651184082,
"step": 7100,
"unclipped_grad_norm": 1.0253499746322632
},
{
"bits_per_byte": 1.41304419998431,
"learning_rate": 3.9033423985459816e-05,
"loss_nats": 0.979447603225708,
"step": 7110,
"unclipped_grad_norm": 0.9803867936134338
},
{
"bits_per_byte": 1.4722433801337027,
"learning_rate": 3.88337972790728e-05,
"loss_nats": 1.0204813480377197,
"step": 7120,
"unclipped_grad_norm": 1.0725315809249878
},
{
"bits_per_byte": 1.5197811046586915,
"learning_rate": 3.8636326404985845e-05,
"loss_nats": 1.0534319877624512,
"step": 7130,
"unclipped_grad_norm": 1.0701161623001099
},
{
"bits_per_byte": 1.465326925846604,
"learning_rate": 3.844101473743143e-05,
"loss_nats": 1.0156872272491455,
"step": 7140,
"unclipped_grad_norm": 1.0326207876205444
},
{
"bits_per_byte": 1.5100518741167317,
"learning_rate": 3.824786561374701e-05,
"loss_nats": 1.046688199043274,
"step": 7150,
"unclipped_grad_norm": 1.0673091411590576
},
{
"bits_per_byte": 1.5022746466620645,
"learning_rate": 3.805688233431824e-05,
"loss_nats": 1.041297435760498,
"step": 7160,
"unclipped_grad_norm": 1.059691071510315
},
{
"bits_per_byte": 1.514605802728928,
"learning_rate": 3.786806816252236e-05,
"loss_nats": 1.049844741821289,
"step": 7170,
"unclipped_grad_norm": 1.085069179534912
},
{
"bits_per_byte": 1.4489440305080636,
"learning_rate": 3.768142632467269e-05,
"loss_nats": 1.0043314695358276,
"step": 7180,
"unclipped_grad_norm": 0.9880333542823792
},
{
"bits_per_byte": 1.5073161460894569,
"learning_rate": 3.7496960009963236e-05,
"loss_nats": 1.0447919368743896,
"step": 7190,
"unclipped_grad_norm": 1.0678606033325195
},
{
"bits_per_byte": 1.4434393818018978,
"learning_rate": 3.7314672370414325e-05,
"loss_nats": 1.0005159378051758,
"step": 7200,
"unclipped_grad_norm": 1.0179466009140015
},
{
"bits_per_byte": 1.4303239848470999,
"learning_rate": 3.713456652081882e-05,
"loss_nats": 0.9914250373840332,
"step": 7210,
"unclipped_grad_norm": 1.005952000617981
},
{
"bits_per_byte": 1.4493132772594661,
"learning_rate": 3.695664553868873e-05,
"loss_nats": 1.0045874118804932,
"step": 7220,
"unclipped_grad_norm": 1.0151865482330322
},
{
"bits_per_byte": 1.475039818036732,
"learning_rate": 3.678091246420277e-05,
"loss_nats": 1.0224196910858154,
"step": 7230,
"unclipped_grad_norm": 1.0841153860092163
},
{
"bits_per_byte": 1.5119313005253603,
"learning_rate": 3.660737030015427e-05,
"loss_nats": 1.0479909181594849,
"step": 7240,
"unclipped_grad_norm": 1.09213387966156
},
{
"bits_per_byte": 1.494972951237033,
"learning_rate": 3.6436022011900056e-05,
"loss_nats": 1.03623628616333,
"step": 7250,
"unclipped_grad_norm": 1.0454415082931519
},
{
"bits_per_byte": 1.4613747645297765,
"learning_rate": 3.626687052730954e-05,
"loss_nats": 1.0129477977752686,
"step": 7260,
"unclipped_grad_norm": 1.0101300477981567
},
{
"bits_per_byte": 1.4686286487778681,
"learning_rate": 3.6099918736714955e-05,
"loss_nats": 1.0179758071899414,
"step": 7270,
"unclipped_grad_norm": 1.0528877973556519
},
{
"bits_per_byte": 1.4749450555961066,
"learning_rate": 3.5935169492861716e-05,
"loss_nats": 1.022354006767273,
"step": 7280,
"unclipped_grad_norm": 1.0265198945999146
},
{
"bits_per_byte": 1.473522931056122,
"learning_rate": 3.5772625610859886e-05,
"loss_nats": 1.0213682651519775,
"step": 7290,
"unclipped_grad_norm": 1.066664457321167
},
{
"bits_per_byte": 1.4559291058728565,
"learning_rate": 3.5612289868135887e-05,
"loss_nats": 1.0091731548309326,
"step": 7300,
"unclipped_grad_norm": 1.0080223083496094
},
{
"bits_per_byte": 1.445694074354709,
"learning_rate": 3.545416500438522e-05,
"loss_nats": 1.0020787715911865,
"step": 7310,
"unclipped_grad_norm": 1.0805282592773438
},
{
"bits_per_byte": 1.4250067972303744,
"learning_rate": 3.52982537215255e-05,
"loss_nats": 0.9877394437789917,
"step": 7320,
"unclipped_grad_norm": 1.0912657976150513
},
{
"bits_per_byte": 1.4783119599520478,
"learning_rate": 3.5144558683650396e-05,
"loss_nats": 1.0246877670288086,
"step": 7330,
"unclipped_grad_norm": 1.0849530696868896
},
{
"bits_per_byte": 1.4380544330207323,
"learning_rate": 3.499308251698404e-05,
"loss_nats": 0.9967833757400513,
"step": 7340,
"unclipped_grad_norm": 1.042580485343933
},
{
"bits_per_byte": 1.4567071553853599,
"learning_rate": 3.484382780983616e-05,
"loss_nats": 1.0097124576568604,
"step": 7350,
"unclipped_grad_norm": 1.0958631038665771
},
{
"bits_per_byte": 1.4481006275882324,
"learning_rate": 3.469679711255795e-05,
"loss_nats": 1.0037468671798706,
"step": 7360,
"unclipped_grad_norm": 1.0337251424789429
},
{
"bits_per_byte": 1.4027950658887918,
"learning_rate": 3.4551992937498284e-05,
"loss_nats": 0.9723434448242188,
"step": 7370,
"unclipped_grad_norm": 1.0347329378128052
},
{
"bits_per_byte": 1.4160821015291507,
"learning_rate": 3.440941775896104e-05,
"loss_nats": 0.981553316116333,
"step": 7380,
"unclipped_grad_norm": 1.0372161865234375
},
{
"bits_per_byte": 1.421894943145663,
"learning_rate": 3.4269074013162596e-05,
"loss_nats": 0.9855824708938599,
"step": 7390,
"unclipped_grad_norm": 1.038377046585083
},
{
"bits_per_byte": 1.4461825050831596,
"learning_rate": 3.41309640981904e-05,
"loss_nats": 1.0024173259735107,
"step": 7400,
"unclipped_grad_norm": 1.057859182357788
},
{
"bits_per_byte": 1.475963364871865,
"learning_rate": 3.3995090373961795e-05,
"loss_nats": 1.0230598449707031,
"step": 7410,
"unclipped_grad_norm": 1.0708297491073608
},
{
"bits_per_byte": 1.469500256852441,
"learning_rate": 3.3861455162183876e-05,
"loss_nats": 1.0185799598693848,
"step": 7420,
"unclipped_grad_norm": 1.062455654144287
},
{
"bits_per_byte": 1.41574011402791,
"learning_rate": 3.3730060746313664e-05,
"loss_nats": 0.9813162684440613,
"step": 7430,
"unclipped_grad_norm": 1.0484251976013184
},
{
"bits_per_byte": 1.3821044351027654,
"learning_rate": 3.3600909371519225e-05,
"loss_nats": 0.9580017924308777,
"step": 7440,
"unclipped_grad_norm": 1.030645728111267
},
{
"bits_per_byte": 1.4987479704234719,
"learning_rate": 3.3474003244641194e-05,
"loss_nats": 1.0388529300689697,
"step": 7450,
"unclipped_grad_norm": 1.0465614795684814
},
{
"bits_per_byte": 1.4339140366925356,
"learning_rate": 3.334934453415514e-05,
"loss_nats": 0.993913471698761,
"step": 7460,
"unclipped_grad_norm": 1.0371458530426025
},
{
"bits_per_byte": 1.4012647642614147,
"learning_rate": 3.3226935370134486e-05,
"loss_nats": 0.9712827205657959,
"step": 7470,
"unclipped_grad_norm": 1.0733681917190552
},
{
"bits_per_byte": 1.5010874504231586,
"learning_rate": 3.31067778442141e-05,
"loss_nats": 1.040474534034729,
"step": 7480,
"unclipped_grad_norm": 1.0303142070770264
},
{
"bits_per_byte": 1.4650142613873354,
"learning_rate": 3.298887400955457e-05,
"loss_nats": 1.0154705047607422,
"step": 7490,
"unclipped_grad_norm": 1.0710126161575317
},
{
"bits_per_byte": 1.472811954777455,
"learning_rate": 3.287322588080714e-05,
"loss_nats": 1.0208754539489746,
"step": 7500,
"unclipped_grad_norm": 1.0901119709014893
},
{
"bits_per_byte": 1.4645168875610401,
"learning_rate": 3.275983543407924e-05,
"loss_nats": 1.0151257514953613,
"step": 7510,
"unclipped_grad_norm": 1.024397611618042
},
{
"bits_per_byte": 1.4743771688829574,
"learning_rate": 3.2648704606900735e-05,
"loss_nats": 1.0219603776931763,
"step": 7520,
"unclipped_grad_norm": 1.0934079885482788
},
{
"bits_per_byte": 1.46691879726299,
"learning_rate": 3.253983529819086e-05,
"loss_nats": 1.0167906284332275,
"step": 7530,
"unclipped_grad_norm": 1.081713318824768
},
{
"bits_per_byte": 1.4640198577000465,
"learning_rate": 3.243322936822571e-05,
"loss_nats": 1.0147812366485596,
"step": 7540,
"unclipped_grad_norm": 1.0780924558639526
},
{
"bits_per_byte": 1.4520784143200383,
"learning_rate": 3.232888863860648e-05,
"loss_nats": 1.0065040588378906,
"step": 7550,
"unclipped_grad_norm": 1.04312264919281
},
{
"bits_per_byte": 1.3812490793886991,
"learning_rate": 3.222681489222838e-05,
"loss_nats": 0.9574089050292969,
"step": 7560,
"unclipped_grad_norm": 1.0638409852981567
},
{
"bits_per_byte": 1.4099118799641457,
"learning_rate": 3.212700987325009e-05,
"loss_nats": 0.9772764444351196,
"step": 7570,
"unclipped_grad_norm": 1.0123975276947021
},
{
"bits_per_byte": 1.3981073347742243,
"learning_rate": 3.202947528706403e-05,
"loss_nats": 0.9690941572189331,
"step": 7580,
"unclipped_grad_norm": 1.0492113828659058
},
{
"bits_per_byte": 1.472455950690169,
"learning_rate": 3.1934212800267116e-05,
"loss_nats": 1.0206286907196045,
"step": 7590,
"unclipped_grad_norm": 1.0436540842056274
},
{
"bits_per_byte": 1.5055485084038889,
"learning_rate": 3.1841224040632484e-05,
"loss_nats": 1.0435667037963867,
"step": 7600,
"unclipped_grad_norm": 1.140743374824524
},
{
"bits_per_byte": 1.4701950667619266,
"learning_rate": 3.175051059708143e-05,
"loss_nats": 1.01906156539917,
"step": 7610,
"unclipped_grad_norm": 1.082565426826477
},
{
"bits_per_byte": 1.458967179400348,
"learning_rate": 3.1662074019656414e-05,
"loss_nats": 1.0112789869308472,
"step": 7620,
"unclipped_grad_norm": 1.0694279670715332
},
{
"bits_per_byte": 1.4490995028244436,
"learning_rate": 3.157591581949455e-05,
"loss_nats": 1.0044392347335815,
"step": 7630,
"unclipped_grad_norm": 1.0964275598526
},
{
"bits_per_byte": 1.4314776444690875,
"learning_rate": 3.149203746880175e-05,
"loss_nats": 0.9922246932983398,
"step": 7640,
"unclipped_grad_norm": 1.15219247341156
},
{
"bits_per_byte": 1.4066524647649938,
"learning_rate": 3.141044040082763e-05,
"loss_nats": 0.9750171899795532,
"step": 7650,
"unclipped_grad_norm": 1.064915418624878
},
{
"bits_per_byte": 1.416584462852321,
"learning_rate": 3.133112600984091e-05,
"loss_nats": 0.9819015264511108,
"step": 7660,
"unclipped_grad_norm": 1.141569972038269
},
{
"bits_per_byte": 1.4608237321164121,
"learning_rate": 3.125409565110571e-05,
"loss_nats": 1.0125658512115479,
"step": 7670,
"unclipped_grad_norm": 1.0876517295837402
},
{
"bits_per_byte": 1.4539619683122877,
"learning_rate": 3.117935064085835e-05,
"loss_nats": 1.0078096389770508,
"step": 7680,
"unclipped_grad_norm": 1.0938481092453003
},
{
"bits_per_byte": 1.433180616677931,
"learning_rate": 3.110689225628479e-05,
"loss_nats": 0.9934051036834717,
"step": 7690,
"unclipped_grad_norm": 1.070209264755249
},
{
"bits_per_byte": 1.4952543148538446,
"learning_rate": 3.1036721735498926e-05,
"loss_nats": 1.0364313125610352,
"step": 7700,
"unclipped_grad_norm": 1.0864635705947876
},
{
"bits_per_byte": 1.412102165014209,
"learning_rate": 3.096884027752132e-05,
"loss_nats": 0.9787946343421936,
"step": 7710,
"unclipped_grad_norm": 1.040504813194275
},
{
"bits_per_byte": 1.4180753804526507,
"learning_rate": 3.090324904225885e-05,
"loss_nats": 0.9829349517822266,
"step": 7720,
"unclipped_grad_norm": 1.0190634727478027
},
{
"bits_per_byte": 1.4255394275000965,
"learning_rate": 3.083994915048472e-05,
"loss_nats": 0.9881086349487305,
"step": 7730,
"unclipped_grad_norm": 1.0293750762939453
},
{
"bits_per_byte": 1.468768986620972,
"learning_rate": 3.077894168381942e-05,
"loss_nats": 1.0180730819702148,
"step": 7740,
"unclipped_grad_norm": 1.0531129837036133
},
{
"bits_per_byte": 1.4037740771288283,
"learning_rate": 3.0720227684712254e-05,
"loss_nats": 0.9730220437049866,
"step": 7750,
"unclipped_grad_norm": 1.1033527851104736
},
{
"bits_per_byte": 1.4111542826339787,
"learning_rate": 3.0663808156423456e-05,
"loss_nats": 0.9781376123428345,
"step": 7760,
"unclipped_grad_norm": 1.0707266330718994
},
{
"bits_per_byte": 1.4583370349675868,
"learning_rate": 3.060968406300711e-05,
"loss_nats": 1.010842204093933,
"step": 7770,
"unclipped_grad_norm": 1.0769577026367188
},
{
"bits_per_byte": 1.4516912813729461,
"learning_rate": 3.055785632929463e-05,
"loss_nats": 1.0062357187271118,
"step": 7780,
"unclipped_grad_norm": 1.1653907299041748
},
{
"bits_per_byte": 1.4586383485718983,
"learning_rate": 3.0508325840878984e-05,
"loss_nats": 1.011051058769226,
"step": 7790,
"unclipped_grad_norm": 1.0739812850952148
},
{
"bits_per_byte": 1.4426457678594913,
"learning_rate": 3.046109344409957e-05,
"loss_nats": 0.9999658465385437,
"step": 7800,
"unclipped_grad_norm": 1.0972295999526978
},
{
"bits_per_byte": 1.4198621082124645,
"learning_rate": 3.0416159946027727e-05,
"loss_nats": 0.9841734170913696,
"step": 7810,
"unclipped_grad_norm": 1.075659990310669
},
{
"bits_per_byte": 1.4512271001982673,
"learning_rate": 3.0373526114452946e-05,
"loss_nats": 1.0059139728546143,
"step": 7820,
"unclipped_grad_norm": 1.1029958724975586
},
{
"bits_per_byte": 1.4583721194283628,
"learning_rate": 3.0333192677869803e-05,
"loss_nats": 1.0108665227890015,
"step": 7830,
"unclipped_grad_norm": 1.081362009048462
},
{
"bits_per_byte": 1.4242661539444337,
"learning_rate": 3.029516032546544e-05,
"loss_nats": 0.9872260689735413,
"step": 7840,
"unclipped_grad_norm": 1.0629743337631226
},
{
"bits_per_byte": 1.423362471105475,
"learning_rate": 3.02594297071078e-05,
"loss_nats": 0.9865996837615967,
"step": 7850,
"unclipped_grad_norm": 1.0624163150787354
},
{
"bits_per_byte": 1.4807702799634808,
"learning_rate": 3.0226001433334603e-05,
"loss_nats": 1.0263917446136475,
"step": 7860,
"unclipped_grad_norm": 1.0853880643844604
},
{
"bits_per_byte": 1.4488509878939468,
"learning_rate": 3.019487607534278e-05,
"loss_nats": 1.0042669773101807,
"step": 7870,
"unclipped_grad_norm": 1.1049087047576904
},
{
"bits_per_byte": 1.4220910033676464,
"learning_rate": 3.016605416497886e-05,
"loss_nats": 0.9857183694839478,
"step": 7880,
"unclipped_grad_norm": 1.0985115766525269
},
{
"bits_per_byte": 1.4594506226319235,
"learning_rate": 3.0139536194729734e-05,
"loss_nats": 1.0116140842437744,
"step": 7890,
"unclipped_grad_norm": 1.0736068487167358
},
{
"bits_per_byte": 1.4105420243969071,
"learning_rate": 3.0115322617714362e-05,
"loss_nats": 0.9777132272720337,
"step": 7900,
"unclipped_grad_norm": 1.1535842418670654
},
{
"bits_per_byte": 1.463574250651661,
"learning_rate": 3.0093413847675965e-05,
"loss_nats": 1.0144723653793335,
"step": 7910,
"unclipped_grad_norm": 1.1884382963180542
},
{
"bits_per_byte": 1.4830432026772835,
"learning_rate": 3.0073810258974985e-05,
"loss_nats": 1.0279672145843506,
"step": 7920,
"unclipped_grad_norm": 1.1013721227645874
},
{
"bits_per_byte": 1.435883066062264,
"learning_rate": 3.0056512186582658e-05,
"loss_nats": 0.9952782988548279,
"step": 7930,
"unclipped_grad_norm": 1.0824542045593262
},
{
"bits_per_byte": 1.4594231054077853,
"learning_rate": 3.0041519926075316e-05,
"loss_nats": 1.0115950107574463,
"step": 7940,
"unclipped_grad_norm": 1.1213319301605225
},
{
"bits_per_byte": 1.408353201199376,
"learning_rate": 3.0028833733629313e-05,
"loss_nats": 0.9761960506439209,
"step": 7950,
"unclipped_grad_norm": 1.0871061086654663
},
{
"bits_per_byte": 1.4735662706841393,
"learning_rate": 3.0018453826016686e-05,
"loss_nats": 1.0213983058929443,
"step": 7960,
"unclipped_grad_norm": 1.1091856956481934
},
{
"bits_per_byte": 1.4267374586460069,
"learning_rate": 3.001038038060143e-05,
"loss_nats": 0.9889390468597412,
"step": 7970,
"unclipped_grad_norm": 1.0478838682174683
},
{
"bits_per_byte": 1.472857186214632,
"learning_rate": 3.0004613535336433e-05,
"loss_nats": 1.0209068059921265,
"step": 7980,
"unclipped_grad_norm": 1.2172881364822388
},
{
"bits_per_byte": 1.4136101948882995,
"learning_rate": 3.000115338876117e-05,
"loss_nats": 0.9798399209976196,
"step": 7990,
"unclipped_grad_norm": 1.1755717992782593
},
{
"bits_per_byte": 1.46345644253582,
"learning_rate": 3e-05,
"loss_nats": 1.0143907070159912,
"step": 8000,
"unclipped_grad_norm": 1.0856302976608276
}
]
}
Reference in New Issue View Git Blame Copy Permalink