wuyang/llm-atlas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
204f5993049e320580841e1fefeaeebd29fd65a6
llm-atlas/experiments/k3/attnres_gradient/results/raw/formal-depth-32-baseline-seed-2026073001.json
T
wuyang 26fc824409 research: audit AttnRes gradient scale study
2026-07-30 09:49:53 +08:00

8615 lines
250 KiB
JSON
Raw Blame History

{
"architecture": "baseline",
"batch_size": 32,
"canonical_sha256_without_self": "6574e479d5e2794b43a6244a723ba3a2b034f8ef3bfeae24909948fb4a990d99",
"depth": 32,
"diagnostics": [
{
"activation_grad_rms_by_block": [
0.0002768727717921138,
0.0002554995589889586,
0.00023783760843798518,
0.00022115583124104887,
0.00020655356638599187,
0.00019376208365429193,
0.00018488570640329272,
0.00017590694187674671,
0.0001684112212387845,
0.0001647331955609843,
0.00015843863366171718,
0.0001538397918920964,
0.00015132295084185898,
0.00014730033581145108,
0.00014253577683120966,
0.0001394751889165491,
0.0001364792260574177,
0.0001337067224085331,
0.00013106675760354847,
0.00012915431580040604,
0.00012709009752143174,
0.00012574320135172457,
0.0001242366706719622,
0.0001228937617270276,
0.00012132258416386321,
0.00011988351616309956,
0.00011879197700181976,
0.00011764531518565491,
0.00011639683361863717,
0.00011549647024367005,
0.00011481995898066089,
0.00011415599146857858
],
"activation_grad_statistics": {
"first_quartile_mean": 0.0002190592585975537,
"first_to_last_ratio": 1.8672887091156776,
"imbalance_abs_log_ratio": 0.6244874905881238,
"last_quartile_mean": 0.00011731408085324802,
"mean": 0.0001546067051094724,
"normalized": [
1.7908199492128654,
1.6525774791465027,
1.538339545297078,
1.4304413969915148,
1.3359935860462033,
1.2532579587482624,
1.1958453307208163,
1.1377704592578457,
1.0892879524179597,
1.065498391187767,
1.0247850088360109,
0.9950395863049215,
0.978760596021445,
0.9527422223192203,
0.9219249367631522,
0.9021289782858439,
0.8827510162691898,
0.8648183939620171,
0.8477430361816715,
0.835373315206192,
0.8220219002238166,
0.8133101424203405,
0.80356586464988,
0.7948798963150414,
0.7847174809007043,
0.775409553409819,
0.7683494510649245,
0.7609328140222236,
0.7528576043077825,
0.7470340316863389,
0.7426583400723811,
0.7383637817502684
],
"population_cv": 0.27888268733798954
},
"activation_output_rms_by_block": [
0.028270306065678596,
0.0284538883715868,
0.02867252007126808,
0.02881420962512493,
0.029061447829008102,
0.02929086796939373,
0.029533835127949715,
0.029811719432473183,
0.030115574598312378,
0.030315518379211426,
0.030812423676252365,
0.03120259940624237,
0.03179442882537842,
0.03198441118001938,
0.03240228444337845,
0.0329107940196991,
0.033366311341524124,
0.033811621367931366,
0.03420914337038994,
0.03470747172832489,
0.03500960022211075,
0.03567802160978317,
0.035978782922029495,
0.036836784332990646,
0.03769209608435631,
0.038124579936265945,
0.038542140275239944,
0.03904740884900093,
0.0400773324072361,
0.04093807935714722,
0.04163828864693642,
0.042313966900110245
],
"activation_output_statistics": {
"first_quartile_mean": 0.028988599311560392,
"first_to_last_ratio": 0.728416493900548,
"imbalance_abs_log_ratio": 0.31688228737002583,
"last_quartile_mean": 0.03979673655703664,
"mean": 0.03379432682413608,
"normalized": [
0.8365399971656723,
0.8419723381282118,
0.8484418174825138,
0.8526345198433033,
0.8599504875549783,
0.8667392051281828,
0.873928789339177,
0.8821515986281313,
0.8911429055838948,
0.8970593951159853,
0.9117632032322649,
0.9233088017589188,
0.9408214873116122,
0.9464432106153376,
0.9588084003566117,
0.9738555879797565,
0.9873346942272493,
1.0005117587897308,
1.0122747391422395,
1.0270206567197142,
1.0359608701276664,
1.055739970659861,
1.0646397281194921,
1.0900286466627183,
1.1153379761196018,
1.1281355043603702,
1.1404914344295491,
1.1554427183059932,
1.1859189447920195,
1.2113891059344621,
1.2321088348236646,
1.2521026715611152
],
"population_cv": 0.12455242949918699
},
"bits_per_byte": 7.95915895791699,
"branch_output_rms_by_sublayer": [
0.0022265268489718437,
0.002717922907322645,
0.0021652730647474527,
0.002733128145337105,
0.0025650423485785723,
0.002695153234526515,
0.002537536434829235,
0.002696225419640541,
0.0027307753916829824,
0.0027451792266219854,
0.0029185698367655277,
0.0026901187375187874,
0.002936159959062934,
0.0027115964330732822,
0.0030124466866254807,
0.0027617362793534994,
0.0031113612931221724,
0.002723218873143196,
0.0034121612552553415,
0.0027921765577048063,
0.0034701526165008545,
0.002744668396189809,
0.0039346772246062756,
0.0027058592531830072,
0.004295050632208586,
0.0026920263189822435,
0.004176756367087364,
0.0027182905469089746,
0.004023758694529533,
0.0027150253299623728,
0.004447397775948048,
0.0027387896552681923,
0.004260339308530092,
0.00275799329392612,
0.005327676888555288,
0.002721488941460848,
0.00474865036085248,
0.0027454239316284657,
0.00498866755515337,
0.002719792304560542,
0.0052231852896511555,
0.0027601351030170918,
0.005184618756175041,
0.002755985129624605,
0.0052438946440815926,
0.0027470553759485483,
0.0059704589657485485,
0.0027777766808867455,
0.0060942270793020725,
0.0026787330862134695,
0.005698176100850105,
0.002644060645252466,
0.006003713235259056,
0.002661966485902667,
0.006405353546142578,
0.002804658142849803,
0.006667613051831722,
0.0026854604948312044,
0.006290186662226915,
0.0027102255262434483,
0.0072579155676066875,
0.0027289679273962975,
0.006407012697309256,
0.0026884053368121386
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.005888019831616423,
0.005779358340731243,
0.0060348395662513914,
0.005586875351280001,
0.005559071516712478,
0.0054082748429522395,
0.005481135467826832,
0.0053805923812947415,
0.004971746645687392,
0.005439263395635174,
0.005139424262259116,
0.005696496994849179,
0.005782818048228185,
0.0054294510739720785,
0.005067071208887471,
0.005218073376917496,
0.005181036474299639,
0.005605812513830226,
0.005170579144406511,
0.004834742615880465,
0.005150211089068095,
0.00476651480071292,
0.004776402486441839,
0.00519066249960807,
0.0053994045182471796,
0.004795508114414784,
0.004895792813412353,
0.00496559115560922,
0.00502742614163479,
0.004657478744424888,
0.005287606581362069,
0.004870555415551657
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.005639770912333169,
"first_to_last_ratio": 1.130799174678946,
"imbalance_abs_log_ratio": 0.12292461698112994,
"last_quartile_mean": 0.0049874204355821175,
"mean": 0.005263682419187692,
"normalized": [
1.1186122875029152,
1.0979686615711766,
1.146505257280075,
1.061400537941684,
1.0561183357962487,
1.0274698228824493,
1.0413119621059315,
1.0222106792919112,
0.9445377303850805,
1.0333570611721248,
0.976393302818647,
1.0822265747043835,
1.0986259404914112,
1.0314929058371969,
0.962647592570647,
0.9913351455049914,
0.9842988352437859,
1.0649982402804865,
0.9823121405573801,
0.9185095586801336,
0.9784425956805525,
0.9055475655859389,
0.9074260386664719,
0.9861275977985596,
1.0257846291342996,
0.9110557462459602,
0.9301079403205877,
0.9433683038148654,
0.9551157804103686,
0.884832779319472,
1.0045451378463044,
0.9253133125579595
],
"population_cv": 0.06750436074287791
},
"depth_weights": [],
"layer_input_rms_by_sublayer": [
0.028027428314089775,
0.028122643008828163,
0.028270306065678596,
0.028331812471151352,
0.0284538883715868,
0.02856677956879139,
0.02867252007126808,
0.02870391495525837,
0.02881420962512493,
0.028933145105838776,
0.029061447829008102,
0.029177924618124962,
0.02929086796939373,
0.029417123645544052,
0.029533835127949715,
0.02969702146947384,
0.029811719432473183,
0.02999095991253853,
0.030115574598312378,
0.0302139800041914,
0.030315518379211426,
0.03069150820374489,
0.030812423676252365,
0.03109365701675415,
0.03120259940624237,
0.03165328875184059,
0.03179442882537842,
0.03191109746694565,
0.03198441118001938,
0.03227011114358902,
0.03240228444337845,
0.03283532336354256,
0.0329107940196991,
0.03322411701083183,
0.033366311341524124,
0.0337243415415287,
0.033811621367931366,
0.03413747623562813,
0.03420914337038994,
0.03456577658653259,
0.03470747172832489,
0.03486170619726181,
0.03500960022211075,
0.03554223105311394,
0.03567802160978317,
0.03587620332837105,
0.035978782922029495,
0.036742135882377625,
0.036836784332990646,
0.03761643171310425,
0.03769209608435631,
0.03803960233926773,
0.038124579936265945,
0.03846931457519531,
0.038542140275239944,
0.03895273432135582,
0.03904740884900093,
0.03994588926434517,
0.0400773324072361,
0.04087882116436958,
0.04093807935714722,
0.04153488203883171,
0.04163828864693642,
0.042190924286842346
],
"loss_nats": 5.516868591308594,
"loss_scale": 1.0,
"output_weights": null,
"step": 0,
"stream_state_rms_by_sublayer": [
0.028122643008828163,
0.028270306065678596,
0.028331812471151352,
0.0284538883715868,
0.02856677956879139,
0.02867252007126808,
0.02870391495525837,
0.02881420962512493,
0.028933145105838776,
0.029061447829008102,
0.029177924618124962,
0.02929086796939373,
0.029417123645544052,
0.029533835127949715,
0.02969702146947384,
0.029811719432473183,
0.02999095991253853,
0.030115574598312378,
0.0302139800041914,
0.030315518379211426,
0.03069150820374489,
0.030812423676252365,
0.03109365701675415,
0.03120259940624237,
0.03165328875184059,
0.03179442882537842,
0.03191109746694565,
0.03198441118001938,
0.03227011114358902,
0.03240228444337845,
0.03283532336354256,
0.0329107940196991,
0.03322411701083183,
0.033366311341524124,
0.0337243415415287,
0.033811621367931366,
0.03413747623562813,
0.03420914337038994,
0.03456577658653259,
0.03470747172832489,
0.03486170619726181,
0.03500960022211075,
0.03554223105311394,
0.03567802160978317,
0.03587620332837105,
0.035978782922029495,
0.036742135882377625,
0.036836784332990646,
0.03761643171310425,
0.03769209608435631,
0.03803960233926773,
0.038124579936265945,
0.03846931457519531,
0.038542140275239944,
0.03895273432135582,
0.03904740884900093,
0.03994588926434517,
0.0400773324072361,
0.04087882116436958,
0.04093807935714722,
0.04153488203883171,
0.04163828864693642,
0.042190924286842346,
0.042313966900110245
]
},
{
"activation_grad_rms_by_block": [
7.172436016844586e-05,
6.834617670392618e-05,
6.473411485785618e-05,
6.156626477604732e-05,
5.8372206694912165e-05,
5.494635479408316e-05,
5.167145354789682e-05,
4.809222446056083e-05,
4.4674157834379e-05,
4.1754625272005796e-05,
3.8718964788131416e-05,
3.608527549658902e-05,
3.353153806529008e-05,
3.1093299185158685e-05,
2.904756365751382e-05,
2.7216165108256973e-05,
2.5600058506825007e-05,
2.4208331524278037e-05,
2.304886766069103e-05,
2.1986410501995124e-05,
2.1054642274975777e-05,
2.025439971475862e-05,
1.956623418664094e-05,
1.9002633052878082e-05,
1.8555338101577945e-05,
1.8124846974387765e-05,
1.7765629308996722e-05,
1.744573273754213e-05,
1.7174377717310563e-05,
1.6944877643254586e-05,
1.6726755347917788e-05,
1.656309177633375e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 5.9931644500466064e-05,
"first_to_last_ratio": 3.4418587232383575,
"imbalance_abs_log_ratio": 1.2360116519842717,
"last_quartile_mean": 1.7412581200915156e-05,
"mean": 3.3612405388794286e-05,
"normalized": [
2.1338657361414946,
2.033361668508004,
1.9258995037420696,
1.831653047852752,
1.7366268798594315,
1.634704632367703,
1.537273305799265,
1.4307879458276982,
1.3290973174229443,
1.242238536309156,
1.1519248426368067,
1.0735701619444094,
0.9975941226886675,
0.9250542716447356,
0.8641917566303572,
0.8097059640167944,
0.761625305023828,
0.7202201462305526,
0.6857250290208349,
0.654115950574753,
0.626394988143127,
0.602587035366146,
0.582113477459246,
0.5653458249439413,
0.5520383884148895,
0.5392308811207612,
0.5285438249212427,
0.5190266074607738,
0.510953545830914,
0.5041257073765888,
0.49763636831222313,
0.4927672264078892
],
"population_cv": 0.513984504821457
},
"activation_output_rms_by_block": [
0.029439672827720642,
0.030752120539546013,
0.03267423436045647,
0.034982286393642426,
0.03797784447669983,
0.04108294099569321,
0.044510453939437866,
0.04866119846701622,
0.05275992304086685,
0.05702463164925575,
0.061875153332948685,
0.06669966131448746,
0.071797676384449,
0.07734881341457367,
0.08311311900615692,
0.08836977183818817,
0.09436330199241638,
0.10090246796607971,
0.10806354880332947,
0.11583591252565384,
0.1238454207777977,
0.13231202960014343,
0.1417466551065445,
0.15185882151126862,
0.16391249001026154,
0.1746179163455963,
0.18763093650341034,
0.20137503743171692,
0.21650369465351105,
0.23270079493522644,
0.2497490495443344,
0.2673082947731018
],
"activation_output_statistics": {
"first_quartile_mean": 0.037510094000026584,
"first_to_last_ratio": 0.17716440452291277,
"imbalance_abs_log_ratio": 1.7306771383999247,
"last_quartile_mean": 0.21172477677464485,
"mean": 0.11005612107692286,
"normalized": [
0.2674969146617844,
0.27942217332966024,
0.29688702491721536,
0.3178586165979067,
0.3450770761778485,
0.37329083192909013,
0.4044341514483109,
0.4421489508339555,
0.4793910827003499,
0.5181413908763769,
0.5622145567869099,
0.6060513550888119,
0.6523733135594211,
0.702812462021184,
0.7551885183020911,
0.8029519028425762,
0.857410756158323,
0.9168274056792781,
0.9818949436515149,
1.0525167650120182,
1.1252933463940358,
1.2022232685055743,
1.2879488548162785,
1.3798307629352853,
1.4893536897933755,
1.586626119809806,
1.7048659780791944,
1.8297486360705681,
1.9672117510137168,
2.114383031658748,
2.269288133185925,
2.428836235162865
],
"population_cv": 0.6268498746310547
},
"bits_per_byte": 5.1517691996807775,
"branch_output_rms_by_sublayer": [
0.0028861628379672766,
0.004992613103240728,
0.002704753540456295,
0.005112498998641968,
0.002895323559641838,
0.005543495528399944,
0.0029729304369539022,
0.005716898012906313,
0.00318311620503664,
0.005760234780609608,
0.003378538880497217,
0.0057550352066755295,
0.0039320578798651695,
0.005740197841078043,
0.0034711055923253298,
0.006492512300610542,
0.0037959336768835783,
0.006124566774815321,
0.004324590787291527,
0.0062123509123921394,
0.004316650331020355,
0.006645897403359413,
0.0046909768134355545,
0.006303280591964722,
0.0053819348104298115,
0.006548910401761532,
0.005383738782256842,
0.006836008746176958,
0.00522694643586874,
0.006793942768126726,
0.005513017997145653,
0.006699956953525543,
0.005531978793442249,
0.0068945568054914474,
0.006940840277820826,
0.0070524453185498714,
0.006634449120610952,
0.007447477430105209,
0.0070588430389761925,
0.007686031982302666,
0.007738462649285793,
0.007927682250738144,
0.008234621025621891,
0.008245103061199188,
0.008838997222483158,
0.008133068680763245,
0.009296931326389313,
0.008390788920223713,
0.013193055056035519,
0.008053022436797619,
0.00934307835996151,
0.008179600350558758,
0.012704792432487011,
0.008694498799741268,
0.012802425771951675,
0.009316418319940567,
0.014256707392632961,
0.008604929782450199,
0.015807906165719032,
0.009046261198818684,
0.0164310485124588,
0.009124966338276863,
0.01581309363245964,
0.009515758603811264
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.0004018231655400003,
0.0003405186139403606,
0.0003221632759320079,
0.0003173401597627506,
0.0002956526561473367,
0.0002913580859417252,
0.00030034368266542127,
0.00026684124508166604,
0.0002497190139849492,
0.00026578754174316435,
0.0002404550275593826,
0.00024493210822064133,
0.0002532169075836586,
0.00023865218150919594,
0.0002121496513808785,
0.0002091708005241373,
0.0001980500112256327,
0.00020258502349527826,
0.00019483821933596074,
0.0001851504169444552,
0.00018439346409065922,
0.00017525536186022104,
0.000171573793780652,
0.0001670148315601783,
0.00018406810665917416,
0.00014791927954418835,
0.00017020274695916728,
0.00015649953794173187,
0.00016066786897129027,
0.00016871534344057424,
0.00016946426752177343,
0.0001617385113992835
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.00031700511062640856,
"first_to_last_ratio": 1.9222979375866573,
"imbalance_abs_log_ratio": 0.653521312887717,
"last_quartile_mean": 0.0001649094578046479,
"mean": 0.00022650815319523427,
"normalized": [
1.773989853661721,
1.5033393241560593,
1.4223032212634112,
1.401009876625613,
1.305262755343313,
1.286302862972861,
1.3259729437047942,
1.1780646361619815,
1.1024725179305521,
1.1734126917456873,
1.061573387833537,
1.0813390368757583,
1.117915200895233,
1.0536140891294732,
0.9366093378458668,
0.9234581518301755,
0.8743615116358631,
0.8943829201621014,
0.8601819254074433,
0.8174117104953322,
0.814069875585118,
0.7737265055936556,
0.7574729269580301,
0.7373457829406393,
0.8126334706394442,
0.6530417446682029,
0.7514199579935843,
0.6909223166321972,
0.7093248817088087,
0.7448532914184036,
0.7481596804849096,
0.7140516097002307
],
"population_cv": 0.27857383691000986
},
"depth_weights": [],
"layer_input_rms_by_sublayer": [
0.0285640936344862,
0.02861291915178299,
0.029439672827720642,
0.02950597181916237,
0.030752120539546013,
0.030888807028532028,
0.03267423436045647,
0.03270668536424637,
0.034982286393642426,
0.03526782616972923,
0.03797784447669983,
0.03822308033704758,
0.04108294099569321,
0.041396427899599075,
0.044510453939437866,
0.04485016688704491,
0.04866119846701622,
0.04912792518734932,
0.05275992304086685,
0.05317053198814392,
0.05702463164925575,
0.05767740681767464,
0.061875153332948685,
0.06268688291311264,
0.06669966131448746,
0.06753430515527725,
0.071797676384449,
0.07279384881258011,
0.07734881341457367,
0.078476682305336,
0.08311311900615692,
0.08398299664258957,
0.08836977183818817,
0.08972389250993729,
0.09436330199241638,
0.09631941467523575,
0.10090246796607971,
0.10291995853185654,
0.10806354880332947,
0.11053454130887985,
0.11583591252565384,
0.11846786737442017,
0.1238454207777977,
0.12666596472263336,
0.13231202960014343,
0.13596534729003906,
0.1417466551065445,
0.1458624303340912,
0.15185882151126862,
0.1581486165523529,
0.16391249001026154,
0.16865970194339752,
0.1746179163455963,
0.18122267723083496,
0.18763093650341034,
0.19437597692012787,
0.20137503743171692,
0.2100502997636795,
0.21650369465351105,
0.22599515318870544,
0.23270079493522644,
0.2427441030740738,
0.2497490495443344,
0.2599930763244629
],
"loss_nats": 3.570934295654297,
"loss_scale": 1.0,
"output_weights": null,
"step": 100,
"stream_state_rms_by_sublayer": [
0.02861291915178299,
0.029439672827720642,
0.02950597181916237,
0.030752120539546013,
0.030888807028532028,
0.03267423436045647,
0.03270668536424637,
0.034982286393642426,
0.03526782616972923,
0.03797784447669983,
0.03822308033704758,
0.04108294099569321,
0.041396427899599075,
0.044510453939437866,
0.04485016688704491,
0.04866119846701622,
0.04912792518734932,
0.05275992304086685,
0.05317053198814392,
0.05702463164925575,
0.05767740681767464,
0.061875153332948685,
0.06268688291311264,
0.06669966131448746,
0.06753430515527725,
0.071797676384449,
0.07279384881258011,
0.07734881341457367,
0.078476682305336,
0.08311311900615692,
0.08398299664258957,
0.08836977183818817,
0.08972389250993729,
0.09436330199241638,
0.09631941467523575,
0.10090246796607971,
0.10291995853185654,
0.10806354880332947,
0.11053454130887985,
0.11583591252565384,
0.11846786737442017,
0.1238454207777977,
0.12666596472263336,
0.13231202960014343,
0.13596534729003906,
0.1417466551065445,
0.1458624303340912,
0.15185882151126862,
0.1581486165523529,
0.16391249001026154,
0.16865970194339752,
0.1746179163455963,
0.18122267723083496,
0.18763093650341034,
0.19437597692012787,
0.20137503743171692,
0.2100502997636795,
0.21650369465351105,
0.22599515318870544,
0.23270079493522644,
0.2427441030740738,
0.2497490495443344,
0.2599930763244629,
0.2673082947731018
]
},
{
"activation_grad_rms_by_block": [
0.00014451092283707112,
0.00014379274216480553,
0.00014628663484472781,
0.0001445338421035558,
0.00013843078340869397,
0.0001292457600357011,
0.00012031046207994223,
0.00011329296830808744,
0.0001077206979971379,
0.0001003394863801077,
9.358100214740261e-05,
8.73711978783831e-05,
8.221827738452703e-05,
7.71504346630536e-05,
7.201205153251067e-05,
6.728569132974371e-05,
6.256035703700036e-05,
5.796365803689696e-05,
5.370515646063723e-05,
4.9705678975442424e-05,
4.572505713440478e-05,
4.197436646791175e-05,
3.853226371575147e-05,
3.558541357051581e-05,
3.2570587791269645e-05,
2.9896507840021513e-05,
2.8078287868993357e-05,
2.6684616386773996e-05,
2.568897070887033e-05,
2.5203555196640082e-05,
2.4927356207626872e-05,
2.5036461011040956e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 0.00013505051447282312,
"first_to_last_ratio": 4.954020049421058,
"imbalance_abs_log_ratio": 1.6001993781766728,
"last_quartile_mean": 2.7260792876404594e-05,
"mean": 7.412253904703903e-05,
"normalized": [
1.9496218652920512,
1.9399327655728709,
1.9735783032458267,
1.9499310730820074,
1.8675936521932173,
1.743676912547211,
1.6231292617160862,
1.5284550389752622,
1.4532785760182485,
1.3536973728926245,
1.2625174926619152,
1.1787399487615544,
1.1092210067487078,
1.040849863684419,
0.9715270477554875,
0.90776290443914,
0.8440126018524382,
0.7819977402570161,
0.7245455586263081,
0.670587915828121,
0.616884657787925,
0.5662834410094119,
0.5198454371793504,
0.4800889719648283,
0.4394154357097774,
0.4033389603808477,
0.3788090401379066,
0.36000677701879086,
0.34657434889767885,
0.34002552422881266,
0.3362992758762309,
0.3377712276579264
],
"population_cv": 0.5690574657123064
},
"activation_output_rms_by_block": [
0.049516160041093826,
0.057937320321798325,
0.06586314737796783,
0.0719481110572815,
0.076947420835495,
0.08064281940460205,
0.08439230173826218,
0.08712083846330643,
0.0897950753569603,
0.09305135160684586,
0.09555979818105698,
0.09745962917804718,
0.10008277744054794,
0.10209457576274872,
0.1030229851603508,
0.10483334213495255,
0.10759729146957397,
0.11058001965284348,
0.11383581161499023,
0.1180567741394043,
0.1237289309501648,
0.13110940158367157,
0.14003770053386688,
0.15068796277046204,
0.16247524321079254,
0.1767880618572235,
0.1922861486673355,
0.20968382060527802,
0.22965215146541595,
0.2505997121334076,
0.27601101994514465,
0.307341605424881
],
"activation_output_statistics": {
"first_quartile_mean": 0.07179601490497589,
"first_to_last_ratio": 0.31823808816289667,
"imbalance_abs_log_ratio": 1.1449554714268964,
"last_quartile_mean": 0.22560472041368484,
"mean": 0.13002310344018042,
"normalized": [
0.3808258588741859,
0.44559250463094413,
0.506549571847971,
0.5533486676879896,
0.5917980635718029,
0.6202191554495797,
0.6490562023624398,
0.6700412169702441,
0.6906086147856962,
0.7156524428726306,
0.7349447571448028,
0.7495562450013764,
0.7697307231756155,
0.785203345109504,
0.7923436868875267,
0.8062670353287107,
0.8275244062227459,
0.8504643923048483,
0.8755044957634206,
0.9079676689436854,
0.9515918915678598,
1.008354654786339,
1.0770216740619014,
1.1589322111496045,
1.2495874880077935,
1.359666529868349,
1.4788613990882042,
1.6126658651994643,
1.7662411170724883,
1.9273475674936549,
2.1227844332459633,
2.363746113522658
],
"population_cv": 0.4886061625493153
},
"bits_per_byte": 3.0864900833624125,
"branch_output_rms_by_sublayer": [
0.01111105177551508,
0.01691170036792755,
0.01229762565344572,
0.013536608777940273,
0.012528475373983383,
0.011229105293750763,
0.017575321719050407,
0.009298570454120636,
0.015520633198320866,
0.008989781141281128,
0.012922349385917187,
0.008074384182691574,
0.013718407601118088,
0.0077475146390497684,
0.012880085967481136,
0.00855444185435772,
0.019964123144745827,
0.007989896461367607,
0.02377571351826191,
0.008533375337719917,
0.02730517089366913,
0.0086820675060153,
0.02310827188193798,
0.008458662778139114,
0.019315505400300026,
0.008968821726739407,
0.031517889350652695,
0.009528025984764099,
0.018633879721164703,
0.008953212760388851,
0.016267014667391777,
0.008700341917574406,
0.016277894377708435,
0.010046470910310745,
0.022248439490795135,
0.010587109252810478,
0.016605904325842857,
0.012151478789746761,
0.01819506660103798,
0.012493276968598366,
0.017910068854689598,
0.013204623945057392,
0.017647475004196167,
0.015671590343117714,
0.02343275025486946,
0.016670797020196915,
0.03074510768055916,
0.019418122246861458,
0.017775433138012886,
0.019835535436868668,
0.017164910212159157,
0.021999742835760117,
0.01125776395201683,
0.023901214823126793,
0.007435314357280731,
0.02528192289173603,
0.006950905080884695,
0.027673469856381416,
0.007996064610779285,
0.026783082634210587,
0.010837847366929054,
0.030296670272946358,
0.021106943488121033,
0.028989506885409355
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.0006755996472372422,
0.0003902215048764158,
0.00033650809592979206,
0.00029675556364760884,
0.00026186200490798707,
0.0002455230107742763,
0.00021966924842671773,
0.00022418062520577193,
0.00022548545903395049,
0.00022091596273961857,
0.00024014950731391624,
0.00021478395214610848,
0.0001956916368008953,
0.00020741213919383669,
0.0001699546796737774,
0.00016808908215382032,
0.00015850476160987511,
0.00015061268581609982,
0.00013300713992511992,
0.0001284985461118983,
0.0001312583398227583,
0.00012527717278626568,
0.0001234119340024874,
0.00012326054887191204,
0.00010914072106764207,
0.00010077507233480893,
9.59134879196041e-05,
8.362219426315277e-05,
8.733675859901561e-05,
7.698995894839292e-05,
8.92787325176121e-05,
9.631544502550806e-05
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.0003312899626257265,
"first_to_last_ratio": 3.584553340265607,
"imbalance_abs_log_ratio": 1.276633875010675,
"last_quartile_mean": 9.242154633446708e-05,
"mean": 0.00019081267561512152,
"normalized": [
3.540643435030279,
2.045050223306504,
1.763552105985586,
1.5552193411206041,
1.3723512028947988,
1.2867227503769625,
1.1512298526215383,
1.17487281430902,
1.1817111117332983,
1.1577635606620644,
1.2585616051960276,
1.1256272753039647,
1.0255693767201026,
1.0869935056735807,
0.8906886249872848,
0.8809115097409158,
0.8306825586869656,
0.7893222257408777,
0.697056102254977,
0.6734277253733716,
0.6878910921385158,
0.6565453389425546,
0.6467701037399387,
0.6459767333305183,
0.5719783589628198,
0.5281361524329609,
0.5026578428839092,
0.43824234419217944,
0.45770941745598764,
0.4034845101364514,
0.4678868017012227,
0.5047643963642175
],
"population_cv": 0.6065148734399883
},
"depth_weights": [],
"layer_input_rms_by_sublayer": [
0.043906208127737045,
0.04462560638785362,
0.049516160041093826,
0.052572574466466904,
0.057937320321798325,
0.060727521777153015,
0.06586314737796783,
0.06803189963102341,
0.0719481110572815,
0.07307156920433044,
0.076947420835495,
0.07774755358695984,
0.08064281940460205,
0.08214583247900009,
0.08439230173826218,
0.08468613773584366,
0.08712083846330643,
0.08735909312963486,
0.0897950753569603,
0.09098032861948013,
0.09305135160684586,
0.09323208779096603,
0.09555979818105698,
0.09555087983608246,
0.09745962917804718,
0.09809984266757965,
0.10008277744054794,
0.09976384043693542,
0.10209457576274872,
0.10044334083795547,
0.1030229851603508,
0.1023804098367691,
0.10483334213495255,
0.10451660305261612,
0.10759729146957397,
0.10710933804512024,
0.11058001965284348,
0.10916344821453094,
0.11383581161499023,
0.11319266259670258,
0.1180567741394043,
0.11788330227136612,
0.1237289309501648,
0.12335778772830963,
0.13110940158367157,
0.13146334886550903,
0.14003770053386688,
0.13964787125587463,
0.15068796277046204,
0.15031130611896515,
0.16247524321079254,
0.16256581246852875,
0.1767880618572235,
0.1762392520904541,
0.1922861486673355,
0.19244804978370667,
0.20968382060527802,
0.21054796874523163,
0.22965215146541595,
0.23175573348999023,
0.2505997121334076,
0.2547222673892975,
0.27601101994514465,
0.28623583912849426
],
"loss_nats": 2.1393918991088867,
"loss_scale": 1.0,
"output_weights": null,
"step": 500,
"stream_state_rms_by_sublayer": [
0.04462560638785362,
0.049516160041093826,
0.052572574466466904,
0.057937320321798325,
0.060727521777153015,
0.06586314737796783,
0.06803189963102341,
0.0719481110572815,
0.07307156920433044,
0.076947420835495,
0.07774755358695984,
0.08064281940460205,
0.08214583247900009,
0.08439230173826218,
0.08468613773584366,
0.08712083846330643,
0.08735909312963486,
0.0897950753569603,
0.09098032861948013,
0.09305135160684586,
0.09323208779096603,
0.09555979818105698,
0.09555087983608246,
0.09745962917804718,
0.09809984266757965,
0.10008277744054794,
0.09976384043693542,
0.10209457576274872,
0.10044334083795547,
0.1030229851603508,
0.1023804098367691,
0.10483334213495255,
0.10451660305261612,
0.10759729146957397,
0.10710933804512024,
0.11058001965284348,
0.10916344821453094,
0.11383581161499023,
0.11319266259670258,
0.1180567741394043,
0.11788330227136612,
0.1237289309501648,
0.12335778772830963,
0.13110940158367157,
0.13146334886550903,
0.14003770053386688,
0.13964787125587463,
0.15068796277046204,
0.15031130611896515,
0.16247524321079254,
0.16256581246852875,
0.1767880618572235,
0.1762392520904541,
0.1922861486673355,
0.19244804978370667,
0.20968382060527802,
0.21054796874523163,
0.22965215146541595,
0.23175573348999023,
0.2505997121334076,
0.2547222673892975,
0.27601101994514465,
0.28623583912849426,
0.307341605424881
]
},
{
"activation_grad_rms_by_block": [
0.00010485891834832728,
0.00010495351307326928,
0.00011622957390500233,
0.00011138651461806148,
0.00010787966311909258,
0.000104747356090229,
0.00010223284334642813,
9.973083797376603e-05,
9.735774074215442e-05,
9.589978435542434e-05,
9.408760524820536e-05,
9.229916759068146e-05,
9.143509669229388e-05,
8.827183046378195e-05,
8.612661622464657e-05,
8.33196536405012e-05,
8.087792957667261e-05,
7.82960414653644e-05,
8.221158350352198e-05,
7.83153300289996e-05,
7.789687515469268e-05,
7.347232894971967e-05,
6.715180643368512e-05,
6.132270209491253e-05,
5.5257889471249655e-05,
5.0392085540806875e-05,
4.541657472145744e-05,
4.046381582156755e-05,
3.4924181818496436e-05,
2.9514063498936594e-05,
2.4251618015114218e-05,
2.1234091036603786e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 0.00010650240255927201,
"first_to_last_ratio": 2.8263626166920495,
"imbalance_abs_log_ratio": 1.038990590500678,
"last_quartile_mean": 3.768178999052907e-05,
"mean": 7.755673851761458e-05,
"normalized": [
1.3520284678359944,
1.353248151989171,
1.4986392688316106,
1.4361938981325721,
1.3909773048874547,
1.3505900079390127,
1.3181684183793927,
1.2859080961883835,
1.255309887999515,
1.236511310150616,
1.213145460298544,
1.1900857276214447,
1.1789445822496427,
1.1381581040019337,
1.1104981703825207,
1.0743057951254327,
1.0428227272386361,
1.0095324140994133,
1.060018575753417,
1.009781116713833,
1.0043856490561534,
0.947336496531926,
0.8658410309303259,
0.7906818062106236,
0.7124834092746121,
0.6497447740064765,
0.585591601574923,
0.5217317875271078,
0.4503049314092096,
0.380548022816011,
0.3126951761851902,
0.2737878286589002
],
"population_cv": 0.34367475536074504
},
"activation_output_rms_by_block": [
0.09794864058494568,
0.13700027763843536,
0.20758503675460815,
0.22207145392894745,
0.22574232518672943,
0.2265917956829071,
0.22746813297271729,
0.22716933488845825,
0.22722382843494415,
0.22613705694675446,
0.22211141884326935,
0.21741008758544922,
0.21311214566230774,
0.20640577375888824,
0.19932369887828827,
0.19335229694843292,
0.19345322251319885,
0.19645750522613525,
0.20112650096416473,
0.2038118690252304,
0.20720544457435608,
0.21031972765922546,
0.2129739224910736,
0.21647654473781586,
0.22099705040454865,
0.23361735045909882,
0.2512694001197815,
0.2773953676223755,
0.3133004307746887,
0.35808902978897095,
0.4303363263607025,
0.54518723487854
],
"activation_output_statistics": {
"first_quartile_mean": 0.1964471247047186,
"first_to_last_ratio": 0.5975141297159507,
"imbalance_abs_log_ratio": 0.5149773473959137,
"last_quartile_mean": 0.32877402380108833,
"mean": 0.2358959447592497,
"normalized": [
0.4152196880065486,
0.5807657176059338,
0.879985609614718,
0.9413958097312304,
0.9569572101678865,
0.9605582491643161,
0.964273180723248,
0.9630065286637394,
0.9632375353753703,
0.958630540162719,
0.9415652272867661,
0.9216355448896473,
0.9034158933075572,
0.8749865283591098,
0.8449644994182233,
0.8196507877478102,
0.8200786270854848,
0.832814253872128,
0.8526068609167068,
0.8639905583507864,
0.8783764586789548,
0.8915783943376951,
0.9028299435517223,
0.9176781099766134,
0.9368412442617167,
0.9903406804959011,
1.065170494457722,
1.1759225785143497,
1.32812978660755,
1.5179956999872501,
1.824263349672647,
2.311134409007948
],
"population_cv": 0.335075401661643
},
"bits_per_byte": 2.08044695422856,
"branch_output_rms_by_sublayer": [
0.018880587071180344,
0.05959044396877289,
0.060257185250520706,
0.016382697969675064,
0.09490419179201126,
0.0035255588591098785,
0.0323294997215271,
0.004731281660497189,
0.02052305079996586,
0.00493080448359251,
0.01820369064807892,
0.005357450805604458,
0.019246362149715424,
0.006479416508227587,
0.024384424090385437,
0.009427462704479694,
0.017510300502181053,
0.009214801713824272,
0.01695968210697174,
0.0109269879758358,
0.019068937748670578,
0.012604814022779465,
0.019973911345005035,
0.013778526335954666,
0.022227400913834572,
0.013526024296879768,
0.027813225984573364,
0.017681727185845375,
0.027307361364364624,
0.018625596538186073,
0.03342962637543678,
0.019818320870399475,
0.030618051066994667,
0.021072659641504288,
0.025097575038671494,
0.022579019889235497,
0.025198329240083694,
0.01933942921459675,
0.03349718078970909,
0.022255338728427887,
0.03462665155529976,
0.024717653170228004,
0.03921883553266525,
0.025107676163315773,
0.043981269001960754,
0.0301958154886961,
0.038896940648555756,
0.03620966896414757,
0.04658674821257591,
0.046204980462789536,
0.033179834485054016,
0.04984369874000549,
0.02075173333287239,
0.05490582063794136,
0.011197025887668133,
0.06379986554384232,
0.012507086619734764,
0.07197272032499313,
0.011961712501943111,
0.07922235131263733,
0.010508585721254349,
0.11548902839422226,
0.021868467330932617,
0.1684120148420334
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.00040070945427075937,
0.00021125547826331077,
0.00020379424936248455,
0.00011927406183950472,
9.446000613801463e-05,
0.00010295610627477164,
9.818733926828671e-05,
0.00012673421109679544,
0.00011573807851122185,
0.00011146965874368272,
0.0001232427596852727,
0.00012517422740451753,
0.000137325623762558,
0.0001763842480315643,
0.00017420922346804142,
0.00019637547238758496,
0.00020814888113781,
0.0002138469295217413,
0.0002110578549773679,
0.0002602958562879386,
0.0002719585176535882,
0.00027346390406242154,
0.00028937535779988523,
0.00028561131716422746,
0.00028061956919165,
0.00023874030969169658,
0.00021716224392712217,
0.0001981302869037355,
0.00018664421118786767,
0.0001673158872138231,
0.0001540361578610053,
0.00014575183058398037
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.00016967136331424098,
"first_to_last_ratio": 0.8545520537501935,
"imbalance_abs_log_ratio": 0.1571778610785998,
"last_quartile_mean": 0.0001985500620701101,
"mean": 0.00019123279105231976,
"normalized": [
2.0954013799920355,
1.104703210682696,
1.0656867383519393,
0.6237113477409442,
0.4939529754192163,
0.5383810261212141,
0.5134440528111281,
0.6627221743687356,
0.605220882225982,
0.582900339059453,
0.6444645764311125,
0.6545646628682571,
0.71810709348998,
0.9223535726323577,
0.9109798717542076,
1.0268922568507521,
1.0884581038241612,
1.1182545019867147,
1.1036697933233854,
1.3611465630741315,
1.4221332866452938,
1.4300052964640568,
1.5132099270606496,
1.493526897727945,
1.4674239059496585,
1.2484276800958218,
1.1355910392360917,
1.0360685832877303,
0.976005266465014,
0.8749330399515364,
0.8054902980465429,
0.7621696560612549
],
"population_cv": 0.3661316865844204
},
"depth_weights": [],
"layer_input_rms_by_sublayer": [
0.0565960556268692,
0.06250853836536407,
0.09794864058494568,
0.12857383489608765,
0.13700027763843536,
0.2065405696630478,
0.20758503675460815,
0.22102168202400208,
0.22207145392894745,
0.2248813658952713,
0.22574232518672943,
0.2264578640460968,
0.2265917956829071,
0.22759772837162018,
0.22746813297271729,
0.22716368734836578,
0.22716933488845825,
0.22757723927497864,
0.22722382843494415,
0.22673146426677704,
0.22613705694675446,
0.2244347184896469,
0.22211141884326935,
0.2183273881673813,
0.21741008758544922,
0.21451692283153534,
0.21311214566230774,
0.20762479305267334,
0.20640577375888824,
0.20071867108345032,
0.19932369887828827,
0.19245882332324982,
0.19335229694843292,
0.19078245759010315,
0.19345322251319885,
0.1916658580303192,
0.19645750522613525,
0.19844752550125122,
0.20112650096416473,
0.2010338455438614,
0.2038118690252304,
0.20468150079250336,
0.20720544457435608,
0.20727655291557312,
0.21031972765922546,
0.20860347151756287,
0.2129739224910736,
0.21107539534568787,
0.21647654473781586,
0.2111099362373352,
0.22099705040454865,
0.2196389138698578,
0.23361735045909882,
0.23286758363246918,
0.2512694001197815,
0.25084227323532104,
0.2773953676223755,
0.27732276916503906,
0.3133004307746887,
0.31342777609825134,
0.35808902978897095,
0.35860511660575867,
0.4303363263607025,
0.4337502121925354
],
"loss_nats": 1.4420559406280518,
"loss_scale": 1.0,
"output_weights": null,
"step": 2000,
"stream_state_rms_by_sublayer": [
0.06250853836536407,
0.09794864058494568,
0.12857383489608765,
0.13700027763843536,
0.2065405696630478,
0.20758503675460815,
0.22102168202400208,
0.22207145392894745,
0.2248813658952713,
0.22574232518672943,
0.2264578640460968,
0.2265917956829071,
0.22759772837162018,
0.22746813297271729,
0.22716368734836578,
0.22716933488845825,
0.22757723927497864,
0.22722382843494415,
0.22673146426677704,
0.22613705694675446,
0.2244347184896469,
0.22211141884326935,
0.2183273881673813,
0.21741008758544922,
0.21451692283153534,
0.21311214566230774,
0.20762479305267334,
0.20640577375888824,
0.20071867108345032,
0.19932369887828827,
0.19245882332324982,
0.19335229694843292,
0.19078245759010315,
0.19345322251319885,
0.1916658580303192,
0.19645750522613525,
0.19844752550125122,
0.20112650096416473,
0.2010338455438614,
0.2038118690252304,
0.20468150079250336,
0.20720544457435608,
0.20727655291557312,
0.21031972765922546,
0.20860347151756287,
0.2129739224910736,
0.21107539534568787,
0.21647654473781586,
0.2111099362373352,
0.22099705040454865,
0.2196389138698578,
0.23361735045909882,
0.23286758363246918,
0.2512694001197815,
0.25084227323532104,
0.2773953676223755,
0.27732276916503906,
0.3133004307746887,
0.31342777609825134,
0.35808902978897095,
0.35860511660575867,
0.4303363263607025,
0.4337502121925354,
0.54518723487854
]
},
{
"activation_grad_rms_by_block": [
0.00010598741209832951,
0.00010775360715342686,
0.00011871542665176094,
0.00011485091090435162,
0.00011244894267292693,
0.00011016445205314085,
0.00010846564691746607,
0.00010656306403689086,
0.0001046601973939687,
0.00010342278255848214,
0.00010179441596847028,
0.00010021102934842929,
9.965883509721607e-05,
9.537018195260316e-05,
9.32025650399737e-05,
9.038933058036491e-05,
8.758632611716166e-05,
8.443772094324231e-05,
8.744140359340236e-05,
8.20954010123387e-05,
8.16298124846071e-05,
7.812595140421763e-05,
7.176945655373856e-05,
6.528423546114936e-05,
5.917211092310026e-05,
5.3287756600184366e-05,
4.791216633748263e-05,
4.251671634847298e-05,
3.64475381502416e-05,
3.0126682759146206e-05,
2.3703445549472235e-05,
1.9818620785372332e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 0.0001106186828110367,
"first_to_last_ratio": 2.827449739094469,
"imbalance_abs_log_ratio": 1.0393751531099071,
"last_quartile_mean": 3.912312968168408e-05,
"mean": 8.203169204534788e-05,
"normalized": [
1.292030061256553,
1.3135607040003474,
1.4471897835062812,
1.4000797501636437,
1.3707987714158585,
1.342949892978447,
1.3222407610159408,
1.2990474946924393,
1.275850769189444,
1.2607661743867122,
1.2409157172108236,
1.2216135843331344,
1.2148821097353903,
1.1626016674126585,
1.1361775274420824,
1.1018830445481589,
1.067713269509827,
1.0293304799389529,
1.06594660445458,
1.000776638460116,
0.9951009231832177,
0.9523874182801066,
0.8748991367149143,
0.7958416294163435,
0.7213323222735595,
0.6495996275528049,
0.5840689755734447,
0.5182962253779837,
0.444310451747028,
0.36725662982170226,
0.28895473150022005,
0.24159712290728336
],
"population_cv": 0.3450366052372569
},
"activation_output_rms_by_block": [
0.11079900711774826,
0.18173734843730927,
0.25566670298576355,
0.2647266387939453,
0.2669948637485504,
0.2675777077674866,
0.2675747275352478,
0.26725223660469055,
0.2656837999820709,
0.265005499124527,
0.25976043939590454,
0.2535160183906555,
0.24569463729858398,
0.23344992101192474,
0.22194796800613403,
0.21299737691879272,
0.21236176788806915,
0.21684354543685913,
0.22108736634254456,
0.22472712397575378,
0.22933396697044373,
0.23535853624343872,
0.2425893247127533,
0.2497948557138443,
0.26040923595428467,
0.27719539403915405,
0.2986113131046295,
0.3278995454311371,
0.36809566617012024,
0.4197922646999359,
0.5092319846153259,
0.6714946031570435
],
"activation_output_statistics": {
"first_quartile_mean": 0.23529115412384272,
"first_to_last_ratio": 0.6008590681870453,
"imbalance_abs_log_ratio": 0.5093948674743478,
"last_quartile_mean": 0.39159125089645386,
"mean": 0.2751628558617085,
"normalized": [
0.40266701975732394,
0.6604719516672225,
0.9291468580855864,
0.962072580490267,
0.9703157895800326,
0.9724339679842758,
0.972423137190165,
0.9712511369594387,
0.9655511066348229,
0.9630860178952118,
0.9440243617999629,
0.9213308155155495,
0.8929062628353637,
0.8484063747661208,
0.806605845513105,
0.7740775049443485,
0.7717675673303741,
0.7880552945919432,
0.803478237097738,
0.8167058859452209,
0.8334481274816485,
0.8553426860845104,
0.881620900295765,
0.9078073235269309,
0.9463822256778773,
1.007386673506787,
1.0852166516788362,
1.191656280802424,
1.3377374828349577,
1.5256138528774221,
1.850656706627787,
2.4403533720209807
],
"population_cv": 0.3576152457433685
},
"bits_per_byte": 1.9501283044158226,
"branch_output_rms_by_sublayer": [
0.02630079723894596,
0.06842244416475296,
0.10497251898050308,
0.019157659262418747,
0.09477359801530838,
0.0031024799682199955,
0.0338861346244812,
0.0047894855961203575,
0.022547677159309387,
0.004550074227154255,
0.021251287311315536,
0.004262411035597324,
0.01840241253376007,
0.006118505261838436,
0.02267448604106903,
0.009554358199238777,
0.022321712225675583,
0.00963375624269247,
0.01962294802069664,
0.011137282475829124,
0.019759519025683403,
0.013955196365714073,
0.021449215710163116,
0.01607326976954937,
0.026240134611725807,
0.015075869858264923,
0.039021123200654984,
0.02189606800675392,
0.031211141496896744,
0.022710295394062996,
0.03785466402769089,
0.026124922558665276,
0.0325845405459404,
0.026591502130031586,
0.03150996193289757,
0.029410794377326965,
0.02998937852680683,
0.024189060553908348,
0.04107306897640228,
0.029402343556284904,
0.040071673691272736,
0.032556869089603424,
0.04498095437884331,
0.03419904410839081,
0.05116673558950424,
0.041954249143600464,
0.04688612371683121,
0.050006624311208725,
0.052477724850177765,
0.06379573792219162,
0.04486159235239029,
0.06914625316858292,
0.032891806215047836,
0.0747767761349678,
0.015299716033041477,
0.08387137204408646,
0.017307085916399956,
0.09213023632764816,
0.017144370824098587,
0.10062393546104431,
0.017122838646173477,
0.15051087737083435,
0.027784937992691994,
0.24076412618160248
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.00032837002715831827,
0.0002079177996867109,
0.00018809406473136247,
0.00010899326506499365,
9.474430680287842e-05,
8.318366756728252e-05,
9.647881676005136e-05,
0.00013404889675397192,
0.00012865721885494213,
0.0001321695767247016,
0.00014056789256563647,
0.00015176892928157556,
0.00016698296163968515,
0.00022933649843535653,
0.000217708018553423,
0.0002507549770035681,
0.00025689149388127776,
0.0002828493196393162,
0.00027498157120012506,
0.0003306027661240732,
0.00032889440594708014,
0.00033897343531885953,
0.000372930739966371,
0.0003793341532875068,
0.0003649148263870391,
0.0003190518951646816,
0.00028800578855572547,
0.0002568521570555004,
0.00024004963937954607,
0.00021095610212654392,
0.00019368945551758278,
0.00017448279084866967
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.00015522885556569618,
"first_to_last_ratio": 0.6063619309634984,
"imbalance_abs_log_ratio": 0.5002782253787689,
"last_quartile_mean": 0.00025600033187941115,
"mean": 0.00022728867056201116,
"normalized": [
1.4447267712304608,
0.9147741467825811,
0.8275558313851142,
0.4795367265578555,
0.41684570800914306,
0.36598246345317736,
0.42447701648080627,
0.5897737728084399,
0.56605205414249,
0.581505344713791,
0.6184553423540983,
0.6677364468114501,
0.7346734935216545,
1.0090098106002459,
0.9578480881387607,
1.1032445056919573,
1.1302432859766764,
1.244449707677561,
1.2098340428503753,
1.4545501335662696,
1.4470338760565185,
1.491378494496396,
1.6407801543483633,
1.668953196609125,
1.605512608634311,
1.4037298664107178,
1.2671365794149818,
1.1300702160841907,
1.05614432424626,
0.928141739774877,
0.8521738238806694,
0.7676704272906798
],
"population_cv": 0.3864254083322711
},
"depth_weights": [],
"layer_input_rms_by_sublayer": [
0.06033600866794586,
0.06945198029279709,
0.11079900711774826,
0.17341101169586182,
0.18173734843730927,
0.25461602210998535,
0.25566670298576355,
0.26339152455329895,
0.2647266387939453,
0.2662918269634247,
0.2669948637485504,
0.2675604224205017,
0.2675777077674866,
0.2677975296974182,
0.2675747275352478,
0.2675320506095886,
0.26725223660469055,
0.26636025309562683,
0.2656837999820709,
0.2656107544898987,
0.265005499124527,
0.26352381706237793,
0.25976043939590454,
0.2553655505180359,
0.2535160183906555,
0.24811074137687683,
0.24569463729858398,
0.23600345849990845,
0.23344992101192474,
0.22498825192451477,
0.22194796800613403,
0.21307076513767242,
0.21299737691879272,
0.20943206548690796,
0.21236176788806915,
0.21104833483695984,
0.21684354543685913,
0.21818861365318298,
0.22108736634254456,
0.22058506309986115,
0.22472712397575378,
0.22517690062522888,
0.22933396697044373,
0.2308174967765808,
0.23535853624343872,
0.23589646816253662,
0.2425893247127533,
0.241608664393425,
0.2497948557138443,
0.24810902774333954,
0.26040923595428467,
0.26088401675224304,
0.27719539403915405,
0.2782979905605316,
0.2986113131046295,
0.2985531687736511,
0.3278995454311371,
0.3279455602169037,
0.36809566617012024,
0.3681766986846924,
0.4197922646999359,
0.4200114607810974,
0.5092319846153259,
0.5118712186813354
],
"loss_nats": 1.3517259359359741,
"loss_scale": 1.0,
"output_weights": null,
"step": 4000,
"stream_state_rms_by_sublayer": [
0.06945198029279709,
0.11079900711774826,
0.17341101169586182,
0.18173734843730927,
0.25461602210998535,
0.25566670298576355,
0.26339152455329895,
0.2647266387939453,
0.2662918269634247,
0.2669948637485504,
0.2675604224205017,
0.2675777077674866,
0.2677975296974182,
0.2675747275352478,
0.2675320506095886,
0.26725223660469055,
0.26636025309562683,
0.2656837999820709,
0.2656107544898987,
0.265005499124527,
0.26352381706237793,
0.25976043939590454,
0.2553655505180359,
0.2535160183906555,
0.24811074137687683,
0.24569463729858398,
0.23600345849990845,
0.23344992101192474,
0.22498825192451477,
0.22194796800613403,
0.21307076513767242,
0.21299737691879272,
0.20943206548690796,
0.21236176788806915,
0.21104833483695984,
0.21684354543685913,
0.21818861365318298,
0.22108736634254456,
0.22058506309986115,
0.22472712397575378,
0.22517690062522888,
0.22933396697044373,
0.2308174967765808,
0.23535853624343872,
0.23589646816253662,
0.2425893247127533,
0.241608664393425,
0.2497948557138443,
0.24810902774333954,
0.26040923595428467,
0.26088401675224304,
0.27719539403915405,
0.2782979905605316,
0.2986113131046295,
0.2985531687736511,
0.3278995454311371,
0.3279455602169037,
0.36809566617012024,
0.3681766986846924,
0.4197922646999359,
0.4200114607810974,
0.5092319846153259,
0.5118712186813354,
0.6714946031570435
]
},
{
"activation_grad_rms_by_block": [
0.00018891454965341836,
0.0001931450387928635,
0.0002085021696984768,
0.00020282011246308684,
0.00019955210154876113,
0.00019673256610985845,
0.00019503913063090295,
0.0001924382959259674,
0.00019001582404598594,
0.00018850783817470074,
0.00018595026631373912,
0.0001838041644077748,
0.00018427071336191148,
0.00017771514831110835,
0.0001728432544041425,
0.0001671368081588298,
0.0001615467481315136,
0.00015400326810777187,
0.000160209892783314,
0.00015067463391460478,
0.00014840754738543183,
0.00014097621897235513,
0.00012854386295657605,
0.00011642893514363095,
0.00010425484651932493,
9.334496280644089e-05,
8.331386925419793e-05,
7.313644891837612e-05,
6.15216777077876e-05,
4.904071465716697e-05,
3.525121428538114e-05,
2.4962779207271524e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 0.00019714299560291693,
"first_to_last_ratio": 3.0050767723956184,
"imbalance_abs_log_ratio": 1.1003031158789967,
"last_quartile_mean": 6.560331416949339e-05,
"mean": 0.00014728142508602104,
"normalized": [
1.282677360998382,
1.311401208129643,
1.4156718647765612,
1.3770922731405397,
1.3549033860326305,
1.3357595229334251,
1.324261566025645,
1.3066026202121013,
1.2901547084773621,
1.2799159029361715,
1.262550699826086,
1.247979263511487,
1.2511470014245618,
1.206636534154339,
1.1735577266664268,
1.1348125404219307,
1.0968575842959203,
1.045639448544342,
1.0877807075110928,
1.023038946198422,
1.0076460578701834,
0.9571894004285788,
0.8727771535446431,
0.7905201560592559,
0.707861473084158,
0.6337863905914947,
0.5656780493910737,
0.4965761899415362,
0.41771511697320496,
0.33297284181304126,
0.23934596141225786,
0.1694903426735027
],
"population_cv": 0.361085575415688
},
"activation_output_rms_by_block": [
0.1016591340303421,
0.18086770176887512,
0.2325568050146103,
0.2376081645488739,
0.23870401084423065,
0.23941391706466675,
0.23885947465896606,
0.2381371706724167,
0.23675362765789032,
0.23536528646945953,
0.22870509326457977,
0.22131286561489105,
0.21254637837409973,
0.19929367303848267,
0.18746964633464813,
0.17704539000988007,
0.17596758902072906,
0.1784123033285141,
0.18117663264274597,
0.18366514146327972,
0.1872515082359314,
0.19141052663326263,
0.1969655156135559,
0.2033752053976059,
0.21314463019371033,
0.22718842327594757,
0.2457481175661087,
0.269707053899765,
0.30305343866348267,
0.347743958234787,
0.4299577474594116,
0.6035616993904114
],
"activation_output_statistics": {
"first_quartile_mean": 0.2134757973253727,
"first_to_last_ratio": 0.6468706108937196,
"imbalance_abs_log_ratio": 0.4356089876432565,
"last_quartile_mean": 0.33001313358545303,
"mean": 0.23576961969956756,
"normalized": [
0.4311799550759871,
0.7671374369579427,
0.986373076017804,
1.0077980566438083,
1.0124460104249322,
1.015457026947505,
1.0131053990897376,
1.010041798327817,
1.0041735994636487,
0.9982850494876174,
0.9700363157730422,
0.9386827102529232,
0.9015002808459363,
0.8452898778580343,
0.7951391132306771,
0.7509253746750002,
0.7463539587711004,
0.7567230398719659,
0.7684477451913126,
0.7790025775895867,
0.7942138960674365,
0.8118540755045961,
0.8354151644496934,
0.8626014057992686,
0.9040377232033228,
0.9636034683579899,
1.0423230858973958,
1.143943202875089,
1.2853795117863462,
1.4749311581276,
1.8236350722679655,
2.5599638331669174
],
"population_cv": 0.3675345504070441
},
"bits_per_byte": 1.8720084529317582,
"branch_output_rms_by_sublayer": [
0.03573180362582207,
0.052011821419000626,
0.10664675384759903,
0.01823282428085804,
0.06781794875860214,
0.0026424091774970293,
0.02450474537909031,
0.003992235753685236,
0.01725216954946518,
0.0036646369844675064,
0.018646538257598877,
0.003319106763228774,
0.015910238027572632,
0.004927239380776882,
0.018212314695119858,
0.008043181151151657,
0.016784843057394028,
0.007957165129482746,
0.01594853773713112,
0.009037249721586704,
0.0176989808678627,
0.012198803015053272,
0.019605962559580803,
0.013432997278869152,
0.022475004196166992,
0.012972000986337662,
0.03453320264816284,
0.018862294033169746,
0.026713354513049126,
0.01990535296499729,
0.0319337360560894,
0.023401649668812752,
0.02801610343158245,
0.023707188665866852,
0.028282491490244865,
0.026515409350395203,
0.02569115161895752,
0.022723721340298653,
0.03440641239285469,
0.027487577870488167,
0.034427426755428314,
0.03072489984333515,
0.039456624537706375,
0.03320617973804474,
0.043320003896951675,
0.04054662957787514,
0.04175209999084473,
0.04858461394906044,
0.043093886226415634,
0.060716040432453156,
0.03683279827237129,
0.0668661892414093,
0.03015786223113537,
0.0722465068101883,
0.015251493081450462,
0.07915013283491135,
0.017382971942424774,
0.08651599287986755,
0.015752796083688736,
0.09494592994451523,
0.018999194726347923,
0.14218401908874512,
0.03338775411248207,
0.24767111241817474
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.0004570302696620844,
0.00031493934518203267,
0.0002143615651481701,
0.00013786409436507824,
0.0001291171736265367,
0.00013682107280354788,
0.00015837516984797755,
0.00019563631023119442,
0.00019674727625245423,
0.000191343414336242,
0.00021955065404037036,
0.0002332172506517699,
0.00027919484030541734,
0.0003858941801364767,
0.0003704235701666674,
0.00042305543536477747,
0.0004472346343569547,
0.0004940425030518999,
0.0004673224693273912,
0.0005594278228367747,
0.0005764247107952261,
0.0005972083894683241,
0.0006512089060709392,
0.0006542639335938355,
0.0006197665059356166,
0.0005424206044372668,
0.0004994517842756415,
0.0004394000632885711,
0.0004055610719669202,
0.0003512581615295656,
0.00030757820471853463,
0.00025263519960351045
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.00021801812510832775,
"first_to_last_ratio": 0.5102716406035511,
"imbalance_abs_log_ratio": 0.6728120664256351,
"last_quartile_mean": 0.0004272589494694534,
"mean": 0.0003721492683555553,
"normalized": [
1.2280832142478726,
0.8462715688618158,
0.5760096374645209,
0.3704537562958782,
0.34694995961452973,
0.36765105950128485,
0.42556894051626687,
0.5256931206546978,
0.5286783905872947,
0.5141577066152674,
0.5899532061705127,
0.6266766334979106,
0.7502227306239702,
1.0369338675356186,
0.9953628870573538,
1.1367896468913272,
1.2017614231332092,
1.3275385579419885,
1.2557393203871798,
1.5032350468100115,
1.5489072794427852,
1.6047549740115168,
1.7498594285795221,
1.7580685741634856,
1.6653707494152192,
1.4575350553128925,
1.3420738041017997,
1.1807091956143891,
1.0897806510785424,
0.9438636359052839,
0.8264914937967075,
0.6788544841693472
],
"population_cv": 0.43653500885793567
},
"depth_weights": [],
"layer_input_rms_by_sublayer": [
0.06193641945719719,
0.07578708976507187,
0.1016591340303421,
0.17375877499580383,
0.18086770176887512,
0.23162810504436493,
0.2325568050146103,
0.2364615947008133,
0.2376081645488739,
0.23820650577545166,
0.23870401084423065,
0.23957808315753937,
0.23941391706466675,
0.2393639236688614,
0.23885947465896606,
0.23879873752593994,
0.2381371706724167,
0.23766019940376282,
0.23675362765789032,
0.23612910509109497,
0.23536528646945953,
0.2328052520751953,
0.22870509326457977,
0.22353777289390564,
0.22131286561489105,
0.21547429263591766,
0.21254637837409973,
0.20247745513916016,
0.19929367303848267,
0.19119344651699066,
0.18746964633464813,
0.1784159243106842,
0.17704539000988007,
0.17436346411705017,
0.17596758902072906,
0.17439287900924683,
0.1784123033285141,
0.1789977103471756,
0.18117663264274597,
0.18044494092464447,
0.18366514146327972,
0.1840001940727234,
0.1872515082359314,
0.18770679831504822,
0.19141052663326263,
0.19118990004062653,
0.1969655156135559,
0.19665203988552094,
0.2033752053976059,
0.20245759189128876,
0.21314463019371033,
0.21310319006443024,
0.22718842327594757,
0.2287222445011139,
0.2457481175661087,
0.2456687092781067,
0.269707053899765,
0.26972076296806335,
0.30305343866348267,
0.30293965339660645,
0.347743958234787,
0.3480367064476013,
0.4299577474594116,
0.43200185894966125
],
"loss_nats": 1.2975773811340332,
"loss_scale": 1.0,
"output_weights": null,
"step": 8000,
"stream_state_rms_by_sublayer": [
0.07578708976507187,
0.1016591340303421,
0.17375877499580383,
0.18086770176887512,
0.23162810504436493,
0.2325568050146103,
0.2364615947008133,
0.2376081645488739,
0.23820650577545166,
0.23870401084423065,
0.23957808315753937,
0.23941391706466675,
0.2393639236688614,
0.23885947465896606,
0.23879873752593994,
0.2381371706724167,
0.23766019940376282,
0.23675362765789032,
0.23612910509109497,
0.23536528646945953,
0.2328052520751953,
0.22870509326457977,
0.22353777289390564,
0.22131286561489105,
0.21547429263591766,
0.21254637837409973,
0.20247745513916016,
0.19929367303848267,
0.19119344651699066,
0.18746964633464813,
0.1784159243106842,
0.17704539000988007,
0.17436346411705017,
0.17596758902072906,
0.17439287900924683,
0.1784123033285141,
0.1789977103471756,
0.18117663264274597,
0.18044494092464447,
0.18366514146327972,
0.1840001940727234,
0.1872515082359314,
0.18770679831504822,
0.19141052663326263,
0.19118990004062653,
0.1969655156135559,
0.19665203988552094,
0.2033752053976059,
0.20245759189128876,
0.21314463019371033,
0.21310319006443024,
0.22718842327594757,
0.2287222445011139,
0.2457481175661087,
0.2456687092781067,
0.269707053899765,
0.26972076296806335,
0.30305343866348267,
0.30293965339660645,
0.347743958234787,
0.3480367064476013,
0.4299577474594116,
0.43200185894966125,
0.6035616993904114
]
}
],
"environment": {
"autocast": "cuda-bfloat16-forward-fp32-cross-entropy",
"compile": false,
"compute_capability": [
12,
0
],
"cublas_workspace_config": ":4096:8",
"cuda": "12.8",
"deterministic_algorithms": true,
"gpu": "NVIDIA GeForce RTX 5090",
"python": "3.10.14",
"torch": "2.11.0+cu128"
},
"evaluations": [
{
"bits_per_byte": 7.95552135286859,
"cross_entropy_nats": 5.514347195625305,
"step": 0
},
{
"bits_per_byte": 5.166060485015187,
"cross_entropy_nats": 3.5808402597904205,
"step": 100
},
{
"bits_per_byte": 3.1277970484292927,
"cross_entropy_nats": 2.168023705482483,
"step": 500
},
{
"bits_per_byte": 1.9613801618593651,
"cross_entropy_nats": 1.359525129199028,
"step": 2000
},
{
"bits_per_byte": 1.7971937855298066,
"cross_entropy_nats": 1.2457198053598404,
"step": 4000
},
{
"bits_per_byte": 1.7178972072507075,
"cross_entropy_nats": 1.1907556056976318,
"step": 8000
}
],
"gradient_gate": null,
"hashes": {
"final_mixer_parameters": null,
"final_model_state": "b93ef93bca883c40ec5d076dddd2b9f0759523f4f41313e8f1ca34c1bdbdcf6c",
"final_optimizer_state": "06d1b38e173978179caca1f5bf426eaad5609e9e9a1b567ef88c8b18eeab7e4d",
"final_public_parameters": "b93ef93bca883c40ec5d076dddd2b9f0759523f4f41313e8f1ca34c1bdbdcf6c",
"initial_mixer_parameters": null,
"initial_public_parameter_elements": 18985152,
"initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670",
"initial_public_parameter_tensors": 227,
"initial_public_parameters": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58"
},
"manifest": {
"diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716",
"file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371",
"formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
"input_gate_tensor_hashes": {
"0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434",
"1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94",
"7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc"
},
"path": "experiments/k3/attnres_gradient/manifest.json",
"validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338"
},
"model": {
"attnres_aggregation_groups": 8,
"context": 256,
"d_ff": 768,
"d_head": 32,
"d_model": 192,
"heads": 6,
"layers": 32,
"parameters": {
"core": 18985152,
"embedding": 98304,
"mixer": 0,
"total": 18985152
},
"sublayers": 64,
"sublayers_per_attnres_group": 8,
"transformer_blocks_per_attnres_group": 4,
"vocabulary": 256
},
"optimizer": {
"betas": [
0.9,
0.95
],
"epsilon": 1e-08,
"grad_clip": 1.0,
"min_lr": 3e-05,
"name": "AdamW",
"peak_lr": 0.0003,
"warmup_steps": 400,
"weight_decay_ndim_ge_2": 0.1
},
"protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
"run_kind": "formal",
"schema_version": 1,
"seed": 2026073001,
"steps": 8000,
"target_bytes_seen": 65536000,
"timing": {
"mean_ms": 42.00224438344842,
"measured_steps": 7980,
"median_ms": 41.91726399585605,
"p95_ms": 42.462232409161516,
"peak_allocated_bytes": 5938562048,
"peak_reserved_bytes": 6325010432,
"warmup_steps_excluded": 20
},
"training_history": [
{
"bits_per_byte": 7.958522622108798,
"learning_rate": 7.499999999999999e-07,
"loss_nats": 5.516427516937256,
"step": 1,
"unclipped_grad_norm": 22.09486961364746
},
{
"bits_per_byte": 7.238571600789322,
"learning_rate": 7.499999999999999e-06,
"loss_nats": 5.017395496368408,
"step": 10,
"unclipped_grad_norm": 11.172341346740723
},
{
"bits_per_byte": 6.729949421197048,
"learning_rate": 1.4999999999999999e-05,
"loss_nats": 4.6648454666137695,
"step": 20,
"unclipped_grad_norm": 3.462282180786133
},
{
"bits_per_byte": 6.60904492970989,
"learning_rate": 2.2499999999999998e-05,
"loss_nats": 4.581040859222412,
"step": 30,
"unclipped_grad_norm": 3.06864070892334
},
{
"bits_per_byte": 6.370936325451538,
"learning_rate": 2.9999999999999997e-05,
"loss_nats": 4.415996551513672,
"step": 40,
"unclipped_grad_norm": 3.234333038330078
},
{
"bits_per_byte": 6.086515545038246,
"learning_rate": 3.75e-05,
"loss_nats": 4.218851089477539,
"step": 50,
"unclipped_grad_norm": 2.8305625915527344
},
{
"bits_per_byte": 5.8833778930063385,
"learning_rate": 4.4999999999999996e-05,
"loss_nats": 4.078046798706055,
"step": 60,
"unclipped_grad_norm": 2.670325517654419
},
{
"bits_per_byte": 5.715139992383416,
"learning_rate": 5.2499999999999995e-05,
"loss_nats": 3.961433172225952,
"step": 70,
"unclipped_grad_norm": 3.2375030517578125
},
{
"bits_per_byte": 5.566718964688812,
"learning_rate": 5.9999999999999995e-05,
"loss_nats": 3.858555555343628,
"step": 80,
"unclipped_grad_norm": 2.6123173236846924
},
{
"bits_per_byte": 5.355997221719156,
"learning_rate": 6.749999999999999e-05,
"loss_nats": 3.712494373321533,
"step": 90,
"unclipped_grad_norm": 2.440842628479004
},
{
"bits_per_byte": 5.182009597112589,
"learning_rate": 7.5e-05,
"loss_nats": 3.591895341873169,
"step": 100,
"unclipped_grad_norm": 2.292849063873291
},
{
"bits_per_byte": 4.992510513259403,
"learning_rate": 8.249999999999999e-05,
"loss_nats": 3.4605445861816406,
"step": 110,
"unclipped_grad_norm": 2.057412624359131
},
{
"bits_per_byte": 4.778040580396842,
"learning_rate": 8.999999999999999e-05,
"loss_nats": 3.311885356903076,
"step": 120,
"unclipped_grad_norm": 2.002248525619507
},
{
"bits_per_byte": 4.588237775182233,
"learning_rate": 9.75e-05,
"loss_nats": 3.180324077606201,
"step": 130,
"unclipped_grad_norm": 1.7531825304031372
},
{
"bits_per_byte": 4.430652823919056,
"learning_rate": 0.00010499999999999999,
"loss_nats": 3.071094512939453,
"step": 140,
"unclipped_grad_norm": 1.4980645179748535
},
{
"bits_per_byte": 4.274817968111058,
"learning_rate": 0.0001125,
"loss_nats": 2.963078022003174,
"step": 150,
"unclipped_grad_norm": 1.3038520812988281
},
{
"bits_per_byte": 4.169977000179788,
"learning_rate": 0.00011999999999999999,
"loss_nats": 2.8904078006744385,
"step": 160,
"unclipped_grad_norm": 1.0998529195785522
},
{
"bits_per_byte": 4.028519206053318,
"learning_rate": 0.00012749999999999998,
"loss_nats": 2.7923567295074463,
"step": 170,
"unclipped_grad_norm": 0.9818326234817505
},
{
"bits_per_byte": 3.950109907767851,
"learning_rate": 0.00013499999999999997,
"loss_nats": 2.7380075454711914,
"step": 180,
"unclipped_grad_norm": 0.8779997229576111
},
{
"bits_per_byte": 3.846655463967837,
"learning_rate": 0.0001425,
"loss_nats": 2.6662983894348145,
"step": 190,
"unclipped_grad_norm": 1.2310962677001953
},
{
"bits_per_byte": 3.7855644746589534,
"learning_rate": 0.00015,
"loss_nats": 2.623953342437744,
"step": 200,
"unclipped_grad_norm": 1.0928086042404175
},
{
"bits_per_byte": 3.7365611139833117,
"learning_rate": 0.0001575,
"loss_nats": 2.589986801147461,
"step": 210,
"unclipped_grad_norm": 0.7771895527839661
},
{
"bits_per_byte": 3.6369384436445213,
"learning_rate": 0.00016499999999999997,
"loss_nats": 2.5209336280822754,
"step": 220,
"unclipped_grad_norm": 0.7917730212211609
},
{
"bits_per_byte": 3.630690657903977,
"learning_rate": 0.0001725,
"loss_nats": 2.5166029930114746,
"step": 230,
"unclipped_grad_norm": 1.0061120986938477
},
{
"bits_per_byte": 3.503533565162051,
"learning_rate": 0.00017999999999999998,
"loss_nats": 2.428464412689209,
"step": 240,
"unclipped_grad_norm": 0.7126089930534363
},
{
"bits_per_byte": 3.5514149110234636,
"learning_rate": 0.0001875,
"loss_nats": 2.461653232574463,
"step": 250,
"unclipped_grad_norm": 1.2347668409347534
},
{
"bits_per_byte": 3.5407041754930297,
"learning_rate": 0.000195,
"loss_nats": 2.4542291164398193,
"step": 260,
"unclipped_grad_norm": 0.9977514147758484
},
{
"bits_per_byte": 3.5337584841552845,
"learning_rate": 0.00020249999999999996,
"loss_nats": 2.4494147300720215,
"step": 270,
"unclipped_grad_norm": 0.7908692955970764
},
{
"bits_per_byte": 3.478037825102243,
"learning_rate": 0.00020999999999999998,
"loss_nats": 2.410792112350464,
"step": 280,
"unclipped_grad_norm": 1.0802956819534302
},
{
"bits_per_byte": 3.3705159916093237,
"learning_rate": 0.00021749999999999997,
"loss_nats": 2.336263656616211,
"step": 290,
"unclipped_grad_norm": 0.9241761565208435
},
{
"bits_per_byte": 3.410749956917471,
"learning_rate": 0.000225,
"loss_nats": 2.3641517162323,
"step": 300,
"unclipped_grad_norm": 0.9868955612182617
},
{
"bits_per_byte": 3.3596274260178975,
"learning_rate": 0.00023249999999999996,
"loss_nats": 2.328716278076172,
"step": 310,
"unclipped_grad_norm": 0.8471657037734985
},
{
"bits_per_byte": 3.3434163413475693,
"learning_rate": 0.00023999999999999998,
"loss_nats": 2.3174796104431152,
"step": 320,
"unclipped_grad_norm": 0.9364105463027954
},
{
"bits_per_byte": 3.3338296844231747,
"learning_rate": 0.0002475,
"loss_nats": 2.3108346462249756,
"step": 330,
"unclipped_grad_norm": 0.9398566484451294
},
{
"bits_per_byte": 3.4261592584694767,
"learning_rate": 0.00025499999999999996,
"loss_nats": 2.3748326301574707,
"step": 340,
"unclipped_grad_norm": 0.949837863445282
},
{
"bits_per_byte": 3.364650007353695,
"learning_rate": 0.0002625,
"loss_nats": 2.332197666168213,
"step": 350,
"unclipped_grad_norm": 0.8382212519645691
},
{
"bits_per_byte": 3.2974426270495267,
"learning_rate": 0.00026999999999999995,
"loss_nats": 2.2856130599975586,
"step": 360,
"unclipped_grad_norm": 1.0729857683181763
},
{
"bits_per_byte": 3.304221839181237,
"learning_rate": 0.00027749999999999997,
"loss_nats": 2.2903120517730713,
"step": 370,
"unclipped_grad_norm": 0.9106518030166626
},
{
"bits_per_byte": 3.3613300542614395,
"learning_rate": 0.000285,
"loss_nats": 2.3298964500427246,
"step": 380,
"unclipped_grad_norm": 1.290928602218628
},
{
"bits_per_byte": 3.2658243046190027,
"learning_rate": 0.0002925,
"loss_nats": 2.2636969089508057,
"step": 390,
"unclipped_grad_norm": 1.1257888078689575
},
{
"bits_per_byte": 3.2284232375358677,
"learning_rate": 0.0003,
"loss_nats": 2.2377724647521973,
"step": 400,
"unclipped_grad_norm": 1.3445736169815063
},
{
"bits_per_byte": 3.1961283353568497,
"learning_rate": 0.0002999988466112388,
"loss_nats": 2.2153873443603516,
"step": 410,
"unclipped_grad_norm": 0.7953492403030396
},
{
"bits_per_byte": 3.2387222466001364,
"learning_rate": 0.00029999538646466355,
"loss_nats": 2.2449111938476562,
"step": 420,
"unclipped_grad_norm": 0.8595701456069946
},
{
"bits_per_byte": 3.2223075344711862,
"learning_rate": 0.0002999896196193985,
"loss_nats": 2.2335333824157715,
"step": 430,
"unclipped_grad_norm": 1.2960833311080933
},
{
"bits_per_byte": 3.2629112624786885,
"learning_rate": 0.00029998154617398326,
"loss_nats": 2.2616777420043945,
"step": 440,
"unclipped_grad_norm": 0.9954231977462769
},
{
"bits_per_byte": 3.169579029577856,
"learning_rate": 0.0002999711662663707,
"loss_nats": 2.1969847679138184,
"step": 450,
"unclipped_grad_norm": 1.0229040384292603
},
{
"bits_per_byte": 3.210129442963591,
"learning_rate": 0.0002999584800739247,
"loss_nats": 2.2250921726226807,
"step": 460,
"unclipped_grad_norm": 1.1146044731140137
},
{
"bits_per_byte": 3.160413042217472,
"learning_rate": 0.0002999434878134173,
"loss_nats": 2.19063138961792,
"step": 470,
"unclipped_grad_norm": 0.9123480319976807
},
{
"bits_per_byte": 3.1596232978847096,
"learning_rate": 0.000299926189741025,
"loss_nats": 2.1900839805603027,
"step": 480,
"unclipped_grad_norm": 0.7869184613227844
},
{
"bits_per_byte": 3.0856473683731847,
"learning_rate": 0.000299906586152324,
"loss_nats": 2.138807773590088,
"step": 490,
"unclipped_grad_norm": 0.8606863021850586
},
{
"bits_per_byte": 3.05385947104891,
"learning_rate": 0.0002998846773822856,
"loss_nats": 2.116774082183838,
"step": 500,
"unclipped_grad_norm": 0.9675024151802063
},
{
"bits_per_byte": 3.0816491157059263,
"learning_rate": 0.0002998604638052702,
"loss_nats": 2.1360363960266113,
"step": 510,
"unclipped_grad_norm": 0.8225864768028259
},
{
"bits_per_byte": 3.0308351216820006,
"learning_rate": 0.0002998339458350211,
"loss_nats": 2.1008148193359375,
"step": 520,
"unclipped_grad_norm": 1.098574161529541
},
{
"bits_per_byte": 3.0891052515514326,
"learning_rate": 0.0002998051239246572,
"loss_nats": 2.141204595565796,
"step": 530,
"unclipped_grad_norm": 0.8098863363265991
},
{
"bits_per_byte": 3.0354875963531414,
"learning_rate": 0.00029977399856666537,
"loss_nats": 2.1040396690368652,
"step": 540,
"unclipped_grad_norm": 1.0088073015213013
},
{
"bits_per_byte": 3.0042414443790846,
"learning_rate": 0.0002997405702928922,
"loss_nats": 2.0823814868927,
"step": 550,
"unclipped_grad_norm": 1.054882287979126
},
{
"bits_per_byte": 2.9880702596837563,
"learning_rate": 0.0002997048396745345,
"loss_nats": 2.0711724758148193,
"step": 560,
"unclipped_grad_norm": 0.723493754863739
},
{
"bits_per_byte": 2.93916561704971,
"learning_rate": 0.0002996668073221302,
"loss_nats": 2.0372743606567383,
"step": 570,
"unclipped_grad_norm": 1.0234330892562866
},
{
"bits_per_byte": 3.0140372322069284,
"learning_rate": 0.000299626473885547,
"loss_nats": 2.0891714096069336,
"step": 580,
"unclipped_grad_norm": 1.0687707662582397
},
{
"bits_per_byte": 2.915940047981291,
"learning_rate": 0.00029958384005397226,
"loss_nats": 2.0211756229400635,
"step": 590,
"unclipped_grad_norm": 1.149570107460022
},
{
"bits_per_byte": 2.9613039117993742,
"learning_rate": 0.0002995389065559004,
"loss_nats": 2.052619457244873,
"step": 600,
"unclipped_grad_norm": 0.7275651693344116
},
{
"bits_per_byte": 2.880448160305384,
"learning_rate": 0.000299491674159121,
"loss_nats": 1.9965745210647583,
"step": 610,
"unclipped_grad_norm": 0.9942755699157715
},
{
"bits_per_byte": 2.9149380770573643,
"learning_rate": 0.00029944214367070535,
"loss_nats": 2.0204811096191406,
"step": 620,
"unclipped_grad_norm": 0.8311108350753784
},
{
"bits_per_byte": 2.871522948656208,
"learning_rate": 0.00029939031593699284,
"loss_nats": 1.990388035774231,
"step": 630,
"unclipped_grad_norm": 1.2078016996383667
},
{
"bits_per_byte": 2.795491826466872,
"learning_rate": 0.00029933619184357654,
"loss_nats": 1.9376872777938843,
"step": 640,
"unclipped_grad_norm": 0.7877771258354187
},
{
"bits_per_byte": 2.7606237198656447,
"learning_rate": 0.00029927977231528774,
"loss_nats": 1.9135185480117798,
"step": 650,
"unclipped_grad_norm": 1.1925899982452393
},
{
"bits_per_byte": 2.6852617821708162,
"learning_rate": 0.00029922105831618057,
"loss_nats": 1.8612816333770752,
"step": 660,
"unclipped_grad_norm": 1.126668930053711
},
{
"bits_per_byte": 2.7363994475436653,
"learning_rate": 0.0002991600508495153,
"loss_nats": 1.8967275619506836,
"step": 670,
"unclipped_grad_norm": 1.136749267578125
},
{
"bits_per_byte": 2.6585009377312616,
"learning_rate": 0.00029909675095774114,
"loss_nats": 1.8427324295043945,
"step": 680,
"unclipped_grad_norm": 1.2957242727279663
},
{
"bits_per_byte": 2.734127384743117,
"learning_rate": 0.0002990311597224787,
"loss_nats": 1.8951526880264282,
"step": 690,
"unclipped_grad_norm": 0.7534775137901306
},
{
"bits_per_byte": 2.6542058430086137,
"learning_rate": 0.00029896327826450103,
"loss_nats": 1.8397552967071533,
"step": 700,
"unclipped_grad_norm": 0.8543418049812317
},
{
"bits_per_byte": 2.6774995172067744,
"learning_rate": 0.0002988931077437152,
"loss_nats": 1.8559012413024902,
"step": 710,
"unclipped_grad_norm": 1.3297739028930664
},
{
"bits_per_byte": 2.6677150802338874,
"learning_rate": 0.00029882064935914164,
"loss_nats": 1.8491191864013672,
"step": 720,
"unclipped_grad_norm": 0.7874022126197815
},
{
"bits_per_byte": 2.6473420154126788,
"learning_rate": 0.0002987459043488943,
"loss_nats": 1.8349976539611816,
"step": 730,
"unclipped_grad_norm": 1.0262641906738281
},
{
"bits_per_byte": 2.582519862493327,
"learning_rate": 0.0002986688739901591,
"loss_nats": 1.7900663614273071,
"step": 740,
"unclipped_grad_norm": 1.1010198593139648
},
{
"bits_per_byte": 2.5681288702170795,
"learning_rate": 0.0002985895595991723,
"loss_nats": 1.7800912857055664,
"step": 750,
"unclipped_grad_norm": 0.9110153913497925
},
{
"bits_per_byte": 2.5014427693276766,
"learning_rate": 0.00029850796253119824,
"loss_nats": 1.7338680028915405,
"step": 760,
"unclipped_grad_norm": 1.0652238130569458
},
{
"bits_per_byte": 2.634176399523829,
"learning_rate": 0.00029842408418050544,
"loss_nats": 1.8258719444274902,
"step": 770,
"unclipped_grad_norm": 0.9231922626495361
},
{
"bits_per_byte": 2.5180344516043642,
"learning_rate": 0.0002983379259803436,
"loss_nats": 1.745368480682373,
"step": 780,
"unclipped_grad_norm": 0.8394486308097839
},
{
"bits_per_byte": 2.482792454720161,
"learning_rate": 0.0002982494894029186,
"loss_nats": 1.7209405899047852,
"step": 790,
"unclipped_grad_norm": 0.8228755593299866
},
{
"bits_per_byte": 2.430754632152696,
"learning_rate": 0.0002981587759593675,
"loss_nats": 1.684870719909668,
"step": 800,
"unclipped_grad_norm": 0.9281390905380249
},
{
"bits_per_byte": 2.489219790348206,
"learning_rate": 0.00029806578719973283,
"loss_nats": 1.725395679473877,
"step": 810,
"unclipped_grad_norm": 0.8680029511451721
},
{
"bits_per_byte": 2.4286650429447127,
"learning_rate": 0.00029797052471293596,
"loss_nats": 1.683422327041626,
"step": 820,
"unclipped_grad_norm": 1.0944814682006836
},
{
"bits_per_byte": 2.483867002322752,
"learning_rate": 0.0002978729901267499,
"loss_nats": 1.7216854095458984,
"step": 830,
"unclipped_grad_norm": 0.8509150743484497
},
{
"bits_per_byte": 2.566762812021276,
"learning_rate": 0.0002977731851077716,
"loss_nats": 1.7791444063186646,
"step": 840,
"unclipped_grad_norm": 1.2704709768295288
},
{
"bits_per_byte": 2.458091274507339,
"learning_rate": 0.0002976711113613935,
"loss_nats": 1.7038190364837646,
"step": 850,
"unclipped_grad_norm": 0.905615508556366
},
{
"bits_per_byte": 2.442831941826886,
"learning_rate": 0.0002975667706317743,
"loss_nats": 1.693242073059082,
"step": 860,
"unclipped_grad_norm": 0.7913721203804016
},
{
"bits_per_byte": 2.3506050414231483,
"learning_rate": 0.00029746016470180915,
"loss_nats": 1.6293152570724487,
"step": 870,
"unclipped_grad_norm": 1.0574872493743896
},
{
"bits_per_byte": 2.3881724157296675,
"learning_rate": 0.00029735129539309926,
"loss_nats": 1.6553549766540527,
"step": 880,
"unclipped_grad_norm": 0.923473060131073
},
{
"bits_per_byte": 2.321517099734769,
"learning_rate": 0.0002972401645659207,
"loss_nats": 1.6091530323028564,
"step": 890,
"unclipped_grad_norm": 1.0251853466033936
},
{
"bits_per_byte": 2.330866592583624,
"learning_rate": 0.00029712677411919283,
"loss_nats": 1.6156336069107056,
"step": 900,
"unclipped_grad_norm": 0.9670726656913757
},
{
"bits_per_byte": 2.356522276508734,
"learning_rate": 0.0002970111259904454,
"loss_nats": 1.633416771888733,
"step": 910,
"unclipped_grad_norm": 0.8682661652565002
},
{
"bits_per_byte": 2.329593920967239,
"learning_rate": 0.0002968932221557859,
"loss_nats": 1.6147514581680298,
"step": 920,
"unclipped_grad_norm": 0.9293603301048279
},
{
"bits_per_byte": 2.356717820782765,
"learning_rate": 0.0002967730646298655,
"loss_nats": 1.6335523128509521,
"step": 930,
"unclipped_grad_norm": 1.1330991983413696
},
{
"bits_per_byte": 2.3506738344834934,
"learning_rate": 0.0002966506554658449,
"loss_nats": 1.629362940788269,
"step": 940,
"unclipped_grad_norm": 0.8093944787979126
},
{
"bits_per_byte": 2.337553106066519,
"learning_rate": 0.00029652599675535876,
"loss_nats": 1.6202683448791504,
"step": 950,
"unclipped_grad_norm": 1.0507299900054932
},
{
"bits_per_byte": 2.3753796582278888,
"learning_rate": 0.0002963990906284808,
"loss_nats": 1.6464877128601074,
"step": 960,
"unclipped_grad_norm": 0.8050268888473511
},
{
"bits_per_byte": 2.2641018956227765,
"learning_rate": 0.0002962699392536863,
"loss_nats": 1.569355845451355,
"step": 970,
"unclipped_grad_norm": 0.8510206341743469
},
{
"bits_per_byte": 2.35890939570271,
"learning_rate": 0.0002961385448378161,
"loss_nats": 1.6350713968276978,
"step": 980,
"unclipped_grad_norm": 0.8616457581520081
},
{
"bits_per_byte": 2.3101203254100433,
"learning_rate": 0.00029600490962603817,
"loss_nats": 1.6012533903121948,
"step": 990,
"unclipped_grad_norm": 0.8957601189613342
},
{
"bits_per_byte": 2.2887248237294555,
"learning_rate": 0.00029586903590180956,
"loss_nats": 1.5864231586456299,
"step": 1000,
"unclipped_grad_norm": 1.0717068910598755
},
{
"bits_per_byte": 2.2687179099719343,
"learning_rate": 0.00029573092598683735,
"loss_nats": 1.572555422782898,
"step": 1010,
"unclipped_grad_norm": 0.9444129467010498
},
{
"bits_per_byte": 2.2310683279583023,
"learning_rate": 0.00029559058224103895,
"loss_nats": 1.5464587211608887,
"step": 1020,
"unclipped_grad_norm": 0.9090887308120728
},
{
"bits_per_byte": 2.2341806979909666,
"learning_rate": 0.0002954480070625017,
"loss_nats": 1.5486160516738892,
"step": 1030,
"unclipped_grad_norm": 0.8268627524375916
},
{
"bits_per_byte": 2.26161898209227,
"learning_rate": 0.00029530320288744205,
"loss_nats": 1.5676348209381104,
"step": 1040,
"unclipped_grad_norm": 0.7536599636077881
},
{
"bits_per_byte": 2.2636605881406626,
"learning_rate": 0.0002951561721901638,
"loss_nats": 1.5690499544143677,
"step": 1050,
"unclipped_grad_norm": 0.8793148398399353
},
{
"bits_per_byte": 2.1893296864377567,
"learning_rate": 0.00029500691748301594,
"loss_nats": 1.51752769947052,
"step": 1060,
"unclipped_grad_norm": 0.9039056301116943
},
{
"bits_per_byte": 2.2431745307178375,
"learning_rate": 0.00029485544131634963,
"loss_nats": 1.5548501014709473,
"step": 1070,
"unclipped_grad_norm": 0.862832248210907
},
{
"bits_per_byte": 2.24520426996332,
"learning_rate": 0.0002947017462784745,
"loss_nats": 1.5562570095062256,
"step": 1080,
"unclipped_grad_norm": 0.9674477577209473
},
{
"bits_per_byte": 2.1985577595351025,
"learning_rate": 0.00029454583499561476,
"loss_nats": 1.5239241123199463,
"step": 1090,
"unclipped_grad_norm": 0.7859865427017212
},
{
"bits_per_byte": 2.2599985615558404,
"learning_rate": 0.00029438771013186413,
"loss_nats": 1.566511631011963,
"step": 1100,
"unclipped_grad_norm": 0.788394570350647
},
{
"bits_per_byte": 2.119760812354591,
"learning_rate": 0.0002942273743891401,
"loss_nats": 1.469306230545044,
"step": 1110,
"unclipped_grad_norm": 0.8257600665092468
},
{
"bits_per_byte": 2.1691607650231224,
"learning_rate": 0.00029406483050713824,
"loss_nats": 1.5035476684570312,
"step": 1120,
"unclipped_grad_norm": 0.9345623254776001
},
{
"bits_per_byte": 2.1036112974732717,
"learning_rate": 0.00029390008126328504,
"loss_nats": 1.4581122398376465,
"step": 1130,
"unclipped_grad_norm": 0.6939264535903931
},
{
"bits_per_byte": 2.2552164119659492,
"learning_rate": 0.00029373312947269045,
"loss_nats": 1.5631968975067139,
"step": 1140,
"unclipped_grad_norm": 0.8951012492179871
},
{
"bits_per_byte": 2.1146625586524137,
"learning_rate": 0.00029356397798809995,
"loss_nats": 1.4657723903656006,
"step": 1150,
"unclipped_grad_norm": 0.9783032536506653
},
{
"bits_per_byte": 2.1242843000375844,
"learning_rate": 0.0002933926296998457,
"loss_nats": 1.4724416732788086,
"step": 1160,
"unclipped_grad_norm": 0.8440186977386475
},
{
"bits_per_byte": 2.174440976369912,
"learning_rate": 0.0002932190875357972,
"loss_nats": 1.5072076320648193,
"step": 1170,
"unclipped_grad_norm": 0.9151178002357483
},
{
"bits_per_byte": 2.1585999983642408,
"learning_rate": 0.00029304335446131126,
"loss_nats": 1.496227502822876,
"step": 1180,
"unclipped_grad_norm": 0.7418354153633118
},
{
"bits_per_byte": 2.091859378974514,
"learning_rate": 0.00029286543347918116,
"loss_nats": 1.4499664306640625,
"step": 1190,
"unclipped_grad_norm": 0.9675256609916687
},
{
"bits_per_byte": 2.0676216920057677,
"learning_rate": 0.0002926853276295856,
"loss_nats": 1.4331661462783813,
"step": 1200,
"unclipped_grad_norm": 0.937224805355072
},
{
"bits_per_byte": 2.116782072841647,
"learning_rate": 0.0002925030399900368,
"loss_nats": 1.4672415256500244,
"step": 1210,
"unclipped_grad_norm": 0.8679532408714294
},
{
"bits_per_byte": 2.1441637746508166,
"learning_rate": 0.00029231857367532726,
"loss_nats": 1.4862210750579834,
"step": 1220,
"unclipped_grad_norm": 0.7687044143676758
},
{
"bits_per_byte": 2.083470065265426,
"learning_rate": 0.0002921319318374776,
"loss_nats": 1.4441514015197754,
"step": 1230,
"unclipped_grad_norm": 0.7017022967338562
},
{
"bits_per_byte": 2.025742368676816,
"learning_rate": 0.00029194311766568174,
"loss_nats": 1.4041376113891602,
"step": 1240,
"unclipped_grad_norm": 0.8267619013786316
},
{
"bits_per_byte": 2.1229982137744323,
"learning_rate": 0.000291752134386253,
"loss_nats": 1.4715502262115479,
"step": 1250,
"unclipped_grad_norm": 1.0167953968048096
},
{
"bits_per_byte": 2.098591123894586,
"learning_rate": 0.00029155898526256856,
"loss_nats": 1.4546325206756592,
"step": 1260,
"unclipped_grad_norm": 0.839733898639679
},
{
"bits_per_byte": 2.0788139789586175,
"learning_rate": 0.0002913636735950141,
"loss_nats": 1.440924048423767,
"step": 1270,
"unclipped_grad_norm": 0.7740461230278015
},
{
"bits_per_byte": 1.9768289549275753,
"learning_rate": 0.0002911662027209272,
"loss_nats": 1.370233416557312,
"step": 1280,
"unclipped_grad_norm": 0.6849040389060974
},
{
"bits_per_byte": 2.098678835046526,
"learning_rate": 0.00029096657601454013,
"loss_nats": 1.45469331741333,
"step": 1290,
"unclipped_grad_norm": 0.7928483486175537
},
{
"bits_per_byte": 2.041355641652745,
"learning_rate": 0.00029076479688692266,
"loss_nats": 1.4149599075317383,
"step": 1300,
"unclipped_grad_norm": 0.7687125205993652
},
{
"bits_per_byte": 2.0466036922438238,
"learning_rate": 0.00029056086878592323,
"loss_nats": 1.4185975790023804,
"step": 1310,
"unclipped_grad_norm": 0.7732207179069519
},
{
"bits_per_byte": 2.038361079735922,
"learning_rate": 0.00029035479519611045,
"loss_nats": 1.41288423538208,
"step": 1320,
"unclipped_grad_norm": 0.8016889691352844
},
{
"bits_per_byte": 2.1092168999954937,
"learning_rate": 0.0002901465796387134,
"loss_nats": 1.4619977474212646,
"step": 1330,
"unclipped_grad_norm": 0.7918544411659241
},
{
"bits_per_byte": 2.0672703314500547,
"learning_rate": 0.00028993622567156154,
"loss_nats": 1.432922601699829,
"step": 1340,
"unclipped_grad_norm": 0.9473646879196167
},
{
"bits_per_byte": 2.0426272813732247,
"learning_rate": 0.0002897237368890237,
"loss_nats": 1.4158413410186768,
"step": 1350,
"unclipped_grad_norm": 0.7250363826751709
},
{
"bits_per_byte": 2.10636370781768,
"learning_rate": 0.0002895091169219469,
"loss_nats": 1.4600200653076172,
"step": 1360,
"unclipped_grad_norm": 0.7928993105888367
},
{
"bits_per_byte": 2.104627027009267,
"learning_rate": 0.00028929236943759425,
"loss_nats": 1.4588162899017334,
"step": 1370,
"unclipped_grad_norm": 0.794548749923706
},
{
"bits_per_byte": 2.0561698832327657,
"learning_rate": 0.00028907349813958225,
"loss_nats": 1.4252283573150635,
"step": 1380,
"unclipped_grad_norm": 0.7989111542701721
},
{
"bits_per_byte": 2.0527231789268243,
"learning_rate": 0.0002888525067678175,
"loss_nats": 1.4228392839431763,
"step": 1390,
"unclipped_grad_norm": 0.8791808485984802
},
{
"bits_per_byte": 1.9977298345042815,
"learning_rate": 0.00028862939909843273,
"loss_nats": 1.384720802307129,
"step": 1400,
"unclipped_grad_norm": 0.7230417132377625
},
{
"bits_per_byte": 2.02384763581226,
"learning_rate": 0.0002884041789437225,
"loss_nats": 1.4028242826461792,
"step": 1410,
"unclipped_grad_norm": 0.7468129396438599
},
{
"bits_per_byte": 2.0636549121636167,
"learning_rate": 0.0002881768501520778,
"loss_nats": 1.4304165840148926,
"step": 1420,
"unclipped_grad_norm": 0.7919746041297913
},
{
"bits_per_byte": 2.0795395737626077,
"learning_rate": 0.0002879474166079205,
"loss_nats": 1.4414269924163818,
"step": 1430,
"unclipped_grad_norm": 0.6952440738677979
},
{
"bits_per_byte": 2.0428340045195617,
"learning_rate": 0.00028771588223163694,
"loss_nats": 1.4159846305847168,
"step": 1440,
"unclipped_grad_norm": 0.9689956307411194
},
{
"bits_per_byte": 1.9802144334098097,
"learning_rate": 0.00028748225097951054,
"loss_nats": 1.3725800514221191,
"step": 1450,
"unclipped_grad_norm": 0.7872179746627808
},
{
"bits_per_byte": 2.013908586436248,
"learning_rate": 0.000287246526843655,
"loss_nats": 1.39593505859375,
"step": 1460,
"unclipped_grad_norm": 0.7631422281265259
},
{
"bits_per_byte": 1.9493597139491168,
"learning_rate": 0.00028700871385194533,
"loss_nats": 1.3511931896209717,
"step": 1470,
"unclipped_grad_norm": 0.7200304865837097
},
{
"bits_per_byte": 2.0209578113298123,
"learning_rate": 0.00028676881606794945,
"loss_nats": 1.4008212089538574,
"step": 1480,
"unclipped_grad_norm": 0.7176085114479065
},
{
"bits_per_byte": 2.0333618880406417,
"learning_rate": 0.00028652683759085854,
"loss_nats": 1.409419059753418,
"step": 1490,
"unclipped_grad_norm": 0.7242412567138672
},
{
"bits_per_byte": 2.033254742849154,
"learning_rate": 0.0002862827825554171,
"loss_nats": 1.4093447923660278,
"step": 1500,
"unclipped_grad_norm": 0.7850398421287537
},
{
"bits_per_byte": 2.0298035669942904,
"learning_rate": 0.00028603665513185245,
"loss_nats": 1.4069526195526123,
"step": 1510,
"unclipped_grad_norm": 0.7650813460350037
},
{
"bits_per_byte": 1.981937699571455,
"learning_rate": 0.000285788459525803,
"loss_nats": 1.373774528503418,
"step": 1520,
"unclipped_grad_norm": 0.7211347222328186
},
{
"bits_per_byte": 2.0555194448472025,
"learning_rate": 0.00028553819997824707,
"loss_nats": 1.4247775077819824,
"step": 1530,
"unclipped_grad_norm": 0.7807993292808533
},
{
"bits_per_byte": 1.9955152139091212,
"learning_rate": 0.00028528588076542964,
"loss_nats": 1.3831857442855835,
"step": 1540,
"unclipped_grad_norm": 0.9850755929946899
},
{
"bits_per_byte": 2.0666652964843193,
"learning_rate": 0.00028503150619878996,
"loss_nats": 1.4325032234191895,
"step": 1550,
"unclipped_grad_norm": 0.8105318546295166
},
{
"bits_per_byte": 1.9693205363562067,
"learning_rate": 0.0002847750806248876,
"loss_nats": 1.365028977394104,
"step": 1560,
"unclipped_grad_norm": 0.6913962364196777
},
{
"bits_per_byte": 2.059840508950131,
"learning_rate": 0.000284516608425328,
"loss_nats": 1.4277726411819458,
"step": 1570,
"unclipped_grad_norm": 0.725780189037323
},
{
"bits_per_byte": 1.9988298355392002,
"learning_rate": 0.000284256094016688,
"loss_nats": 1.3854832649230957,
"step": 1580,
"unclipped_grad_norm": 0.7167323231697083
},
{
"bits_per_byte": 2.0030889858878176,
"learning_rate": 0.0002839935418504399,
"loss_nats": 1.3884354829788208,
"step": 1590,
"unclipped_grad_norm": 0.7256693840026855
},
{
"bits_per_byte": 2.027964040560662,
"learning_rate": 0.000283728956412876,
"loss_nats": 1.4056775569915771,
"step": 1600,
"unclipped_grad_norm": 0.709171712398529
},
{
"bits_per_byte": 1.961726126459407,
"learning_rate": 0.00028346234222503136,
"loss_nats": 1.3597649335861206,
"step": 1610,
"unclipped_grad_norm": 0.6563887000083923
},
{
"bits_per_byte": 1.9260072216843611,
"learning_rate": 0.00028319370384260675,
"loss_nats": 1.3350064754486084,
"step": 1620,
"unclipped_grad_norm": 0.743120014667511
},
{
"bits_per_byte": 1.954758421342351,
"learning_rate": 0.00028292304585589105,
"loss_nats": 1.3549352884292603,
"step": 1630,
"unclipped_grad_norm": 0.6699017286300659
},
{
"bits_per_byte": 1.9950130245686017,
"learning_rate": 0.0002826503728896824,
"loss_nats": 1.3828376531600952,
"step": 1640,
"unclipped_grad_norm": 0.7793066501617432
},
{
"bits_per_byte": 2.0499475509245495,
"learning_rate": 0.0002823756896032095,
"loss_nats": 1.4209153652191162,
"step": 1650,
"unclipped_grad_norm": 0.6984328031539917
},
{
"bits_per_byte": 1.996117325169792,
"learning_rate": 0.0002820990006900518,
"loss_nats": 1.3836030960083008,
"step": 1660,
"unclipped_grad_norm": 0.7241947054862976
},
{
"bits_per_byte": 2.017874850330446,
"learning_rate": 0.00028182031087805947,
"loss_nats": 1.3986842632293701,
"step": 1670,
"unclipped_grad_norm": 0.7746724486351013
},
{
"bits_per_byte": 2.0244201660569825,
"learning_rate": 0.0002815396249292725,
"loss_nats": 1.4032211303710938,
"step": 1680,
"unclipped_grad_norm": 0.7699829936027527
},
{
"bits_per_byte": 2.06043436504356,
"learning_rate": 0.0002812569476398391,
"loss_nats": 1.4281842708587646,
"step": 1690,
"unclipped_grad_norm": 0.7417407035827637
},
{
"bits_per_byte": 1.95363795437198,
"learning_rate": 0.00028097228383993446,
"loss_nats": 1.354158639907837,
"step": 1700,
"unclipped_grad_norm": 0.6731021404266357
},
{
"bits_per_byte": 1.8890682719840945,
"learning_rate": 0.0002806856383936774,
"loss_nats": 1.309402346611023,
"step": 1710,
"unclipped_grad_norm": 0.6633850932121277
},
{
"bits_per_byte": 1.8893066399381904,
"learning_rate": 0.00028039701619904774,
"loss_nats": 1.3095675706863403,
"step": 1720,
"unclipped_grad_norm": 0.7114297151565552
},
{
"bits_per_byte": 1.8703618910323976,
"learning_rate": 0.0002801064221878024,
"loss_nats": 1.296436071395874,
"step": 1730,
"unclipped_grad_norm": 0.6913781762123108
},
{
"bits_per_byte": 1.988379825707474,
"learning_rate": 0.00027981386132539134,
"loss_nats": 1.3782398700714111,
"step": 1740,
"unclipped_grad_norm": 0.7100496888160706
},
{
"bits_per_byte": 2.0086424776668284,
"learning_rate": 0.0002795193386108724,
"loss_nats": 1.392284870147705,
"step": 1750,
"unclipped_grad_norm": 0.7558912038803101
},
{
"bits_per_byte": 1.996569639541561,
"learning_rate": 0.00027922285907682625,
"loss_nats": 1.3839166164398193,
"step": 1760,
"unclipped_grad_norm": 0.6800652742385864
},
{
"bits_per_byte": 2.0072186333003352,
"learning_rate": 0.00027892442778927005,
"loss_nats": 1.3912979364395142,
"step": 1770,
"unclipped_grad_norm": 0.7516586780548096
},
{
"bits_per_byte": 1.8758278436421194,
"learning_rate": 0.00027862404984757116,
"loss_nats": 1.300224781036377,
"step": 1780,
"unclipped_grad_norm": 0.6571854948997498
},
{
"bits_per_byte": 1.963898439322455,
"learning_rate": 0.0002783217303843597,
"loss_nats": 1.3612706661224365,
"step": 1790,
"unclipped_grad_norm": 0.6896912455558777
},
{
"bits_per_byte": 1.880059304783948,
"learning_rate": 0.00027801747456544134,
"loss_nats": 1.3031578063964844,
"step": 1800,
"unclipped_grad_norm": 0.6719921827316284
},
{
"bits_per_byte": 1.9405041552735398,
"learning_rate": 0.00027771128758970843,
"loss_nats": 1.3450549840927124,
"step": 1810,
"unclipped_grad_norm": 0.7259154915809631
},
{
"bits_per_byte": 1.9116848504858082,
"learning_rate": 0.00027740317468905165,
"loss_nats": 1.3250789642333984,
"step": 1820,
"unclipped_grad_norm": 0.6911426782608032
},
{
"bits_per_byte": 1.887859061965878,
"learning_rate": 0.0002770931411282704,
"loss_nats": 1.3085641860961914,
"step": 1830,
"unclipped_grad_norm": 0.6866450309753418
},
{
"bits_per_byte": 1.904207044826293,
"learning_rate": 0.0002767811922049827,
"loss_nats": 1.3198957443237305,
"step": 1840,
"unclipped_grad_norm": 0.6746156215667725
},
{
"bits_per_byte": 1.913379911492712,
"learning_rate": 0.00027646733324953503,
"loss_nats": 1.326253890991211,
"step": 1850,
"unclipped_grad_norm": 0.6992620825767517
},
{
"bits_per_byte": 1.8924212457453158,
"learning_rate": 0.00027615156962491097,
"loss_nats": 1.311726450920105,
"step": 1860,
"unclipped_grad_norm": 0.7686212658882141
},
{
"bits_per_byte": 1.9034685513234884,
"learning_rate": 0.0002758339067266396,
"loss_nats": 1.3193838596343994,
"step": 1870,
"unclipped_grad_norm": 0.6957508325576782
},
{
"bits_per_byte": 1.8975736739825146,
"learning_rate": 0.0002755143499827035,
"loss_nats": 1.3152978420257568,
"step": 1880,
"unclipped_grad_norm": 0.686790406703949
},
{
"bits_per_byte": 1.9108474669587572,
"learning_rate": 0.0002751929048534456,
"loss_nats": 1.3244985342025757,
"step": 1890,
"unclipped_grad_norm": 0.6941424608230591
},
{
"bits_per_byte": 1.9178445811091105,
"learning_rate": 0.0002748695768314763,
"loss_nats": 1.3293485641479492,
"step": 1900,
"unclipped_grad_norm": 0.6622544527053833
},
{
"bits_per_byte": 1.857984643615103,
"learning_rate": 0.00027454437144157947,
"loss_nats": 1.2878568172454834,
"step": 1910,
"unclipped_grad_norm": 0.6576761603355408
},
{
"bits_per_byte": 1.8615044405476613,
"learning_rate": 0.00027421729424061787,
"loss_nats": 1.2902965545654297,
"step": 1920,
"unclipped_grad_norm": 0.7134246826171875
},
{
"bits_per_byte": 1.9495869030309065,
"learning_rate": 0.0002738883508174385,
"loss_nats": 1.3513506650924683,
"step": 1930,
"unclipped_grad_norm": 0.68708336353302
},
{
"bits_per_byte": 1.8269496863363057,
"learning_rate": 0.0002735575467927768,
"loss_nats": 1.2663450241088867,
"step": 1940,
"unclipped_grad_norm": 0.671497106552124
},
{
"bits_per_byte": 1.904890503880822,
"learning_rate": 0.0002732248878191609,
"loss_nats": 1.3203694820404053,
"step": 1950,
"unclipped_grad_norm": 0.6914620995521545
},
{
"bits_per_byte": 1.81755221032786,
"learning_rate": 0.0002728903795808148,
"loss_nats": 1.259831190109253,
"step": 1960,
"unclipped_grad_norm": 0.654005765914917
},
{
"bits_per_byte": 1.9009173606805894,
"learning_rate": 0.0002725540277935614,
"loss_nats": 1.3176155090332031,
"step": 1970,
"unclipped_grad_norm": 0.6446248292922974
},
{
"bits_per_byte": 1.9534350148439619,
"learning_rate": 0.00027221583820472456,
"loss_nats": 1.354017972946167,
"step": 1980,
"unclipped_grad_norm": 0.6792307496070862
},
{
"bits_per_byte": 1.9578730271494766,
"learning_rate": 0.0002718758165930314,
"loss_nats": 1.357094168663025,
"step": 1990,
"unclipped_grad_norm": 0.8862876892089844
},
{
"bits_per_byte": 1.9120136813142579,
"learning_rate": 0.0002715339687685131,
"loss_nats": 1.3253068923950195,
"step": 2000,
"unclipped_grad_norm": 0.6821991801261902
},
{
"bits_per_byte": 1.879799782963796,
"learning_rate": 0.00027119030057240576,
"loss_nats": 1.3029779195785522,
"step": 2010,
"unclipped_grad_norm": 0.6441471576690674
},
{
"bits_per_byte": 1.8960643542385427,
"learning_rate": 0.0002708448178770506,
"loss_nats": 1.3142516613006592,
"step": 2020,
"unclipped_grad_norm": 0.6215534806251526
},
{
"bits_per_byte": 1.9880719767624297,
"learning_rate": 0.00027049752658579376,
"loss_nats": 1.3780264854431152,
"step": 2030,
"unclipped_grad_norm": 0.6645570993423462
},
{
"bits_per_byte": 1.9132447331291338,
"learning_rate": 0.00027014843263288514,
"loss_nats": 1.326160192489624,
"step": 2040,
"unclipped_grad_norm": 0.6700881719589233
},
{
"bits_per_byte": 1.943815681215903,
"learning_rate": 0.00026979754198337724,
"loss_nats": 1.3473503589630127,
"step": 2050,
"unclipped_grad_norm": 0.6627214550971985
},
{
"bits_per_byte": 1.9209480080439298,
"learning_rate": 0.00026944486063302305,
"loss_nats": 1.331499695777893,
"step": 2060,
"unclipped_grad_norm": 0.6492555737495422
},
{
"bits_per_byte": 1.8667327131338907,
"learning_rate": 0.0002690903946081738,
"loss_nats": 1.2939205169677734,
"step": 2070,
"unclipped_grad_norm": 0.733633279800415
},
{
"bits_per_byte": 1.8747223391623733,
"learning_rate": 0.00026873414996567596,
"loss_nats": 1.2994585037231445,
"step": 2080,
"unclipped_grad_norm": 0.7179054021835327
},
{
"bits_per_byte": 1.870721334772701,
"learning_rate": 0.00026837613279276736,
"loss_nats": 1.2966852188110352,
"step": 2090,
"unclipped_grad_norm": 0.6603640913963318
},
{
"bits_per_byte": 1.8546041526297437,
"learning_rate": 0.0002680163492069739,
"loss_nats": 1.2855136394500732,
"step": 2100,
"unclipped_grad_norm": 0.6752005815505981
},
{
"bits_per_byte": 1.9160201891487578,
"learning_rate": 0.00026765480535600414,
"loss_nats": 1.3280839920043945,
"step": 2110,
"unclipped_grad_norm": 0.6652648448944092
},
{
"bits_per_byte": 1.871099696604599,
"learning_rate": 0.0002672915074176452,
"loss_nats": 1.2969474792480469,
"step": 2120,
"unclipped_grad_norm": 0.7032200694084167
},
{
"bits_per_byte": 1.8333405616423677,
"learning_rate": 0.00026692646159965624,
"loss_nats": 1.2707748413085938,
"step": 2130,
"unclipped_grad_norm": 0.6664465069770813
},
{
"bits_per_byte": 1.966146252569232,
"learning_rate": 0.0002665596741396633,
"loss_nats": 1.3628287315368652,
"step": 2140,
"unclipped_grad_norm": 0.7021564245223999
},
{
"bits_per_byte": 1.835432558607463,
"learning_rate": 0.000266191151305052,
"loss_nats": 1.2722249031066895,
"step": 2150,
"unclipped_grad_norm": 0.6662127375602722
},
{
"bits_per_byte": 1.8787831935145458,
"learning_rate": 0.00026582089939286075,
"loss_nats": 1.3022732734680176,
"step": 2160,
"unclipped_grad_norm": 0.6593465209007263
},
{
"bits_per_byte": 1.9513302911627028,
"learning_rate": 0.0002654489247296732,
"loss_nats": 1.3525590896606445,
"step": 2170,
"unclipped_grad_norm": 0.721863329410553
},
{
"bits_per_byte": 1.8313610413309367,
"learning_rate": 0.00026507523367151007,
"loss_nats": 1.2694027423858643,
"step": 2180,
"unclipped_grad_norm": 0.661525547504425
},
{
"bits_per_byte": 1.8437549710653651,
"learning_rate": 0.00026469983260372037,
"loss_nats": 1.2779935598373413,
"step": 2190,
"unclipped_grad_norm": 0.6355888247489929
},
{
"bits_per_byte": 1.8767926663134598,
"learning_rate": 0.0002643227279408727,
"loss_nats": 1.3008935451507568,
"step": 2200,
"unclipped_grad_norm": 0.650397777557373
},
{
"bits_per_byte": 1.8339084483555168,
"learning_rate": 0.0002639439261266453,
"loss_nats": 1.2711684703826904,
"step": 2210,
"unclipped_grad_norm": 0.6309477686882019
},
{
"bits_per_byte": 1.8366491638796665,
"learning_rate": 0.000263563433633716,
"loss_nats": 1.2730681896209717,
"step": 2220,
"unclipped_grad_norm": 0.6397845149040222
},
{
"bits_per_byte": 1.9122785345965865,
"learning_rate": 0.00026318125696365156,
"loss_nats": 1.3254904747009277,
"step": 2230,
"unclipped_grad_norm": 0.7579862475395203
},
{
"bits_per_byte": 1.8301518313127203,
"learning_rate": 0.000262797402646797,
"loss_nats": 1.2685645818710327,
"step": 2240,
"unclipped_grad_norm": 0.6486738324165344
},
{
"bits_per_byte": 1.854603636681791,
"learning_rate": 0.00026241187724216334,
"loss_nats": 1.2855132818222046,
"step": 2250,
"unclipped_grad_norm": 0.6761392951011658
},
{
"bits_per_byte": 1.8292511581701518,
"learning_rate": 0.0002620246873373162,
"loss_nats": 1.2679402828216553,
"step": 2260,
"unclipped_grad_norm": 0.6474469900131226
},
{
"bits_per_byte": 1.8217264012469514,
"learning_rate": 0.00026163583954826264,
"loss_nats": 1.26272451877594,
"step": 2270,
"unclipped_grad_norm": 0.6697631478309631
},
{
"bits_per_byte": 1.839495476751446,
"learning_rate": 0.0002612453405193386,
"loss_nats": 1.275041103363037,
"step": 2280,
"unclipped_grad_norm": 0.672606885433197
},
{
"bits_per_byte": 1.8894086256501519,
"learning_rate": 0.000260853196923095,
"loss_nats": 1.309638261795044,
"step": 2290,
"unclipped_grad_norm": 0.7677630186080933
},
{
"bits_per_byte": 1.8035906587308173,
"learning_rate": 0.0002604594154601839,
"loss_nats": 1.2501537799835205,
"step": 2300,
"unclipped_grad_norm": 0.6455734372138977
},
{
"bits_per_byte": 1.834394643309506,
"learning_rate": 0.0002600640028592441,
"loss_nats": 1.2715054750442505,
"step": 2310,
"unclipped_grad_norm": 0.6606747508049011
},
{
"bits_per_byte": 1.8063805612931139,
"learning_rate": 0.0002596669658767859,
"loss_nats": 1.2520875930786133,
"step": 2320,
"unclipped_grad_norm": 0.634936511516571
},
{
"bits_per_byte": 1.8747587994843562,
"learning_rate": 0.000259268311297076,
"loss_nats": 1.2994837760925293,
"step": 2330,
"unclipped_grad_norm": 0.67148357629776
},
{
"bits_per_byte": 1.7920827836136342,
"learning_rate": 0.00025886804593202114,
"loss_nats": 1.242177128791809,
"step": 2340,
"unclipped_grad_norm": 0.6197777390480042
},
{
"bits_per_byte": 1.8459198886744261,
"learning_rate": 0.00025846617662105193,
"loss_nats": 1.2794941663742065,
"step": 2350,
"unclipped_grad_norm": 0.6368024349212646
},
{
"bits_per_byte": 1.8583048753110096,
"learning_rate": 0.00025806271023100637,
"loss_nats": 1.288078784942627,
"step": 2360,
"unclipped_grad_norm": 0.662507951259613
},
{
"bits_per_byte": 1.8678564477746282,
"learning_rate": 0.00025765765365601165,
"loss_nats": 1.2946994304656982,
"step": 2370,
"unclipped_grad_norm": 0.6344826221466064
},
{
"bits_per_byte": 1.8307873072076584,
"learning_rate": 0.00025725101381736707,
"loss_nats": 1.2690050601959229,
"step": 2380,
"unclipped_grad_norm": 0.680084764957428
},
{
"bits_per_byte": 1.888488346485385,
"learning_rate": 0.00025684279766342567,
"loss_nats": 1.3090003728866577,
"step": 2390,
"unclipped_grad_norm": 0.6502870917320251
},
{
"bits_per_byte": 1.8376442554975587,
"learning_rate": 0.00025643301216947504,
"loss_nats": 1.2737579345703125,
"step": 2400,
"unclipped_grad_norm": 0.6415471434593201
},
{
"bits_per_byte": 1.8272317378837208,
"learning_rate": 0.00025602166433761866,
"loss_nats": 1.26654052734375,
"step": 2410,
"unclipped_grad_norm": 0.6551245450973511
},
{
"bits_per_byte": 1.8084292186301916,
"learning_rate": 0.0002556087611966561,
"loss_nats": 1.2535076141357422,
"step": 2420,
"unclipped_grad_norm": 0.6829442381858826
},
{
"bits_per_byte": 1.8282901191171306,
"learning_rate": 0.00025519430980196277,
"loss_nats": 1.2672741413116455,
"step": 2430,
"unclipped_grad_norm": 0.6490575075149536
},
{
"bits_per_byte": 1.8846385148458211,
"learning_rate": 0.0002547783172353695,
"loss_nats": 1.3063318729400635,
"step": 2440,
"unclipped_grad_norm": 0.6281237006187439
},
{
"bits_per_byte": 1.8146233457836665,
"learning_rate": 0.0002543607906050415,
"loss_nats": 1.2578010559082031,
"step": 2450,
"unclipped_grad_norm": 0.634181559085846
},
{
"bits_per_byte": 1.8319019267679002,
"learning_rate": 0.0002539417370453568,
"loss_nats": 1.2697776556015015,
"step": 2460,
"unclipped_grad_norm": 0.6390114426612854
},
{
"bits_per_byte": 1.8154657168075925,
"learning_rate": 0.00025352116371678464,
"loss_nats": 1.2583849430084229,
"step": 2470,
"unclipped_grad_norm": 0.6522298455238342
},
{
"bits_per_byte": 1.8417133650169726,
"learning_rate": 0.0002530990778057627,
"loss_nats": 1.276578426361084,
"step": 2480,
"unclipped_grad_norm": 0.6301186680793762
},
{
"bits_per_byte": 1.7981788806561174,
"learning_rate": 0.00025267548652457475,
"loss_nats": 1.246402621269226,
"step": 2490,
"unclipped_grad_norm": 0.613478422164917
},
{
"bits_per_byte": 1.8422764362158974,
"learning_rate": 0.00025225039711122696,
"loss_nats": 1.2769687175750732,
"step": 2500,
"unclipped_grad_norm": 0.6680112481117249
},
{
"bits_per_byte": 1.7934540012889635,
"learning_rate": 0.00025182381682932464,
"loss_nats": 1.2431275844573975,
"step": 2510,
"unclipped_grad_norm": 0.6324169635772705
},
{
"bits_per_byte": 1.8373082013977726,
"learning_rate": 0.0002513957529679477,
"loss_nats": 1.2735249996185303,
"step": 2520,
"unclipped_grad_norm": 0.6562070250511169
},
{
"bits_per_byte": 1.8465858054985669,
"learning_rate": 0.00025096621284152656,
"loss_nats": 1.2799557447433472,
"step": 2530,
"unclipped_grad_norm": 0.67587810754776
},
{
"bits_per_byte": 1.8180633427662245,
"learning_rate": 0.0002505352037897168,
"loss_nats": 1.2601854801177979,
"step": 2540,
"unclipped_grad_norm": 0.6210066080093384
},
{
"bits_per_byte": 1.8116361791208304,
"learning_rate": 0.0002501027331772739,
"loss_nats": 1.2557305097579956,
"step": 2550,
"unclipped_grad_norm": 0.656908392906189
},
{
"bits_per_byte": 1.8229865181298233,
"learning_rate": 0.00024966880839392746,
"loss_nats": 1.2635979652404785,
"step": 2560,
"unclipped_grad_norm": 0.6474040746688843
},
{
"bits_per_byte": 1.8231736352539618,
"learning_rate": 0.0002492334368542548,
"loss_nats": 1.2637276649475098,
"step": 2570,
"unclipped_grad_norm": 0.6998929977416992
},
{
"bits_per_byte": 1.8511411099719701,
"learning_rate": 0.00024879662599755416,
"loss_nats": 1.2831132411956787,
"step": 2580,
"unclipped_grad_norm": 0.6996825337409973
},
{
"bits_per_byte": 1.805761251767357,
"learning_rate": 0.00024835838328771803,
"loss_nats": 1.251658320426941,
"step": 2590,
"unclipped_grad_norm": 0.6730060577392578
},
{
"bits_per_byte": 1.7987187341971758,
"learning_rate": 0.0002479187162131051,
"loss_nats": 1.246776819229126,
"step": 2600,
"unclipped_grad_norm": 0.6812606453895569
},
{
"bits_per_byte": 1.7378126262379698,
"learning_rate": 0.00024747763228641264,
"loss_nats": 1.2045599222183228,
"step": 2610,
"unclipped_grad_norm": 0.6152759194374084
},
{
"bits_per_byte": 1.8067871282797534,
"learning_rate": 0.00024703513904454795,
"loss_nats": 1.2523694038391113,
"step": 2620,
"unclipped_grad_norm": 0.6381004452705383
},
{
"bits_per_byte": 1.815846142431301,
"learning_rate": 0.00024659124404849956,
"loss_nats": 1.2586486339569092,
"step": 2630,
"unclipped_grad_norm": 0.6937149167060852
},
{
"bits_per_byte": 1.6830253170311609,
"learning_rate": 0.00024614595488320833,
"loss_nats": 1.1665842533111572,
"step": 2640,
"unclipped_grad_norm": 0.5872177481651306
},
{
"bits_per_byte": 1.7705318096166665,
"learning_rate": 0.00024569927915743745,
"loss_nats": 1.2272391319274902,
"step": 2650,
"unclipped_grad_norm": 0.6221135854721069
},
{
"bits_per_byte": 1.813369076310924,
"learning_rate": 0.0002452512245036427,
"loss_nats": 1.2569316625595093,
"step": 2660,
"unclipped_grad_norm": 0.6537755727767944
},
{
"bits_per_byte": 1.8137383230623265,
"learning_rate": 0.00024480179857784177,
"loss_nats": 1.2571876049041748,
"step": 2670,
"unclipped_grad_norm": 0.6585802435874939
},
{
"bits_per_byte": 1.7472491423081613,
"learning_rate": 0.00024435100905948387,
"loss_nats": 1.2111008167266846,
"step": 2680,
"unclipped_grad_norm": 0.6200863122940063
},
{
"bits_per_byte": 1.9002486921340347,
"learning_rate": 0.00024389886365131793,
"loss_nats": 1.3171520233154297,
"step": 2690,
"unclipped_grad_norm": 0.6396089792251587
},
{
"bits_per_byte": 1.8072413344606821,
"learning_rate": 0.00024344537007926163,
"loss_nats": 1.252684235572815,
"step": 2700,
"unclipped_grad_norm": 0.6738898158073425
},
{
"bits_per_byte": 1.7328827435509875,
"learning_rate": 0.00024299053609226878,
"loss_nats": 1.2011427879333496,
"step": 2710,
"unclipped_grad_norm": 0.7000491619110107
},
{
"bits_per_byte": 1.7288494064229527,
"learning_rate": 0.00024253436946219733,
"loss_nats": 1.1983470916748047,
"step": 2720,
"unclipped_grad_norm": 0.6326305270195007
},
{
"bits_per_byte": 1.776559801529408,
"learning_rate": 0.0002420768779836764,
"loss_nats": 1.2314174175262451,
"step": 2730,
"unclipped_grad_norm": 0.647907555103302
},
{
"bits_per_byte": 1.783826928441616,
"learning_rate": 0.00024161806947397305,
"loss_nats": 1.2364546060562134,
"step": 2740,
"unclipped_grad_norm": 0.6690548658370972
},
{
"bits_per_byte": 1.7821005665922551,
"learning_rate": 0.00024115795177285886,
"loss_nats": 1.2352579832077026,
"step": 2750,
"unclipped_grad_norm": 0.664342999458313
},
{
"bits_per_byte": 1.8057218677403093,
"learning_rate": 0.0002406965327424758,
"loss_nats": 1.2516310214996338,
"step": 2760,
"unclipped_grad_norm": 0.6517228484153748
},
{
"bits_per_byte": 1.7048344369870736,
"learning_rate": 0.00024023382026720197,
"loss_nats": 1.1817011833190918,
"step": 2770,
"unclipped_grad_norm": 0.684334397315979
},
{
"bits_per_byte": 1.7173274287283988,
"learning_rate": 0.00023976982225351698,
"loss_nats": 1.19036066532135,
"step": 2780,
"unclipped_grad_norm": 0.6013396382331848
},
{
"bits_per_byte": 1.804812079517245,
"learning_rate": 0.0002393045466298666,
"loss_nats": 1.2510004043579102,
"step": 2790,
"unclipped_grad_norm": 0.6293013095855713
},
{
"bits_per_byte": 1.8557677872454814,
"learning_rate": 0.00023883800134652763,
"loss_nats": 1.2863202095031738,
"step": 2800,
"unclipped_grad_norm": 0.7250765562057495
},
{
"bits_per_byte": 1.7793906359626097,
"learning_rate": 0.0002383701943754715,
"loss_nats": 1.233379602432251,
"step": 2810,
"unclipped_grad_norm": 0.7088303565979004
},
{
"bits_per_byte": 1.7744306563117265,
"learning_rate": 0.00023790113371022884,
"loss_nats": 1.2299416065216064,
"step": 2820,
"unclipped_grad_norm": 0.6359960436820984
},
{
"bits_per_byte": 1.8690950668261421,
"learning_rate": 0.0002374308273657521,
"loss_nats": 1.295557975769043,
"step": 2830,
"unclipped_grad_norm": 0.6622759103775024
},
{
"bits_per_byte": 1.7494551637707785,
"learning_rate": 0.00023695928337827911,
"loss_nats": 1.2126299142837524,
"step": 2840,
"unclipped_grad_norm": 0.6139791011810303
},
{
"bits_per_byte": 1.7661483158114752,
"learning_rate": 0.00023648650980519564,
"loss_nats": 1.22420072555542,
"step": 2850,
"unclipped_grad_norm": 0.6314926743507385
},
{
"bits_per_byte": 1.7415515290677273,
"learning_rate": 0.00023601251472489757,
"loss_nats": 1.2071515321731567,
"step": 2860,
"unclipped_grad_norm": 0.664522111415863
},
{
"bits_per_byte": 1.8281711071227333,
"learning_rate": 0.00023553730623665305,
"loss_nats": 1.2671916484832764,
"step": 2870,
"unclipped_grad_norm": 0.6572005152702332
},
{
"bits_per_byte": 1.8333617155084239,
"learning_rate": 0.00023506089246046403,
"loss_nats": 1.2707895040512085,
"step": 2880,
"unclipped_grad_norm": 0.6237366795539856
},
{
"bits_per_byte": 1.7312996432497951,
"learning_rate": 0.00023458328153692736,
"loss_nats": 1.2000454664230347,
"step": 2890,
"unclipped_grad_norm": 0.6335425972938538
},
{
"bits_per_byte": 1.7034657990515074,
"learning_rate": 0.0002341044816270961,
"loss_nats": 1.1807525157928467,
"step": 2900,
"unclipped_grad_norm": 0.6068143844604492
},
{
"bits_per_byte": 1.7597350827608012,
"learning_rate": 0.00023362450091233953,
"loss_nats": 1.2197554111480713,
"step": 2910,
"unclipped_grad_norm": 0.6452642679214478
},
{
"bits_per_byte": 1.7579383800072375,
"learning_rate": 0.00023314334759420388,
"loss_nats": 1.2185100317001343,
"step": 2920,
"unclipped_grad_norm": 0.6799261569976807
},
{
"bits_per_byte": 1.7601423376780443,
"learning_rate": 0.0002326610298942717,
"loss_nats": 1.2200376987457275,
"step": 2930,
"unclipped_grad_norm": 0.6305575370788574
},
{
"bits_per_byte": 1.77748111259008,
"learning_rate": 0.00023217755605402175,
"loss_nats": 1.2320560216903687,
"step": 2940,
"unclipped_grad_norm": 0.6869999766349792
},
{
"bits_per_byte": 1.7661192507434795,
"learning_rate": 0.00023169293433468802,
"loss_nats": 1.2241805791854858,
"step": 2950,
"unclipped_grad_norm": 0.6630687713623047
},
{
"bits_per_byte": 1.7912226983766693,
"learning_rate": 0.00023120717301711857,
"loss_nats": 1.2415809631347656,
"step": 2960,
"unclipped_grad_norm": 0.6753889918327332
},
{
"bits_per_byte": 1.7573228540997994,
"learning_rate": 0.00023072028040163408,
"loss_nats": 1.218083381652832,
"step": 2970,
"unclipped_grad_norm": 0.6314569115638733
},
{
"bits_per_byte": 1.7556446473926801,
"learning_rate": 0.00023023226480788593,
"loss_nats": 1.2169201374053955,
"step": 2980,
"unclipped_grad_norm": 0.685643196105957
},
{
"bits_per_byte": 1.778294762511312,
"learning_rate": 0.00022974313457471416,
"loss_nats": 1.2326200008392334,
"step": 2990,
"unclipped_grad_norm": 0.7002031803131104
},
{
"bits_per_byte": 1.7854260231293384,
"learning_rate": 0.0002292528980600049,
"loss_nats": 1.2375630140304565,
"step": 3000,
"unclipped_grad_norm": 0.6241077184677124
},
{
"bits_per_byte": 1.7589776711664014,
"learning_rate": 0.00022876156364054758,
"loss_nats": 1.2192304134368896,
"step": 3010,
"unclipped_grad_norm": 0.5821192860603333
},
{
"bits_per_byte": 1.7213342805282006,
"learning_rate": 0.0002282691397118918,
"loss_nats": 1.1931380033493042,
"step": 3020,
"unclipped_grad_norm": 0.6891434192657471
},
{
"bits_per_byte": 1.8083322204151049,
"learning_rate": 0.00022777563468820376,
"loss_nats": 1.2534403800964355,
"step": 3030,
"unclipped_grad_norm": 0.6679258346557617
},
{
"bits_per_byte": 1.7077268412092843,
"learning_rate": 0.00022728105700212282,
"loss_nats": 1.1837060451507568,
"step": 3040,
"unclipped_grad_norm": 0.6403138637542725
},
{
"bits_per_byte": 1.7462220619172086,
"learning_rate": 0.00022678541510461698,
"loss_nats": 1.2103888988494873,
"step": 3050,
"unclipped_grad_norm": 0.6389932632446289
},
{
"bits_per_byte": 1.7619969985849488,
"learning_rate": 0.00022628871746483876,
"loss_nats": 1.2213232517242432,
"step": 3060,
"unclipped_grad_norm": 0.676324188709259
},
{
"bits_per_byte": 1.6793940753408436,
"learning_rate": 0.0002257909725699804,
"loss_nats": 1.164067268371582,
"step": 3070,
"unclipped_grad_norm": 0.5887739062309265
},
{
"bits_per_byte": 1.723725699288448,
"learning_rate": 0.00022529218892512886,
"loss_nats": 1.1947956085205078,
"step": 3080,
"unclipped_grad_norm": 0.6651143431663513
},
{
"bits_per_byte": 1.792202311555984,
"learning_rate": 0.00022479237505312034,
"loss_nats": 1.2422599792480469,
"step": 3090,
"unclipped_grad_norm": 0.7115829586982727
},
{
"bits_per_byte": 1.6949688242029795,
"learning_rate": 0.00022429153949439496,
"loss_nats": 1.1748628616333008,
"step": 3100,
"unclipped_grad_norm": 0.648972749710083
},
{
"bits_per_byte": 1.7872602181007902,
"learning_rate": 0.00022378969080685053,
"loss_nats": 1.2388343811035156,
"step": 3110,
"unclipped_grad_norm": 0.6126834154129028
},
{
"bits_per_byte": 1.7418012478767801,
"learning_rate": 0.0002232868375656965,
"loss_nats": 1.2073246240615845,
"step": 3120,
"unclipped_grad_norm": 0.6380596160888672
},
{
"bits_per_byte": 1.7994572276999805,
"learning_rate": 0.00022278298836330733,
"loss_nats": 1.247288703918457,
"step": 3130,
"unclipped_grad_norm": 0.6736929416656494
},
{
"bits_per_byte": 1.7652089465724625,
"learning_rate": 0.00022227815180907572,
"loss_nats": 1.2235496044158936,
"step": 3140,
"unclipped_grad_norm": 0.6578004956245422
},
{
"bits_per_byte": 1.7230821402089194,
"learning_rate": 0.00022177233652926546,
"loss_nats": 1.1943495273590088,
"step": 3150,
"unclipped_grad_norm": 0.6959549784660339
},
{
"bits_per_byte": 1.745404112429705,
"learning_rate": 0.00022126555116686416,
"loss_nats": 1.2098219394683838,
"step": 3160,
"unclipped_grad_norm": 0.6303315758705139
},
{
"bits_per_byte": 1.6651441088383039,
"learning_rate": 0.00022075780438143532,
"loss_nats": 1.154189944267273,
"step": 3170,
"unclipped_grad_norm": 0.6514579057693481
},
{
"bits_per_byte": 1.748495672561615,
"learning_rate": 0.00022024910484897065,
"loss_nats": 1.2119648456573486,
"step": 3180,
"unclipped_grad_norm": 0.6430810689926147
},
{
"bits_per_byte": 1.7413002624148166,
"learning_rate": 0.00021973946126174158,
"loss_nats": 1.206977367401123,
"step": 3190,
"unclipped_grad_norm": 0.6930623650550842
},
{
"bits_per_byte": 1.7004863716079601,
"learning_rate": 0.00021922888232815086,
"loss_nats": 1.178687334060669,
"step": 3200,
"unclipped_grad_norm": 0.6456758975982666
},
{
"bits_per_byte": 1.6959322710131128,
"learning_rate": 0.0002187173767725837,
"loss_nats": 1.1755306720733643,
"step": 3210,
"unclipped_grad_norm": 0.6182776689529419
},
{
"bits_per_byte": 1.7081593775762043,
"learning_rate": 0.0002182049533352588,
"loss_nats": 1.184005856513977,
"step": 3220,
"unclipped_grad_norm": 0.6259344816207886
},
{
"bits_per_byte": 1.8040880325571125,
"learning_rate": 0.00021769162077207884,
"loss_nats": 1.2504985332489014,
"step": 3230,
"unclipped_grad_norm": 0.6837271451950073
},
{
"bits_per_byte": 1.680291136847744,
"learning_rate": 0.00021717738785448102,
"loss_nats": 1.164689064025879,
"step": 3240,
"unclipped_grad_norm": 0.6078165173530579
},
{
"bits_per_byte": 1.833327662943553,
"learning_rate": 0.00021666226336928708,
"loss_nats": 1.2707659006118774,
"step": 3250,
"unclipped_grad_norm": 0.6908280849456787
},
{
"bits_per_byte": 1.6877314502893708,
"learning_rate": 0.0002161462561185532,
"loss_nats": 1.1698462963104248,
"step": 3260,
"unclipped_grad_norm": 0.6267281174659729
},
{
"bits_per_byte": 1.7156755353668616,
"learning_rate": 0.00021562937491941957,
"loss_nats": 1.1892156600952148,
"step": 3270,
"unclipped_grad_norm": 0.6226681470870972
},
{
"bits_per_byte": 1.77760270432424,
"learning_rate": 0.00021511162860395975,
"loss_nats": 1.232140302658081,
"step": 3280,
"unclipped_grad_norm": 0.62666916847229
},
{
"bits_per_byte": 1.7647146684338828,
"learning_rate": 0.0002145930260190298,
"loss_nats": 1.2232069969177246,
"step": 3290,
"unclipped_grad_norm": 0.6992575526237488
},
{
"bits_per_byte": 1.6858417049216903,
"learning_rate": 0.00021407357602611698,
"loss_nats": 1.1685364246368408,
"step": 3300,
"unclipped_grad_norm": 0.6361157894134521
},
{
"bits_per_byte": 1.7274259060217614,
"learning_rate": 0.0002135532875011885,
"loss_nats": 1.1973603963851929,
"step": 3310,
"unclipped_grad_norm": 0.6828696727752686
},
{
"bits_per_byte": 1.7324201102201664,
"learning_rate": 0.0002130321693345397,
"loss_nats": 1.200822114944458,
"step": 3320,
"unclipped_grad_norm": 0.6315762996673584
},
{
"bits_per_byte": 1.7279007501207935,
"learning_rate": 0.00021251023043064224,
"loss_nats": 1.1976895332336426,
"step": 3330,
"unclipped_grad_norm": 0.6427719593048096
},
{
"bits_per_byte": 1.6987109947031034,
"learning_rate": 0.00021198747970799188,
"loss_nats": 1.1774567365646362,
"step": 3340,
"unclipped_grad_norm": 0.6303005814552307
},
{
"bits_per_byte": 1.7007044456092542,
"learning_rate": 0.00021146392609895618,
"loss_nats": 1.1788384914398193,
"step": 3350,
"unclipped_grad_norm": 0.6034587025642395
},
{
"bits_per_byte": 1.7301495952634756,
"learning_rate": 0.00021093957854962178,
"loss_nats": 1.1992483139038086,
"step": 3360,
"unclipped_grad_norm": 0.6297745108604431
},
{
"bits_per_byte": 1.6889530430584494,
"learning_rate": 0.00021041444601964157,
"loss_nats": 1.170693039894104,
"step": 3370,
"unclipped_grad_norm": 0.6365876197814941
},
{
"bits_per_byte": 1.7185201284121325,
"learning_rate": 0.00020988853748208164,
"loss_nats": 1.1911873817443848,
"step": 3380,
"unclipped_grad_norm": 0.612015962600708
},
{
"bits_per_byte": 1.7554757604295328,
"learning_rate": 0.00020936186192326782,
"loss_nats": 1.2168030738830566,
"step": 3390,
"unclipped_grad_norm": 0.7148704528808594
},
{
"bits_per_byte": 1.805198180568432,
"learning_rate": 0.00020883442834263225,
"loss_nats": 1.2512680292129517,
"step": 3400,
"unclipped_grad_norm": 0.7259010672569275
},
{
"bits_per_byte": 1.8248889902136676,
"learning_rate": 0.0002083062457525596,
"loss_nats": 1.2649166584014893,
"step": 3410,
"unclipped_grad_norm": 0.6750312447547913
},
{
"bits_per_byte": 1.6908183668897072,
"learning_rate": 0.00020777732317823306,
"loss_nats": 1.1719859838485718,
"step": 3420,
"unclipped_grad_norm": 0.6068294644355774
},
{
"bits_per_byte": 1.6181469257849768,
"learning_rate": 0.00020724766965748005,
"loss_nats": 1.1216139793395996,
"step": 3430,
"unclipped_grad_norm": 0.6401124596595764
},
{
"bits_per_byte": 1.7073711810873,
"learning_rate": 0.00020671729424061788,
"loss_nats": 1.1834595203399658,
"step": 3440,
"unclipped_grad_norm": 0.6253624558448792
},
{
"bits_per_byte": 1.7759955264519272,
"learning_rate": 0.00020618620599029906,
"loss_nats": 1.231026291847229,
"step": 3450,
"unclipped_grad_norm": 0.6825960874557495
},
{
"bits_per_byte": 1.7035373438342662,
"learning_rate": 0.00020565441398135649,
"loss_nats": 1.1808021068572998,
"step": 3460,
"unclipped_grad_norm": 0.6307380795478821
},
{
"bits_per_byte": 1.694909318205781,
"learning_rate": 0.00020512192730064827,
"loss_nats": 1.1748216152191162,
"step": 3470,
"unclipped_grad_norm": 0.6624554395675659
},
{
"bits_per_byte": 1.640782594360936,
"learning_rate": 0.00020458875504690273,
"loss_nats": 1.1373038291931152,
"step": 3480,
"unclipped_grad_norm": 0.6500871777534485
},
{
"bits_per_byte": 1.8527976468650806,
"learning_rate": 0.00020405490633056245,
"loss_nats": 1.2842614650726318,
"step": 3490,
"unclipped_grad_norm": 0.6960679888725281
},
{
"bits_per_byte": 1.7002731131208901,
"learning_rate": 0.00020352039027362917,
"loss_nats": 1.178539514541626,
"step": 3500,
"unclipped_grad_norm": 0.7090469002723694
},
{
"bits_per_byte": 1.7481523951904927,
"learning_rate": 0.0002029852160095075,
"loss_nats": 1.2117269039154053,
"step": 3510,
"unclipped_grad_norm": 0.6682069301605225
},
{
"bits_per_byte": 1.7343439081527179,
"learning_rate": 0.000202449392682849,
"loss_nats": 1.202155590057373,
"step": 3520,
"unclipped_grad_norm": 0.6754779815673828
},
{
"bits_per_byte": 1.703965236669613,
"learning_rate": 0.0002019129294493959,
"loss_nats": 1.1810986995697021,
"step": 3530,
"unclipped_grad_norm": 0.625368595123291
},
{
"bits_per_byte": 1.7204380789345546,
"learning_rate": 0.00020137583547582478,
"loss_nats": 1.192516803741455,
"step": 3540,
"unclipped_grad_norm": 0.6671978235244751
},
{
"bits_per_byte": 1.7171148581719324,
"learning_rate": 0.00020083811993958973,
"loss_nats": 1.1902133226394653,
"step": 3550,
"unclipped_grad_norm": 0.6819398999214172
},
{
"bits_per_byte": 1.6516749435706812,
"learning_rate": 0.00020029979202876563,
"loss_nats": 1.1448538303375244,
"step": 3560,
"unclipped_grad_norm": 0.6344290375709534
},
{
"bits_per_byte": 1.6805387918649863,
"learning_rate": 0.00019976086094189118,
"loss_nats": 1.164860725402832,
"step": 3570,
"unclipped_grad_norm": 0.6333562135696411
},
{
"bits_per_byte": 1.6952945593437136,
"learning_rate": 0.00019922133588781175,
"loss_nats": 1.17508864402771,
"step": 3580,
"unclipped_grad_norm": 0.655963659286499
},
{
"bits_per_byte": 1.7344106374212527,
"learning_rate": 0.0001986812260855219,
"loss_nats": 1.2022018432617188,
"step": 3590,
"unclipped_grad_norm": 0.7093035578727722
},
{
"bits_per_byte": 1.7542852965202604,
"learning_rate": 0.00019814054076400787,
"loss_nats": 1.2159779071807861,
"step": 3600,
"unclipped_grad_norm": 0.668968915939331
},
{
"bits_per_byte": 1.6751485116216442,
"learning_rate": 0.00019759928916209004,
"loss_nats": 1.1611244678497314,
"step": 3610,
"unclipped_grad_norm": 0.6502929925918579
},
{
"bits_per_byte": 1.6413291552253781,
"learning_rate": 0.00019705748052826493,
"loss_nats": 1.1376826763153076,
"step": 3620,
"unclipped_grad_norm": 0.6599135398864746
},
{
"bits_per_byte": 1.6914022479893867,
"learning_rate": 0.00019651512412054723,
"loss_nats": 1.1723906993865967,
"step": 3630,
"unclipped_grad_norm": 0.6377425789833069
},
{
"bits_per_byte": 1.6712699588793862,
"learning_rate": 0.00019597222920631144,
"loss_nats": 1.1584360599517822,
"step": 3640,
"unclipped_grad_norm": 0.6370565891265869
},
{
"bits_per_byte": 1.7179845744373456,
"learning_rate": 0.0001954288050621338,
"loss_nats": 1.1908161640167236,
"step": 3650,
"unclipped_grad_norm": 0.6391654014587402
},
{
"bits_per_byte": 1.6386034021918534,
"learning_rate": 0.00019488486097363363,
"loss_nats": 1.1357933282852173,
"step": 3660,
"unclipped_grad_norm": 0.651786208152771
},
{
"bits_per_byte": 1.7137386667528447,
"learning_rate": 0.00019434040623531452,
"loss_nats": 1.187873125076294,
"step": 3670,
"unclipped_grad_norm": 0.64645916223526
},
{
"bits_per_byte": 1.651732729741371,
"learning_rate": 0.0001937954501504058,
"loss_nats": 1.1448938846588135,
"step": 3680,
"unclipped_grad_norm": 0.6200698018074036
},
{
"bits_per_byte": 1.6382305438047828,
"learning_rate": 0.00019325000203070333,
"loss_nats": 1.1355348825454712,
"step": 3690,
"unclipped_grad_norm": 0.6166419386863708
},
{
"bits_per_byte": 1.6489116983192684,
"learning_rate": 0.0001927040711964106,
"loss_nats": 1.142938494682312,
"step": 3700,
"unclipped_grad_norm": 0.6672006845474243
},
{
"bits_per_byte": 1.7161945790071655,
"learning_rate": 0.00019215766697597923,
"loss_nats": 1.189575433731079,
"step": 3710,
"unclipped_grad_norm": 0.6522265672683716
},
{
"bits_per_byte": 1.6845286173823526,
"learning_rate": 0.00019161079870594978,
"loss_nats": 1.1676262617111206,
"step": 3720,
"unclipped_grad_norm": 0.6373200416564941
},
{
"bits_per_byte": 1.6158851819434799,
"learning_rate": 0.0001910634757307922,
"loss_nats": 1.1200462579727173,
"step": 3730,
"unclipped_grad_norm": 0.6535916328430176
},
{
"bits_per_byte": 1.6679503497524322,
"learning_rate": 0.0001905157074027459,
"loss_nats": 1.156135082244873,
"step": 3740,
"unclipped_grad_norm": 0.6719768643379211
},
{
"bits_per_byte": 1.6882193650698685,
"learning_rate": 0.00018996750308166037,
"loss_nats": 1.1701844930648804,
"step": 3750,
"unclipped_grad_norm": 0.6530092358589172
},
{
"bits_per_byte": 1.7695160800806709,
"learning_rate": 0.00018941887213483482,
"loss_nats": 1.2265350818634033,
"step": 3760,
"unclipped_grad_norm": 0.6655569672584534
},
{
"bits_per_byte": 1.7330591977507726,
"learning_rate": 0.00018886982393685845,
"loss_nats": 1.2012650966644287,
"step": 3770,
"unclipped_grad_norm": 0.6482370495796204
},
{
"bits_per_byte": 1.555590988256044,
"learning_rate": 0.00018832036786945008,
"loss_nats": 1.0782535076141357,
"step": 3780,
"unclipped_grad_norm": 0.6347804665565491
},
{
"bits_per_byte": 1.661505987841952,
"learning_rate": 0.0001877705133212979,
"loss_nats": 1.1516681909561157,
"step": 3790,
"unclipped_grad_norm": 0.6471394896507263
},
{
"bits_per_byte": 1.671339095905033,
"learning_rate": 0.00018722026968789906,
"loss_nats": 1.1584839820861816,
"step": 3800,
"unclipped_grad_norm": 0.6512831449508667
},
{
"bits_per_byte": 1.6910716973344282,
"learning_rate": 0.00018666964637139907,
"loss_nats": 1.17216157913208,
"step": 3810,
"unclipped_grad_norm": 0.66470867395401
},
{
"bits_per_byte": 1.662058912064476,
"learning_rate": 0.00018611865278043115,
"loss_nats": 1.1520514488220215,
"step": 3820,
"unclipped_grad_norm": 0.6363425254821777
},
{
"bits_per_byte": 1.7124487968713735,
"learning_rate": 0.00018556729832995555,
"loss_nats": 1.186979055404663,
"step": 3830,
"unclipped_grad_norm": 0.6397905945777893
},
{
"bits_per_byte": 1.6528970522877124,
"learning_rate": 0.00018501559244109864,
"loss_nats": 1.1457009315490723,
"step": 3840,
"unclipped_grad_norm": 0.6409087777137756
},
{
"bits_per_byte": 1.5958772362900533,
"learning_rate": 0.00018446354454099178,
"loss_nats": 1.106177806854248,
"step": 3850,
"unclipped_grad_norm": 0.6576514840126038
},
{
"bits_per_byte": 1.6597844415068153,
"learning_rate": 0.00018391116406261048,
"loss_nats": 1.1504749059677124,
"step": 3860,
"unclipped_grad_norm": 0.6520916819572449
},
{
"bits_per_byte": 1.7387071080051073,
"learning_rate": 0.00018335846044461305,
"loss_nats": 1.2051799297332764,
"step": 3870,
"unclipped_grad_norm": 0.6458169221878052
},
{
"bits_per_byte": 1.631997548572213,
"learning_rate": 0.0001828054431311793,
"loss_nats": 1.1312144994735718,
"step": 3880,
"unclipped_grad_norm": 0.6073523163795471
},
{
"bits_per_byte": 1.6031751480967658,
"learning_rate": 0.00018225212157184934,
"loss_nats": 1.111236333847046,
"step": 3890,
"unclipped_grad_norm": 0.632803201675415
},
{
"bits_per_byte": 1.6778582702686387,
"learning_rate": 0.00018169850522136192,
"loss_nats": 1.1630027294158936,
"step": 3900,
"unclipped_grad_norm": 0.7183825969696045
},
{
"bits_per_byte": 1.69528785202033,
"learning_rate": 0.000181144603539493,
"loss_nats": 1.1750839948654175,
"step": 3910,
"unclipped_grad_norm": 0.6837812662124634
},
{
"bits_per_byte": 1.6365686754494955,
"learning_rate": 0.00018059042599089412,
"loss_nats": 1.134382963180542,
"step": 3920,
"unclipped_grad_norm": 0.6382866501808167
},
{
"bits_per_byte": 1.6653448125918606,
"learning_rate": 0.00018003598204493048,
"loss_nats": 1.1543290615081787,
"step": 3930,
"unclipped_grad_norm": 0.6285881400108337
},
{
"bits_per_byte": 1.6987961261152804,
"learning_rate": 0.00017948128117551936,
"loss_nats": 1.1775157451629639,
"step": 3940,
"unclipped_grad_norm": 0.666064977645874
},
{
"bits_per_byte": 1.7229675997634448,
"learning_rate": 0.00017892633286096816,
"loss_nats": 1.194270133972168,
"step": 3950,
"unclipped_grad_norm": 0.6532164812088013
},
{
"bits_per_byte": 1.7031080751377128,
"learning_rate": 0.00017837114658381247,
"loss_nats": 1.180504560470581,
"step": 3960,
"unclipped_grad_norm": 0.651078999042511
},
{
"bits_per_byte": 1.631284336519085,
"learning_rate": 0.00017781573183065386,
"loss_nats": 1.1307201385498047,
"step": 3970,
"unclipped_grad_norm": 0.6460844278335571
},
{
"bits_per_byte": 1.6744433827531067,
"learning_rate": 0.0001772600980919981,
"loss_nats": 1.1606357097625732,
"step": 3980,
"unclipped_grad_norm": 0.6715673208236694
},
{
"bits_per_byte": 1.6984889651008395,
"learning_rate": 0.0001767042548620927,
"loss_nats": 1.1773028373718262,
"step": 3990,
"unclipped_grad_norm": 0.7048564553260803
},
{
"bits_per_byte": 1.5840455178412955,
"learning_rate": 0.00017614821163876486,
"loss_nats": 1.0979766845703125,
"step": 4000,
"unclipped_grad_norm": 0.6564667224884033
},
{
"bits_per_byte": 1.67799809216379,
"learning_rate": 0.00017559197792325906,
"loss_nats": 1.1630996465682983,
"step": 4010,
"unclipped_grad_norm": 0.6846939921379089
},
{
"bits_per_byte": 1.645402220345762,
"learning_rate": 0.0001750355632200748,
"loss_nats": 1.1405059099197388,
"step": 4020,
"unclipped_grad_norm": 0.6446395516395569
},
{
"bits_per_byte": 1.678588508604202,
"learning_rate": 0.00017447897703680418,
"loss_nats": 1.1635088920593262,
"step": 4030,
"unclipped_grad_norm": 0.744084894657135
},
{
"bits_per_byte": 1.6238756678852169,
"learning_rate": 0.0001739222288839694,
"loss_nats": 1.1255848407745361,
"step": 4040,
"unclipped_grad_norm": 0.6811469793319702
},
{
"bits_per_byte": 1.6502275375810198,
"learning_rate": 0.00017336532827486026,
"loss_nats": 1.143850564956665,
"step": 4050,
"unclipped_grad_norm": 0.6219719052314758
},
{
"bits_per_byte": 1.655425885185999,
"learning_rate": 0.00017280828472537163,
"loss_nats": 1.147453784942627,
"step": 4060,
"unclipped_grad_norm": 0.6728122234344482
},
{
"bits_per_byte": 1.6906052803852882,
"learning_rate": 0.0001722511077538408,
"loss_nats": 1.1718382835388184,
"step": 4070,
"unclipped_grad_norm": 0.6714834570884705
},
{
"bits_per_byte": 1.7592499197027172,
"learning_rate": 0.00017169380688088497,
"loss_nats": 1.2194191217422485,
"step": 4080,
"unclipped_grad_norm": 0.7062271237373352
},
{
"bits_per_byte": 1.656125338627058,
"learning_rate": 0.0001711363916292384,
"loss_nats": 1.14793860912323,
"step": 4090,
"unclipped_grad_norm": 0.7272074222564697
},
{
"bits_per_byte": 1.6359631245358075,
"learning_rate": 0.00017057887152358974,
"loss_nats": 1.1339632272720337,
"step": 4100,
"unclipped_grad_norm": 0.6811342239379883
},
{
"bits_per_byte": 1.6284930580955814,
"learning_rate": 0.0001700212560904194,
"loss_nats": 1.1287853717803955,
"step": 4110,
"unclipped_grad_norm": 0.6341036558151245
},
{
"bits_per_byte": 1.6773249520683131,
"learning_rate": 0.00016946355485783656,
"loss_nats": 1.1626330614089966,
"step": 4120,
"unclipped_grad_norm": 0.6589056849479675
},
{
"bits_per_byte": 1.6258371300183072,
"learning_rate": 0.0001689057773554166,
"loss_nats": 1.1269444227218628,
"step": 4130,
"unclipped_grad_norm": 0.6739165186882019
},
{
"bits_per_byte": 1.688376729195408,
"learning_rate": 0.00016834793311403806,
"loss_nats": 1.1702935695648193,
"step": 4140,
"unclipped_grad_norm": 0.6829613447189331
},
{
"bits_per_byte": 1.6211284170203344,
"learning_rate": 0.00016779003166571981,
"loss_nats": 1.123680591583252,
"step": 4150,
"unclipped_grad_norm": 0.6710932850837708
},
{
"bits_per_byte": 1.6548134549662765,
"learning_rate": 0.00016723208254345838,
"loss_nats": 1.1470292806625366,
"step": 4160,
"unclipped_grad_norm": 0.7125453948974609
},
{
"bits_per_byte": 1.6949535177470527,
"learning_rate": 0.00016667409528106481,
"loss_nats": 1.1748522520065308,
"step": 4170,
"unclipped_grad_norm": 0.6789602041244507
},
{
"bits_per_byte": 1.7346840898361244,
"learning_rate": 0.00016611607941300184,
"loss_nats": 1.2023913860321045,
"step": 4180,
"unclipped_grad_norm": 0.7571533918380737
},
{
"bits_per_byte": 1.6432809863300202,
"learning_rate": 0.0001655580444742211,
"loss_nats": 1.1390355825424194,
"step": 4190,
"unclipped_grad_norm": 0.6785401701927185
},
{
"bits_per_byte": 1.6844624040617706,
"learning_rate": 0.00016499999999999997,
"loss_nats": 1.1675803661346436,
"step": 4200,
"unclipped_grad_norm": 0.7054091095924377
},
{
"bits_per_byte": 1.672555185229284,
"learning_rate": 0.0001644419555257789,
"loss_nats": 1.1593269109725952,
"step": 4210,
"unclipped_grad_norm": 0.6699897050857544
},
{
"bits_per_byte": 1.6983758005165717,
"learning_rate": 0.0001638839205869981,
"loss_nats": 1.1772243976593018,
"step": 4220,
"unclipped_grad_norm": 0.6806776523590088
},
{
"bits_per_byte": 1.6692897506373519,
"learning_rate": 0.0001633259047189352,
"loss_nats": 1.1570634841918945,
"step": 4230,
"unclipped_grad_norm": 0.6995632648468018
},
{
"bits_per_byte": 1.611002250520183,
"learning_rate": 0.0001627679174565416,
"loss_nats": 1.1166616678237915,
"step": 4240,
"unclipped_grad_norm": 0.6468433737754822
},
{
"bits_per_byte": 1.646616933808806,
"learning_rate": 0.00016220996833428016,
"loss_nats": 1.141347885131836,
"step": 4250,
"unclipped_grad_norm": 0.6655133962631226
},
{
"bits_per_byte": 1.61702336312689,
"learning_rate": 0.00016165206688596191,
"loss_nats": 1.1208351850509644,
"step": 4260,
"unclipped_grad_norm": 0.6660183072090149
},
{
"bits_per_byte": 1.6213850151354217,
"learning_rate": 0.00016109422264458335,
"loss_nats": 1.1238584518432617,
"step": 4270,
"unclipped_grad_norm": 0.6792552471160889
},
{
"bits_per_byte": 1.6175876382043708,
"learning_rate": 0.00016053644514216336,
"loss_nats": 1.1212263107299805,
"step": 4280,
"unclipped_grad_norm": 0.6596150398254395
},
{
"bits_per_byte": 1.673857609844268,
"learning_rate": 0.00015997874390958058,
"loss_nats": 1.1602296829223633,
"step": 4290,
"unclipped_grad_norm": 0.6736516952514648
},
{
"bits_per_byte": 1.5981075073064421,
"learning_rate": 0.00015942112847641018,
"loss_nats": 1.1077237129211426,
"step": 4300,
"unclipped_grad_norm": 0.65042644739151
},
{
"bits_per_byte": 1.6804373221009774,
"learning_rate": 0.00015886360837076156,
"loss_nats": 1.164790391921997,
"step": 4310,
"unclipped_grad_norm": 0.7301046252250671
},
{
"bits_per_byte": 1.6671354959526443,
"learning_rate": 0.00015830619311911495,
"loss_nats": 1.1555702686309814,
"step": 4320,
"unclipped_grad_norm": 0.6921650767326355
},
{
"bits_per_byte": 1.6208601240849883,
"learning_rate": 0.00015774889224615915,
"loss_nats": 1.1234946250915527,
"step": 4330,
"unclipped_grad_norm": 0.6875792145729065
},
{
"bits_per_byte": 1.651699193124453,
"learning_rate": 0.00015719171527462834,
"loss_nats": 1.144870638847351,
"step": 4340,
"unclipped_grad_norm": 0.6566780209541321
},
{
"bits_per_byte": 1.638596522885819,
"learning_rate": 0.0001566346717251397,
"loss_nats": 1.1357885599136353,
"step": 4350,
"unclipped_grad_norm": 0.6942516565322876
},
{
"bits_per_byte": 1.6271663839268258,
"learning_rate": 0.00015607777111603056,
"loss_nats": 1.1278657913208008,
"step": 4360,
"unclipped_grad_norm": 0.6855849623680115
},
{
"bits_per_byte": 1.6394584999319433,
"learning_rate": 0.00015552102296319576,
"loss_nats": 1.1363860368728638,
"step": 4370,
"unclipped_grad_norm": 0.7280991673469543
},
{
"bits_per_byte": 1.69825386481711,
"learning_rate": 0.00015496443677992516,
"loss_nats": 1.1771398782730103,
"step": 4380,
"unclipped_grad_norm": 0.6875903010368347
},
{
"bits_per_byte": 1.6491371675745496,
"learning_rate": 0.0001544080220767409,
"loss_nats": 1.143094778060913,
"step": 4390,
"unclipped_grad_norm": 0.7626568078994751
},
{
"bits_per_byte": 1.5979716410122606,
"learning_rate": 0.00015385178836123511,
"loss_nats": 1.1076295375823975,
"step": 4400,
"unclipped_grad_norm": 0.6716449856758118
},
{
"bits_per_byte": 1.6724951632841327,
"learning_rate": 0.00015329574513790726,
"loss_nats": 1.159285306930542,
"step": 4410,
"unclipped_grad_norm": 0.7023071646690369
},
{
"bits_per_byte": 1.6178187828871304,
"learning_rate": 0.00015273990190800186,
"loss_nats": 1.1213865280151367,
"step": 4420,
"unclipped_grad_norm": 0.6506969332695007
},
{
"bits_per_byte": 1.6647528483075908,
"learning_rate": 0.0001521842681693461,
"loss_nats": 1.153918743133545,
"step": 4430,
"unclipped_grad_norm": 0.6848753094673157
},
{
"bits_per_byte": 1.604941065955825,
"learning_rate": 0.00015162885341618752,
"loss_nats": 1.1124603748321533,
"step": 4440,
"unclipped_grad_norm": 0.6678289771080017
},
{
"bits_per_byte": 1.6073562183218915,
"learning_rate": 0.00015107366713903178,
"loss_nats": 1.114134430885315,
"step": 4450,
"unclipped_grad_norm": 0.6393097639083862
},
{
"bits_per_byte": 1.581368607880616,
"learning_rate": 0.0001505187188244806,
"loss_nats": 1.0961211919784546,
"step": 4460,
"unclipped_grad_norm": 0.6358886957168579
},
{
"bits_per_byte": 1.6384745871863573,
"learning_rate": 0.00014996401795506952,
"loss_nats": 1.1357040405273438,
"step": 4470,
"unclipped_grad_norm": 0.7018935680389404
},
{
"bits_per_byte": 1.6524677835911588,
"learning_rate": 0.00014940957400910585,
"loss_nats": 1.1454033851623535,
"step": 4480,
"unclipped_grad_norm": 0.7296347618103027
},
{
"bits_per_byte": 1.6287001252072204,
"learning_rate": 0.00014885539646050699,
"loss_nats": 1.1289288997650146,
"step": 4490,
"unclipped_grad_norm": 0.6741621494293213
},
{
"bits_per_byte": 1.644477125666771,
"learning_rate": 0.00014830149477863805,
"loss_nats": 1.1398646831512451,
"step": 4500,
"unclipped_grad_norm": 0.656035304069519
},
{
"bits_per_byte": 1.6005383100937371,
"learning_rate": 0.00014774787842815066,
"loss_nats": 1.1094086170196533,
"step": 4510,
"unclipped_grad_norm": 0.6776733994483948
},
{
"bits_per_byte": 1.6433757487706455,
"learning_rate": 0.00014719455686882066,
"loss_nats": 1.139101266860962,
"step": 4520,
"unclipped_grad_norm": 0.6692283153533936
},
{
"bits_per_byte": 1.6564529655769518,
"learning_rate": 0.00014664153955538695,
"loss_nats": 1.1481657028198242,
"step": 4530,
"unclipped_grad_norm": 0.6900064945220947
},
{
"bits_per_byte": 1.7115691056122104,
"learning_rate": 0.00014608883593738946,
"loss_nats": 1.1863692998886108,
"step": 4540,
"unclipped_grad_norm": 0.7291412353515625
},
{
"bits_per_byte": 1.568905197155239,
"learning_rate": 0.00014553645545900817,
"loss_nats": 1.087482213973999,
"step": 4550,
"unclipped_grad_norm": 0.6766284108161926
},
{
"bits_per_byte": 1.6280190739098037,
"learning_rate": 0.0001449844075589013,
"loss_nats": 1.1284568309783936,
"step": 4560,
"unclipped_grad_norm": 0.7198469042778015
},
{
"bits_per_byte": 1.6109731854521872,
"learning_rate": 0.0001444327016700444,
"loss_nats": 1.1166415214538574,
"step": 4570,
"unclipped_grad_norm": 0.6274889707565308
},
{
"bits_per_byte": 1.5632356170868957,
"learning_rate": 0.0001438813472195688,
"loss_nats": 1.083552360534668,
"step": 4580,
"unclipped_grad_norm": 0.7059792280197144
},
{
"bits_per_byte": 1.6166451732776426,
"learning_rate": 0.00014333035362860093,
"loss_nats": 1.1205730438232422,
"step": 4590,
"unclipped_grad_norm": 0.7005748748779297
},
{
"bits_per_byte": 1.6071720249028174,
"learning_rate": 0.00014277973031210088,
"loss_nats": 1.114006757736206,
"step": 4600,
"unclipped_grad_norm": 0.6841927170753479
},
{
"bits_per_byte": 1.66910400937442,
"learning_rate": 0.00014222948667870207,
"loss_nats": 1.1569347381591797,
"step": 4610,
"unclipped_grad_norm": 0.7007519006729126
},
{
"bits_per_byte": 1.6744798430750896,
"learning_rate": 0.00014167963213054991,
"loss_nats": 1.160660982131958,
"step": 4620,
"unclipped_grad_norm": 0.7218911051750183
},
{
"bits_per_byte": 1.648005693714523,
"learning_rate": 0.00014113017606314152,
"loss_nats": 1.1423105001449585,
"step": 4630,
"unclipped_grad_norm": 0.7027076482772827
},
{
"bits_per_byte": 1.692054922149411,
"learning_rate": 0.00014058112786516518,
"loss_nats": 1.172843098640442,
"step": 4640,
"unclipped_grad_norm": 0.6847496628761292
},
{
"bits_per_byte": 1.6069941948418252,
"learning_rate": 0.0001400324969183396,
"loss_nats": 1.1138834953308105,
"step": 4650,
"unclipped_grad_norm": 0.695062518119812
},
{
"bits_per_byte": 1.5888770264519845,
"learning_rate": 0.00013948429259725406,
"loss_nats": 1.1013256311416626,
"step": 4660,
"unclipped_grad_norm": 0.7018181681632996
},
{
"bits_per_byte": 1.721095740591454,
"learning_rate": 0.00013893652426920776,
"loss_nats": 1.1929726600646973,
"step": 4670,
"unclipped_grad_norm": 0.7332665920257568
},
{
"bits_per_byte": 1.7135371030860336,
"learning_rate": 0.00013838920129405016,
"loss_nats": 1.1877334117889404,
"step": 4680,
"unclipped_grad_norm": 0.7111047506332397
},
{
"bits_per_byte": 1.6797499074454787,
"learning_rate": 0.0001378423330240207,
"loss_nats": 1.1643139123916626,
"step": 4690,
"unclipped_grad_norm": 0.7494509816169739
},
{
"bits_per_byte": 1.6073170062774949,
"learning_rate": 0.00013729592880358936,
"loss_nats": 1.1141072511672974,
"step": 4700,
"unclipped_grad_norm": 0.6868770718574524
},
{
"bits_per_byte": 1.6766973873753146,
"learning_rate": 0.00013674999796929659,
"loss_nats": 1.1621980667114258,
"step": 4710,
"unclipped_grad_norm": 0.7650524377822876
},
{
"bits_per_byte": 1.6150646527162134,
"learning_rate": 0.00013620454984959417,
"loss_nats": 1.1194775104522705,
"step": 4720,
"unclipped_grad_norm": 0.7059918642044067
},
{
"bits_per_byte": 1.5217877982289587,
"learning_rate": 0.00013565959376468542,
"loss_nats": 1.0548229217529297,
"step": 4730,
"unclipped_grad_norm": 0.6677312850952148
},
{
"bits_per_byte": 1.5772625220912664,
"learning_rate": 0.00013511513902636634,
"loss_nats": 1.0932750701904297,
"step": 4740,
"unclipped_grad_norm": 0.6796184182167053
},
{
"bits_per_byte": 1.6299009080755444,
"learning_rate": 0.0001345711949378662,
"loss_nats": 1.1297612190246582,
"step": 4750,
"unclipped_grad_norm": 0.7213169932365417
},
{
"bits_per_byte": 1.6096504668844014,
"learning_rate": 0.00013402777079368853,
"loss_nats": 1.1157246828079224,
"step": 4760,
"unclipped_grad_norm": 0.7036519050598145
},
{
"bits_per_byte": 1.5962356481344513,
"learning_rate": 0.0001334848758794528,
"loss_nats": 1.1064262390136719,
"step": 4770,
"unclipped_grad_norm": 0.7376033663749695
},
{
"bits_per_byte": 1.626534347684905,
"learning_rate": 0.00013294251947173504,
"loss_nats": 1.1274276971817017,
"step": 4780,
"unclipped_grad_norm": 0.7321033477783203
},
{
"bits_per_byte": 1.5732460392630163,
"learning_rate": 0.00013240071083790996,
"loss_nats": 1.0904910564422607,
"step": 4790,
"unclipped_grad_norm": 0.6640080809593201
},
{
"bits_per_byte": 1.5751923669228307,
"learning_rate": 0.0001318594592359921,
"loss_nats": 1.091840147972107,
"step": 4800,
"unclipped_grad_norm": 0.7126999497413635
},
{
"bits_per_byte": 1.637988564215019,
"learning_rate": 0.0001313187739144781,
"loss_nats": 1.1353671550750732,
"step": 4810,
"unclipped_grad_norm": 0.7521406412124634
},
{
"bits_per_byte": 1.57020504203046,
"learning_rate": 0.0001307786641121882,
"loss_nats": 1.0883831977844238,
"step": 4820,
"unclipped_grad_norm": 0.7331681847572327
},
{
"bits_per_byte": 1.6098945102659756,
"learning_rate": 0.00013023913905810876,
"loss_nats": 1.115893840789795,
"step": 4830,
"unclipped_grad_norm": 0.7234310507774353
},
{
"bits_per_byte": 1.6131886659606018,
"learning_rate": 0.00012970020797123432,
"loss_nats": 1.1181771755218506,
"step": 4840,
"unclipped_grad_norm": 0.7273361682891846
},
{
"bits_per_byte": 1.6277132887565695,
"learning_rate": 0.00012916188006041027,
"loss_nats": 1.1282448768615723,
"step": 4850,
"unclipped_grad_norm": 0.6802273988723755
},
{
"bits_per_byte": 1.5401256203598566,
"learning_rate": 0.00012862416452417516,
"loss_nats": 1.0675337314605713,
"step": 4860,
"unclipped_grad_norm": 0.6796969771385193
},
{
"bits_per_byte": 1.619305056955887,
"learning_rate": 0.00012808707055060405,
"loss_nats": 1.1224167346954346,
"step": 4870,
"unclipped_grad_norm": 0.7333647608757019
},
{
"bits_per_byte": 1.4835698135542255,
"learning_rate": 0.000127550607317151,
"loss_nats": 1.028332233428955,
"step": 4880,
"unclipped_grad_norm": 0.7043541073799133
},
{
"bits_per_byte": 1.5411697270332447,
"learning_rate": 0.00012701478399049248,
"loss_nats": 1.068257451057434,
"step": 4890,
"unclipped_grad_norm": 0.741724967956543
},
{
"bits_per_byte": 1.588994318619873,
"learning_rate": 0.0001264796097263708,
"loss_nats": 1.1014069318771362,
"step": 4900,
"unclipped_grad_norm": 0.6961487531661987
},
{
"bits_per_byte": 1.693890149016768,
"learning_rate": 0.0001259450936694375,
"loss_nats": 1.1741151809692383,
"step": 4910,
"unclipped_grad_norm": 0.7309879064559937
},
{
"bits_per_byte": 1.6231887691776707,
"learning_rate": 0.00012541124495309727,
"loss_nats": 1.1251087188720703,
"step": 4920,
"unclipped_grad_norm": 0.7229281067848206
},
{
"bits_per_byte": 1.5846615596966862,
"learning_rate": 0.00012487807269935167,
"loss_nats": 1.0984036922454834,
"step": 4930,
"unclipped_grad_norm": 0.6784905791282654
},
{
"bits_per_byte": 1.5777433855830787,
"learning_rate": 0.00012434558601864349,
"loss_nats": 1.0936083793640137,
"step": 4940,
"unclipped_grad_norm": 0.6933380365371704
},
{
"bits_per_byte": 1.5189858568811019,
"learning_rate": 0.0001238137940097009,
"loss_nats": 1.0528807640075684,
"step": 4950,
"unclipped_grad_norm": 0.6876996159553528
},
{
"bits_per_byte": 1.5293708572708018,
"learning_rate": 0.0001232827057593821,
"loss_nats": 1.0600790977478027,
"step": 4960,
"unclipped_grad_norm": 0.7548285126686096
},
{
"bits_per_byte": 1.613208615948102,
"learning_rate": 0.00012275233034251992,
"loss_nats": 1.1181910037994385,
"step": 4970,
"unclipped_grad_norm": 0.7192531824111938
},
{
"bits_per_byte": 1.6464662770066503,
"learning_rate": 0.0001222226768217669,
"loss_nats": 1.1412434577941895,
"step": 4980,
"unclipped_grad_norm": 0.7251392602920532
},
{
"bits_per_byte": 1.5932895853251714,
"learning_rate": 0.00012169375424744034,
"loss_nats": 1.104384183883667,
"step": 4990,
"unclipped_grad_norm": 0.770077109336853
},
{
"bits_per_byte": 1.5394951319617936,
"learning_rate": 0.00012116557165736772,
"loss_nats": 1.0670967102050781,
"step": 5000,
"unclipped_grad_norm": 0.689707338809967
},
{
"bits_per_byte": 1.5463551759394092,
"learning_rate": 0.00012063813807673214,
"loss_nats": 1.0718517303466797,
"step": 5010,
"unclipped_grad_norm": 0.6723923087120056
},
{
"bits_per_byte": 1.5950135394174203,
"learning_rate": 0.00012011146251791832,
"loss_nats": 1.105579137802124,
"step": 5020,
"unclipped_grad_norm": 0.6902953386306763
},
{
"bits_per_byte": 1.575558174021216,
"learning_rate": 0.0001195855539803584,
"loss_nats": 1.0920937061309814,
"step": 5030,
"unclipped_grad_norm": 0.7085601687431335
},
{
"bits_per_byte": 1.5476398863413545,
"learning_rate": 0.00011906042145037817,
"loss_nats": 1.072742223739624,
"step": 5040,
"unclipped_grad_norm": 0.7076513767242432
},
{
"bits_per_byte": 1.517178147237883,
"learning_rate": 0.0001185360739010438,
"loss_nats": 1.0516277551651,
"step": 5050,
"unclipped_grad_norm": 0.6860785484313965
},
{
"bits_per_byte": 1.5405944450661087,
"learning_rate": 0.00011801252029200808,
"loss_nats": 1.0678586959838867,
"step": 5060,
"unclipped_grad_norm": 0.6969667673110962
},
{
"bits_per_byte": 1.603204729112714,
"learning_rate": 0.00011748976956935777,
"loss_nats": 1.1112568378448486,
"step": 5070,
"unclipped_grad_norm": 0.6925143003463745
},
{
"bits_per_byte": 1.568753508457178,
"learning_rate": 0.00011696783066546024,
"loss_nats": 1.0873770713806152,
"step": 5080,
"unclipped_grad_norm": 0.7227545380592346
},
{
"bits_per_byte": 1.5571398640097143,
"learning_rate": 0.00011644671249881147,
"loss_nats": 1.07932710647583,
"step": 5090,
"unclipped_grad_norm": 0.7346438765525818
},
{
"bits_per_byte": 1.5756090808858714,
"learning_rate": 0.00011592642397388293,
"loss_nats": 1.0921289920806885,
"step": 5100,
"unclipped_grad_norm": 0.687656581401825
},
{
"bits_per_byte": 1.4874313400140482,
"learning_rate": 0.00011540697398097015,
"loss_nats": 1.0310088396072388,
"step": 5110,
"unclipped_grad_norm": 0.7022747993469238
},
{
"bits_per_byte": 1.5848280389027214,
"learning_rate": 0.00011488837139604016,
"loss_nats": 1.0985190868377686,
"step": 5120,
"unclipped_grad_norm": 0.6745842695236206
},
{
"bits_per_byte": 1.5742394110543998,
"learning_rate": 0.00011437062508058039,
"loss_nats": 1.091179609298706,
"step": 5130,
"unclipped_grad_norm": 0.6994429230690002
},
{
"bits_per_byte": 1.5480249554966363,
"learning_rate": 0.00011385374388144673,
"loss_nats": 1.0730091333389282,
"step": 5140,
"unclipped_grad_norm": 0.7030988335609436
},
{
"bits_per_byte": 1.639304919424723,
"learning_rate": 0.00011333773663071286,
"loss_nats": 1.136279582977295,
"step": 5150,
"unclipped_grad_norm": 0.7419465184211731
},
{
"bits_per_byte": 1.5116879450743894,
"learning_rate": 0.00011282261214551897,
"loss_nats": 1.0478222370147705,
"step": 5160,
"unclipped_grad_norm": 0.7094154953956604
},
{
"bits_per_byte": 1.6830882626813766,
"learning_rate": 0.00011230837922792112,
"loss_nats": 1.1666278839111328,
"step": 5170,
"unclipped_grad_norm": 0.7866556644439697
},
{
"bits_per_byte": 1.5735232752962072,
"learning_rate": 0.0001117950466647412,
"loss_nats": 1.0906832218170166,
"step": 5180,
"unclipped_grad_norm": 0.7095000743865967
},
{
"bits_per_byte": 1.51639820591622,
"learning_rate": 0.00011128262322741627,
"loss_nats": 1.0510871410369873,
"step": 5190,
"unclipped_grad_norm": 0.701707124710083
},
{
"bits_per_byte": 1.549430569702138,
"learning_rate": 0.00011077111767184913,
"loss_nats": 1.0739834308624268,
"step": 5200,
"unclipped_grad_norm": 0.7002435922622681
},
{
"bits_per_byte": 1.5581800150821326,
"learning_rate": 0.00011026053873825837,
"loss_nats": 1.0800480842590332,
"step": 5210,
"unclipped_grad_norm": 0.6994377374649048
},
{
"bits_per_byte": 1.6434304392536199,
"learning_rate": 0.0001097508951510293,
"loss_nats": 1.139139175415039,
"step": 5220,
"unclipped_grad_norm": 0.764930009841919
},
{
"bits_per_byte": 1.5617875231666307,
"learning_rate": 0.0001092421956185646,
"loss_nats": 1.0825486183166504,
"step": 5230,
"unclipped_grad_norm": 0.7441672086715698
},
{
"bits_per_byte": 1.5466518460121477,
"learning_rate": 0.00010873444883313581,
"loss_nats": 1.0720573663711548,
"step": 5240,
"unclipped_grad_norm": 0.688214123249054
},
{
"bits_per_byte": 1.5248055778036484,
"learning_rate": 0.00010822766347073447,
"loss_nats": 1.0569146871566772,
"step": 5250,
"unclipped_grad_norm": 0.6841322779655457
},
{
"bits_per_byte": 1.5447307998020101,
"learning_rate": 0.00010772184819092424,
"loss_nats": 1.0707257986068726,
"step": 5260,
"unclipped_grad_norm": 0.694692850112915
},
{
"bits_per_byte": 1.539800573149726,
"learning_rate": 0.0001072170116366926,
"loss_nats": 1.0673084259033203,
"step": 5270,
"unclipped_grad_norm": 0.6789148449897766
},
{
"bits_per_byte": 1.5548948024853513,
"learning_rate": 0.00010671316243430345,
"loss_nats": 1.0777709484100342,
"step": 5280,
"unclipped_grad_norm": 0.7349528074264526
},
{
"bits_per_byte": 1.5569278094012005,
"learning_rate": 0.00010621030919314944,
"loss_nats": 1.079180121421814,
"step": 5290,
"unclipped_grad_norm": 0.7407549023628235
},
{
"bits_per_byte": 1.5483455311578447,
"learning_rate": 0.000105708460505605,
"loss_nats": 1.0732313394546509,
"step": 5300,
"unclipped_grad_norm": 0.7291215658187866
},
{
"bits_per_byte": 1.5444855525418797,
"learning_rate": 0.00010520762494687964,
"loss_nats": 1.0705558061599731,
"step": 5310,
"unclipped_grad_norm": 0.7253612279891968
},
{
"bits_per_byte": 1.5505493168460005,
"learning_rate": 0.00010470781107487111,
"loss_nats": 1.0747588872909546,
"step": 5320,
"unclipped_grad_norm": 0.7087648510932922
},
{
"bits_per_byte": 1.5517671259967603,
"learning_rate": 0.00010420902743001955,
"loss_nats": 1.0756030082702637,
"step": 5330,
"unclipped_grad_norm": 0.7454737424850464
},
{
"bits_per_byte": 1.5001372462771416,
"learning_rate": 0.00010371128253516117,
"loss_nats": 1.039815902709961,
"step": 5340,
"unclipped_grad_norm": 0.7009119391441345
},
{
"bits_per_byte": 1.5857225206698589,
"learning_rate": 0.000103214584895383,
"loss_nats": 1.0991390943527222,
"step": 5350,
"unclipped_grad_norm": 0.7474352121353149
},
{
"bits_per_byte": 1.4946740453898335,
"learning_rate": 0.00010271894299787715,
"loss_nats": 1.0360291004180908,
"step": 5360,
"unclipped_grad_norm": 0.7168655395507812
},
{
"bits_per_byte": 1.4766953230339372,
"learning_rate": 0.00010222436531179622,
"loss_nats": 1.0235671997070312,
"step": 5370,
"unclipped_grad_norm": 0.7242471575737
},
{
"bits_per_byte": 1.5813711876203789,
"learning_rate": 0.0001017308602881082,
"loss_nats": 1.0961229801177979,
"step": 5380,
"unclipped_grad_norm": 0.7494375109672546
},
{
"bits_per_byte": 1.5460944502407012,
"learning_rate": 0.00010123843635945242,
"loss_nats": 1.0716710090637207,
"step": 5390,
"unclipped_grad_norm": 0.7461474537849426
},
{
"bits_per_byte": 1.5453937929210861,
"learning_rate": 0.00010074710193999505,
"loss_nats": 1.0711853504180908,
"step": 5400,
"unclipped_grad_norm": 0.7291334271430969
},
{
"bits_per_byte": 1.5672311180317404,
"learning_rate": 0.0001002568654252858,
"loss_nats": 1.0863218307495117,
"step": 5410,
"unclipped_grad_norm": 0.720815896987915
},
{
"bits_per_byte": 1.541436472124733,
"learning_rate": 9.976773519211404e-05,
"loss_nats": 1.0684423446655273,
"step": 5420,
"unclipped_grad_norm": 0.743694543838501
},
{
"bits_per_byte": 1.5782952779096975,
"learning_rate": 9.92797195983659e-05,
"loss_nats": 1.0939909219741821,
"step": 5430,
"unclipped_grad_norm": 0.7555011510848999
},
{
"bits_per_byte": 1.508865365808429,
"learning_rate": 9.879282698288142e-05,
"loss_nats": 1.045865774154663,
"step": 5440,
"unclipped_grad_norm": 0.7259441018104553
},
{
"bits_per_byte": 1.5083157092562716,
"learning_rate": 9.830706566531197e-05,
"loss_nats": 1.0454847812652588,
"step": 5450,
"unclipped_grad_norm": 0.7505731582641602
},
{
"bits_per_byte": 1.5045496331676778,
"learning_rate": 9.782244394597826e-05,
"loss_nats": 1.0428743362426758,
"step": 5460,
"unclipped_grad_norm": 0.7536665201187134
},
{
"bits_per_byte": 1.5449808625763646,
"learning_rate": 9.733897010572829e-05,
"loss_nats": 1.0708991289138794,
"step": 5470,
"unclipped_grad_norm": 0.7403784394264221
},
{
"bits_per_byte": 1.550094938682421,
"learning_rate": 9.685665240579612e-05,
"loss_nats": 1.0744439363479614,
"step": 5480,
"unclipped_grad_norm": 0.725517213344574
},
{
"bits_per_byte": 1.5697909078071826,
"learning_rate": 9.637549908766041e-05,
"loss_nats": 1.0880961418151855,
"step": 5490,
"unclipped_grad_norm": 0.776546061038971
},
{
"bits_per_byte": 1.546138821764624,
"learning_rate": 9.589551837290389e-05,
"loss_nats": 1.0717017650604248,
"step": 5500,
"unclipped_grad_norm": 0.7528209686279297
},
{
"bits_per_byte": 1.6319261757721049,
"learning_rate": 9.541671846307257e-05,
"loss_nats": 1.1311650276184082,
"step": 5510,
"unclipped_grad_norm": 0.8170714974403381
},
{
"bits_per_byte": 1.5554193495704827,
"learning_rate": 9.493910753953597e-05,
"loss_nats": 1.078134536743164,
"step": 5520,
"unclipped_grad_norm": 0.7434027194976807
},
{
"bits_per_byte": 1.5251368163892103,
"learning_rate": 9.446269376334689e-05,
"loss_nats": 1.057144284248352,
"step": 5530,
"unclipped_grad_norm": 0.7136013507843018
},
{
"bits_per_byte": 1.586380354309409,
"learning_rate": 9.39874852751024e-05,
"loss_nats": 1.099595069885254,
"step": 5540,
"unclipped_grad_norm": 0.7905043959617615
},
{
"bits_per_byte": 1.5182255215816374,
"learning_rate": 9.351349019480433e-05,
"loss_nats": 1.0523537397384644,
"step": 5550,
"unclipped_grad_norm": 0.7156078815460205
},
{
"bits_per_byte": 1.5143791295950908,
"learning_rate": 9.304071662172086e-05,
"loss_nats": 1.0496876239776611,
"step": 5560,
"unclipped_grad_norm": 0.7718254923820496
},
{
"bits_per_byte": 1.4761380992451418,
"learning_rate": 9.256917263424787e-05,
"loss_nats": 1.0231809616088867,
"step": 5570,
"unclipped_grad_norm": 0.7351405620574951
},
{
"bits_per_byte": 1.5698151573609542,
"learning_rate": 9.209886628977114e-05,
"loss_nats": 1.0881129503250122,
"step": 5580,
"unclipped_grad_norm": 0.7265479564666748
},
{
"bits_per_byte": 1.5069491351125157,
"learning_rate": 9.162980562452843e-05,
"loss_nats": 1.0445375442504883,
"step": 5590,
"unclipped_grad_norm": 0.7370763421058655
},
{
"bits_per_byte": 1.5604274843636077,
"learning_rate": 9.116199865347238e-05,
"loss_nats": 1.0816059112548828,
"step": 5600,
"unclipped_grad_norm": 0.7772327661514282
},
{
"bits_per_byte": 1.5487797873512732,
"learning_rate": 9.069545337013333e-05,
"loss_nats": 1.0735323429107666,
"step": 5610,
"unclipped_grad_norm": 0.6945621371269226
},
{
"bits_per_byte": 1.5643851491252627,
"learning_rate": 9.0230177746483e-05,
"loss_nats": 1.0843491554260254,
"step": 5620,
"unclipped_grad_norm": 0.772422194480896
},
{
"bits_per_byte": 1.5390314667350675,
"learning_rate": 8.976617973279797e-05,
"loss_nats": 1.0667753219604492,
"step": 5630,
"unclipped_grad_norm": 0.7249912023544312
},
{
"bits_per_byte": 1.5236104703628028,
"learning_rate": 8.930346725752419e-05,
"loss_nats": 1.0560863018035889,
"step": 5640,
"unclipped_grad_norm": 0.7168114185333252
},
{
"bits_per_byte": 1.5290047062071148,
"learning_rate": 8.88420482271411e-05,
"loss_nats": 1.0598253011703491,
"step": 5650,
"unclipped_grad_norm": 0.7599905729293823
},
{
"bits_per_byte": 1.6052637054088437,
"learning_rate": 8.838193052602691e-05,
"loss_nats": 1.1126840114593506,
"step": 5660,
"unclipped_grad_norm": 0.8106476068496704
},
{
"bits_per_byte": 1.5110031101586539,
"learning_rate": 8.792312201632352e-05,
"loss_nats": 1.0473475456237793,
"step": 5670,
"unclipped_grad_norm": 0.7452697157859802
},
{
"bits_per_byte": 1.532641279359609,
"learning_rate": 8.746563053780261e-05,
"loss_nats": 1.0623459815979004,
"step": 5680,
"unclipped_grad_norm": 0.7387374043464661
},
{
"bits_per_byte": 1.6103865526300942,
"learning_rate": 8.700946390773114e-05,
"loss_nats": 1.1162348985671997,
"step": 5690,
"unclipped_grad_norm": 0.7787995338439941
},
{
"bits_per_byte": 1.5250291552497701,
"learning_rate": 8.655462992073833e-05,
"loss_nats": 1.0570696592330933,
"step": 5700,
"unclipped_grad_norm": 0.7521193027496338
},
{
"bits_per_byte": 1.5451155249919901,
"learning_rate": 8.610113634868199e-05,
"loss_nats": 1.0709924697875977,
"step": 5710,
"unclipped_grad_norm": 0.7714623808860779
},
{
"bits_per_byte": 1.5521905472831843,
"learning_rate": 8.564899094051613e-05,
"loss_nats": 1.0758965015411377,
"step": 5720,
"unclipped_grad_norm": 0.7254540324211121
},
{
"bits_per_byte": 1.554917504195265,
"learning_rate": 8.519820142215816e-05,
"loss_nats": 1.0777866840362549,
"step": 5730,
"unclipped_grad_norm": 0.7474860548973083
},
{
"bits_per_byte": 1.5468128217733552,
"learning_rate": 8.474877549635728e-05,
"loss_nats": 1.0721689462661743,
"step": 5740,
"unclipped_grad_norm": 0.7887082099914551
},
{
"bits_per_byte": 1.541234736475271,
"learning_rate": 8.430072084256247e-05,
"loss_nats": 1.0683025121688843,
"step": 5750,
"unclipped_grad_norm": 0.7614914774894714
},
{
"bits_per_byte": 1.5622339901282707,
"learning_rate": 8.385404511679161e-05,
"loss_nats": 1.0828580856323242,
"step": 5760,
"unclipped_grad_norm": 0.7596094608306885
},
{
"bits_per_byte": 1.5113494832174914,
"learning_rate": 8.340875595150034e-05,
"loss_nats": 1.0475876331329346,
"step": 5770,
"unclipped_grad_norm": 0.7298161387443542
},
{
"bits_per_byte": 1.4968841944360716,
"learning_rate": 8.296486095545203e-05,
"loss_nats": 1.037561058998108,
"step": 5780,
"unclipped_grad_norm": 0.7402821779251099
},
{
"bits_per_byte": 1.524952966935438,
"learning_rate": 8.252236771358729e-05,
"loss_nats": 1.0570168495178223,
"step": 5790,
"unclipped_grad_norm": 0.7289884686470032
},
{
"bits_per_byte": 1.4886130328081264,
"learning_rate": 8.208128378689484e-05,
"loss_nats": 1.0318279266357422,
"step": 5800,
"unclipped_grad_norm": 0.7239735722541809
},
{
"bits_per_byte": 1.5780909625204724,
"learning_rate": 8.16416167122819e-05,
"loss_nats": 1.0938493013381958,
"step": 5810,
"unclipped_grad_norm": 0.7893304824829102
},
{
"bits_per_byte": 1.5071924905634866,
"learning_rate": 8.120337400244578e-05,
"loss_nats": 1.0447062253952026,
"step": 5820,
"unclipped_grad_norm": 0.7217836976051331
},
{
"bits_per_byte": 1.516913465938205,
"learning_rate": 8.076656314574521e-05,
"loss_nats": 1.0514442920684814,
"step": 5830,
"unclipped_grad_norm": 0.7600290775299072
},
{
"bits_per_byte": 1.5193552756151552,
"learning_rate": 8.03311916060725e-05,
"loss_nats": 1.0531368255615234,
"step": 5840,
"unclipped_grad_norm": 0.7521220445632935
},
{
"bits_per_byte": 1.4707125625583728,
"learning_rate": 7.989726682272606e-05,
"loss_nats": 1.0194202661514282,
"step": 5850,
"unclipped_grad_norm": 0.7758342027664185
},
{
"bits_per_byte": 1.5340661556220072,
"learning_rate": 7.946479621028318e-05,
"loss_nats": 1.0633336305618286,
"step": 5860,
"unclipped_grad_norm": 0.7845423817634583
},
{
"bits_per_byte": 1.5391652692374387,
"learning_rate": 7.903378715847343e-05,
"loss_nats": 1.0668680667877197,
"step": 5870,
"unclipped_grad_norm": 0.7568844556808472
},
{
"bits_per_byte": 1.550266749350633,
"learning_rate": 7.860424703205224e-05,
"loss_nats": 1.0745630264282227,
"step": 5880,
"unclipped_grad_norm": 0.7594192028045654
},
{
"bits_per_byte": 1.5444004211297027,
"learning_rate": 7.817618317067534e-05,
"loss_nats": 1.0704967975616455,
"step": 5890,
"unclipped_grad_norm": 0.722252607345581
},
{
"bits_per_byte": 1.5452520792167752,
"learning_rate": 7.774960288877295e-05,
"loss_nats": 1.071087121963501,
"step": 5900,
"unclipped_grad_norm": 0.7907101511955261
},
{
"bits_per_byte": 1.554658154357764,
"learning_rate": 7.732451347542522e-05,
"loss_nats": 1.0776069164276123,
"step": 5910,
"unclipped_grad_norm": 0.7964528799057007
},
{
"bits_per_byte": 1.4817756905404247,
"learning_rate": 7.690092219423722e-05,
"loss_nats": 1.0270886421203613,
"step": 5920,
"unclipped_grad_norm": 0.7341460585594177
},
{
"bits_per_byte": 1.5817024262059407,
"learning_rate": 7.647883628321536e-05,
"loss_nats": 1.0963525772094727,
"step": 5930,
"unclipped_grad_norm": 0.8211336731910706
},
{
"bits_per_byte": 1.4638614616786019,
"learning_rate": 7.605826295464316e-05,
"loss_nats": 1.0146714448928833,
"step": 5940,
"unclipped_grad_norm": 0.7324587106704712
},
{
"bits_per_byte": 1.5159049596735457,
"learning_rate": 7.563920939495851e-05,
"loss_nats": 1.0507452487945557,
"step": 5950,
"unclipped_grad_norm": 0.7514658570289612
},
{
"bits_per_byte": 1.553970911684916,
"learning_rate": 7.522168276463047e-05,
"loss_nats": 1.0771305561065674,
"step": 5960,
"unclipped_grad_norm": 0.8067389130592346
},
{
"bits_per_byte": 1.5228774803048255,
"learning_rate": 7.480569019803719e-05,
"loss_nats": 1.0555782318115234,
"step": 5970,
"unclipped_grad_norm": 0.8027852177619934
},
{
"bits_per_byte": 1.564259601790133,
"learning_rate": 7.439123880334388e-05,
"loss_nats": 1.0842621326446533,
"step": 5980,
"unclipped_grad_norm": 0.7799409031867981
},
{
"bits_per_byte": 1.5324249251848234,
"learning_rate": 7.39783356623813e-05,
"loss_nats": 1.0621960163116455,
"step": 5990,
"unclipped_grad_norm": 0.7881770133972168
},
{
"bits_per_byte": 1.480203081180935,
"learning_rate": 7.356698783052497e-05,
"loss_nats": 1.025998592376709,
"step": 6000,
"unclipped_grad_norm": 0.7544389367103577
},
{
"bits_per_byte": 1.5204526969103107,
"learning_rate": 7.315720233657431e-05,
"loss_nats": 1.053897500038147,
"step": 6010,
"unclipped_grad_norm": 0.7601525783538818
},
{
"bits_per_byte": 1.4842316027947455,
"learning_rate": 7.274898618263286e-05,
"loss_nats": 1.0287909507751465,
"step": 6020,
"unclipped_grad_norm": 0.7522627711296082
},
{
"bits_per_byte": 1.524829999340071,
"learning_rate": 7.234234634398831e-05,
"loss_nats": 1.0569316148757935,
"step": 6030,
"unclipped_grad_norm": 0.7871755361557007
},
{
"bits_per_byte": 1.5740971814021363,
"learning_rate": 7.193728976899362e-05,
"loss_nats": 1.0910810232162476,
"step": 6040,
"unclipped_grad_norm": 0.8342390656471252
},
{
"bits_per_byte": 1.475741851217554,
"learning_rate": 7.153382337894797e-05,
"loss_nats": 1.0229063034057617,
"step": 6050,
"unclipped_grad_norm": 0.7723444104194641
},
{
"bits_per_byte": 1.4753325325085003,
"learning_rate": 7.113195406797887e-05,
"loss_nats": 1.0226225852966309,
"step": 6060,
"unclipped_grad_norm": 0.769443154335022
},
{
"bits_per_byte": 1.5574803896584226,
"learning_rate": 7.073168870292397e-05,
"loss_nats": 1.0795631408691406,
"step": 6070,
"unclipped_grad_norm": 0.8035310506820679
},
{
"bits_per_byte": 1.4788967009649812,
"learning_rate": 7.033303412321404e-05,
"loss_nats": 1.0250930786132812,
"step": 6080,
"unclipped_grad_norm": 0.7756995558738708
},
{
"bits_per_byte": 1.4841172343319218,
"learning_rate": 6.993599714075586e-05,
"loss_nats": 1.0287116765975952,
"step": 6090,
"unclipped_grad_norm": 0.7475231289863586
},
{
"bits_per_byte": 1.4853364193438883,
"learning_rate": 6.954058453981609e-05,
"loss_nats": 1.0295567512512207,
"step": 6100,
"unclipped_grad_norm": 0.7613393664360046
},
{
"bits_per_byte": 1.5560739155396666,
"learning_rate": 6.914680307690504e-05,
"loss_nats": 1.0785882472991943,
"step": 6110,
"unclipped_grad_norm": 0.8045461177825928
},
{
"bits_per_byte": 1.548306147130797,
"learning_rate": 6.875465948066139e-05,
"loss_nats": 1.0732040405273438,
"step": 6120,
"unclipped_grad_norm": 0.8037658333778381
},
{
"bits_per_byte": 1.482889794152714,
"learning_rate": 6.836416045173733e-05,
"loss_nats": 1.0278608798980713,
"step": 6130,
"unclipped_grad_norm": 0.7671685814857483
},
{
"bits_per_byte": 1.4915240111566304,
"learning_rate": 6.797531266268377e-05,
"loss_nats": 1.0338456630706787,
"step": 6140,
"unclipped_grad_norm": 0.7935377359390259
},
{
"bits_per_byte": 1.5420002312542613,
"learning_rate": 6.758812275783663e-05,
"loss_nats": 1.0688331127166748,
"step": 6150,
"unclipped_grad_norm": 0.7919352650642395
},
{
"bits_per_byte": 1.4144471484587233,
"learning_rate": 6.720259735320298e-05,
"loss_nats": 0.9804200530052185,
"step": 6160,
"unclipped_grad_norm": 0.7595162987709045
},
{
"bits_per_byte": 1.5291687776560379,
"learning_rate": 6.681874303634842e-05,
"loss_nats": 1.0599390268325806,
"step": 6170,
"unclipped_grad_norm": 0.7883366942405701
},
{
"bits_per_byte": 1.4831051164315943,
"learning_rate": 6.643656636628402e-05,
"loss_nats": 1.0280101299285889,
"step": 6180,
"unclipped_grad_norm": 0.7718276977539062
},
{
"bits_per_byte": 1.4948415564917739,
"learning_rate": 6.605607387335466e-05,
"loss_nats": 1.0361452102661133,
"step": 6190,
"unclipped_grad_norm": 0.7770188450813293
},
{
"bits_per_byte": 1.5074203675758797,
"learning_rate": 6.567727205912724e-05,
"loss_nats": 1.0448641777038574,
"step": 6200,
"unclipped_grad_norm": 0.7592265009880066
},
{
"bits_per_byte": 1.5232448352470684,
"learning_rate": 6.530016739627959e-05,
"loss_nats": 1.055832862854004,
"step": 6210,
"unclipped_grad_norm": 0.8102302551269531
},
{
"bits_per_byte": 1.5164056011702072,
"learning_rate": 6.492476632848993e-05,
"loss_nats": 1.051092267036438,
"step": 6220,
"unclipped_grad_norm": 0.787604033946991
},
{
"bits_per_byte": 1.4586058438508853,
"learning_rate": 6.455107527032678e-05,
"loss_nats": 1.011028528213501,
"step": 6230,
"unclipped_grad_norm": 0.7935925126075745
},
{
"bits_per_byte": 1.4934343944424144,
"learning_rate": 6.417910060713926e-05,
"loss_nats": 1.0351698398590088,
"step": 6240,
"unclipped_grad_norm": 0.8212590217590332
},
{
"bits_per_byte": 1.5503186881111937,
"learning_rate": 6.380884869494799e-05,
"loss_nats": 1.074599027633667,
"step": 6250,
"unclipped_grad_norm": 0.8379690647125244
},
{
"bits_per_byte": 1.403218659157867,
"learning_rate": 6.34403258603367e-05,
"loss_nats": 0.9726370573043823,
"step": 6260,
"unclipped_grad_norm": 0.7395487427711487
},
{
"bits_per_byte": 1.5229197880369378,
"learning_rate": 6.30735384003437e-05,
"loss_nats": 1.055607557296753,
"step": 6270,
"unclipped_grad_norm": 0.8249105215072632
},
{
"bits_per_byte": 1.509112332895068,
"learning_rate": 6.270849258235484e-05,
"loss_nats": 1.046036958694458,
"step": 6280,
"unclipped_grad_norm": 0.7966778874397278
},
{
"bits_per_byte": 1.4545608119025921,
"learning_rate": 6.234519464399581e-05,
"loss_nats": 1.0082247257232666,
"step": 6290,
"unclipped_grad_norm": 0.7541455030441284
},
{
"bits_per_byte": 1.5209793077872527,
"learning_rate": 6.198365079302614e-05,
"loss_nats": 1.0542625188827515,
"step": 6300,
"unclipped_grad_norm": 0.8022943139076233
},
{
"bits_per_byte": 1.4739374092447013,
"learning_rate": 6.16238672072326e-05,
"loss_nats": 1.021655559539795,
"step": 6310,
"unclipped_grad_norm": 0.7839627861976624
},
{
"bits_per_byte": 1.4204840834693098,
"learning_rate": 6.126585003432406e-05,
"loss_nats": 0.98460453748703,
"step": 6320,
"unclipped_grad_norm": 0.7670372128486633
},
{
"bits_per_byte": 1.5195884840897251,
"learning_rate": 6.090960539182614e-05,
"loss_nats": 1.0532984733581543,
"step": 6330,
"unclipped_grad_norm": 0.7843371033668518
},
{
"bits_per_byte": 1.4664117924082465,
"learning_rate": 6.0555139366976935e-05,
"loss_nats": 1.0164391994476318,
"step": 6340,
"unclipped_grad_norm": 0.7673349380493164
},
{
"bits_per_byte": 1.577301734135663,
"learning_rate": 6.020245801662274e-05,
"loss_nats": 1.0933022499084473,
"step": 6350,
"unclipped_grad_norm": 0.8766481280326843
},
{
"bits_per_byte": 1.4611757806027283,
"learning_rate": 5.985156736711483e-05,
"loss_nats": 1.0128098726272583,
"step": 6360,
"unclipped_grad_norm": 0.8388718962669373
},
{
"bits_per_byte": 1.4358727471032122,
"learning_rate": 5.950247341420618e-05,
"loss_nats": 0.9952711462974548,
"step": 6370,
"unclipped_grad_norm": 0.775425136089325
},
{
"bits_per_byte": 1.4795478272811478,
"learning_rate": 5.915518212294935e-05,
"loss_nats": 1.0255444049835205,
"step": 6380,
"unclipped_grad_norm": 0.7803093194961548
},
{
"bits_per_byte": 1.5296866174177859,
"learning_rate": 5.880969942759425e-05,
"loss_nats": 1.060297966003418,
"step": 6390,
"unclipped_grad_norm": 0.8152122497558594
},
{
"bits_per_byte": 1.5162291469704219,
"learning_rate": 5.846603123148688e-05,
"loss_nats": 1.0509699583053589,
"step": 6400,
"unclipped_grad_norm": 0.792400598526001
},
{
"bits_per_byte": 1.5084371290077807,
"learning_rate": 5.812418340696858e-05,
"loss_nats": 1.0455689430236816,
"step": 6410,
"unclipped_grad_norm": 0.8240766525268555
},
{
"bits_per_byte": 1.4750114408993396,
"learning_rate": 5.778416179527541e-05,
"loss_nats": 1.0224000215530396,
"step": 6420,
"unclipped_grad_norm": 0.853355348110199
},
{
"bits_per_byte": 1.5300785658791023,
"learning_rate": 5.7445972206438635e-05,
"loss_nats": 1.0605696439743042,
"step": 6430,
"unclipped_grad_norm": 0.8168259859085083
},
{
"bits_per_byte": 1.4651398087224652,
"learning_rate": 5.710962041918516e-05,
"loss_nats": 1.0155575275421143,
"step": 6440,
"unclipped_grad_norm": 0.8302746415138245
},
{
"bits_per_byte": 1.53347470728569,
"learning_rate": 5.677511218083908e-05,
"loss_nats": 1.0629236698150635,
"step": 6450,
"unclipped_grad_norm": 0.8165877461433411
},
{
"bits_per_byte": 1.585776179256928,
"learning_rate": 5.644245320722316e-05,
"loss_nats": 1.099176287651062,
"step": 6460,
"unclipped_grad_norm": 0.8728512525558472
},
{
"bits_per_byte": 1.5696127337808887,
"learning_rate": 5.61116491825615e-05,
"loss_nats": 1.087972640991211,
"step": 6470,
"unclipped_grad_norm": 0.8505071401596069
},
{
"bits_per_byte": 1.4561444281517368,
"learning_rate": 5.5782705759382104e-05,
"loss_nats": 1.0093224048614502,
"step": 6480,
"unclipped_grad_norm": 0.7639529705047607
},
{
"bits_per_byte": 1.5541174409034513,
"learning_rate": 5.5455628558420546e-05,
"loss_nats": 1.0772321224212646,
"step": 6490,
"unclipped_grad_norm": 0.8572817444801331
},
{
"bits_per_byte": 1.5173276001614828,
"learning_rate": 5.513042316852368e-05,
"loss_nats": 1.0517313480377197,
"step": 6500,
"unclipped_grad_norm": 0.825986921787262
},
{
"bits_per_byte": 1.4589312350263177,
"learning_rate": 5.480709514655443e-05,
"loss_nats": 1.011254072189331,
"step": 6510,
"unclipped_grad_norm": 0.7764277458190918
},
{
"bits_per_byte": 1.4310413244838487,
"learning_rate": 5.448565001729653e-05,
"loss_nats": 0.9919222593307495,
"step": 6520,
"unclipped_grad_norm": 0.7579286098480225
},
{
"bits_per_byte": 1.4675702675444584,
"learning_rate": 5.41660932733604e-05,
"loss_nats": 1.017242193222046,
"step": 6530,
"unclipped_grad_norm": 0.8082018494606018
},
{
"bits_per_byte": 1.5128018767040279,
"learning_rate": 5.384843037508902e-05,
"loss_nats": 1.048594355583191,
"step": 6540,
"unclipped_grad_norm": 0.8025459051132202
},
{
"bits_per_byte": 1.4392037930764485,
"learning_rate": 5.353266675046498e-05,
"loss_nats": 0.9975800514221191,
"step": 6550,
"unclipped_grad_norm": 0.790289044380188
},
{
"bits_per_byte": 1.527554032547087,
"learning_rate": 5.321880779501729e-05,
"loss_nats": 1.0588197708129883,
"step": 6560,
"unclipped_grad_norm": 0.818975567817688
},
{
"bits_per_byte": 1.47142164702788,
"learning_rate": 5.290685887172962e-05,
"loss_nats": 1.019911766052246,
"step": 6570,
"unclipped_grad_norm": 0.8489425778388977
},
{
"bits_per_byte": 1.514791200026558,
"learning_rate": 5.2596825310948295e-05,
"loss_nats": 1.0499732494354248,
"step": 6580,
"unclipped_grad_norm": 0.8219936490058899
},
{
"bits_per_byte": 1.5010776474120593,
"learning_rate": 5.228871241029154e-05,
"loss_nats": 1.0404677391052246,
"step": 6590,
"unclipped_grad_norm": 0.7910152077674866
},
{
"bits_per_byte": 1.4169470882716653,
"learning_rate": 5.198252543455865e-05,
"loss_nats": 0.9821528792381287,
"step": 6600,
"unclipped_grad_norm": 0.7658081650733948
},
{
"bits_per_byte": 1.448498423459678,
"learning_rate": 5.167826961564027e-05,
"loss_nats": 1.0040225982666016,
"step": 6610,
"unclipped_grad_norm": 0.7563595771789551
},
{
"bits_per_byte": 1.4411885588587308,
"learning_rate": 5.137595015242884e-05,
"loss_nats": 0.9989557862281799,
"step": 6620,
"unclipped_grad_norm": 0.8031777143478394
},
{
"bits_per_byte": 1.471761484745985,
"learning_rate": 5.1075572210729925e-05,
"loss_nats": 1.0201473236083984,
"step": 6630,
"unclipped_grad_norm": 0.802291989326477
},
{
"bits_per_byte": 1.511551906797557,
"learning_rate": 5.07771409231737e-05,
"loss_nats": 1.0477279424667358,
"step": 6640,
"unclipped_grad_norm": 0.8319684863090515
},
{
"bits_per_byte": 1.5027432993856658,
"learning_rate": 5.048066138912756e-05,
"loss_nats": 1.041622281074524,
"step": 6650,
"unclipped_grad_norm": 0.7944393754005432
},
{
"bits_per_byte": 1.5208112807373597,
"learning_rate": 5.018613867460863e-05,
"loss_nats": 1.0541460514068604,
"step": 6660,
"unclipped_grad_norm": 0.7983800768852234
},
{
"bits_per_byte": 1.499563684136514,
"learning_rate": 4.989357781219755e-05,
"loss_nats": 1.039418339729309,
"step": 6670,
"unclipped_grad_norm": 0.8299691677093506
},
{
"bits_per_byte": 1.4686320884308854,
"learning_rate": 4.960298380095222e-05,
"loss_nats": 1.0179781913757324,
"step": 6680,
"unclipped_grad_norm": 0.8146572113037109
},
{
"bits_per_byte": 1.4999098852127009,
"learning_rate": 4.931436160632256e-05,
"loss_nats": 1.0396583080291748,
"step": 6690,
"unclipped_grad_norm": 0.8203337788581848
},
{
"bits_per_byte": 1.5587066259590745,
"learning_rate": 4.9027716160065485e-05,
"loss_nats": 1.0804131031036377,
"step": 6700,
"unclipped_grad_norm": 0.8495987057685852
},
{
"bits_per_byte": 1.5200210204566451,
"learning_rate": 4.874305236016084e-05,
"loss_nats": 1.0535982847213745,
"step": 6710,
"unclipped_grad_norm": 0.7851044535636902
},
{
"bits_per_byte": 1.5404384568017762,
"learning_rate": 4.846037507072752e-05,
"loss_nats": 1.0677505731582642,
"step": 6720,
"unclipped_grad_norm": 0.8347281217575073
},
{
"bits_per_byte": 1.4888245714686876,
"learning_rate": 4.817968912194049e-05,
"loss_nats": 1.0319745540618896,
"step": 6730,
"unclipped_grad_norm": 0.8212841153144836
},
{
"bits_per_byte": 1.4304995791336308,
"learning_rate": 4.790099930994813e-05,
"loss_nats": 0.9915467500686646,
"step": 6740,
"unclipped_grad_norm": 0.7957568168640137
},
{
"bits_per_byte": 1.5212219753076202,
"learning_rate": 4.762431039679047e-05,
"loss_nats": 1.0544307231903076,
"step": 6750,
"unclipped_grad_norm": 0.7967842817306519
},
{
"bits_per_byte": 1.4749558905031108,
"learning_rate": 4.734962711031755e-05,
"loss_nats": 1.0223615169525146,
"step": 6760,
"unclipped_grad_norm": 0.8139022588729858
},
{
"bits_per_byte": 1.3835263876600992,
"learning_rate": 4.707695414410892e-05,
"loss_nats": 0.9589874148368835,
"step": 6770,
"unclipped_grad_norm": 0.785581648349762
},
{
"bits_per_byte": 1.4903192726873364,
"learning_rate": 4.680629615739322e-05,
"loss_nats": 1.0330106019973755,
"step": 6780,
"unclipped_grad_norm": 0.8513718843460083
},
{
"bits_per_byte": 1.4442869123053497,
"learning_rate": 4.653765777496863e-05,
"loss_nats": 1.001103401184082,
"step": 6790,
"unclipped_grad_norm": 0.8081458210945129
},
{
"bits_per_byte": 1.5181372944817448,
"learning_rate": 4.6271043587123986e-05,
"loss_nats": 1.0522925853729248,
"step": 6800,
"unclipped_grad_norm": 0.8491948843002319
},
{
"bits_per_byte": 1.5056836867674672,
"learning_rate": 4.600645814956006e-05,
"loss_nats": 1.0436604022979736,
"step": 6810,
"unclipped_grad_norm": 0.8421627879142761
},
{
"bits_per_byte": 1.5049580919634769,
"learning_rate": 4.574390598331204e-05,
"loss_nats": 1.0431574583053589,
"step": 6820,
"unclipped_grad_norm": 0.8799064755439758
},
{
"bits_per_byte": 1.4726698971078425,
"learning_rate": 4.548339157467199e-05,
"loss_nats": 1.0207769870758057,
"step": 6830,
"unclipped_grad_norm": 0.8110744953155518
},
{
"bits_per_byte": 1.4402100635666468,
"learning_rate": 4.52249193751124e-05,
"loss_nats": 0.9982775449752808,
"step": 6840,
"unclipped_grad_norm": 0.8030053973197937
},
{
"bits_per_byte": 1.4826992373755579,
"learning_rate": 4.4968493801209985e-05,
"loss_nats": 1.027728796005249,
"step": 6850,
"unclipped_grad_norm": 0.8305994868278503
},
{
"bits_per_byte": 1.4689192994578262,
"learning_rate": 4.471411923457034e-05,
"loss_nats": 1.0181772708892822,
"step": 6860,
"unclipped_grad_norm": 0.8366835117340088
},
{
"bits_per_byte": 1.419515391188325,
"learning_rate": 4.446180002175291e-05,
"loss_nats": 0.9839330911636353,
"step": 6870,
"unclipped_grad_norm": 0.8165588974952698
},
{
"bits_per_byte": 1.4317985640955977,
"learning_rate": 4.421154047419693e-05,
"loss_nats": 0.9924471378326416,
"step": 6880,
"unclipped_grad_norm": 0.7981967926025391
},
{
"bits_per_byte": 1.426708565560662,
"learning_rate": 4.396334486814753e-05,
"loss_nats": 0.9889190196990967,
"step": 6890,
"unclipped_grad_norm": 0.8219130635261536
},
{
"bits_per_byte": 1.544462506866664,
"learning_rate": 4.371721744458285e-05,
"loss_nats": 1.0705398321151733,
"step": 6900,
"unclipped_grad_norm": 0.8717687129974365
},
{
"bits_per_byte": 1.4952656657088017,
"learning_rate": 4.347316240914145e-05,
"loss_nats": 1.0364391803741455,
"step": 6910,
"unclipped_grad_norm": 0.8866252899169922
},
{
"bits_per_byte": 1.4499126367977229,
"learning_rate": 4.3231183932050546e-05,
"loss_nats": 1.0050028562545776,
"step": 6920,
"unclipped_grad_norm": 0.8052312135696411
},
{
"bits_per_byte": 1.437854245215128,
"learning_rate": 4.299128614805464e-05,
"loss_nats": 0.9966446161270142,
"step": 6930,
"unclipped_grad_norm": 0.8604745268821716
},
{
"bits_per_byte": 1.4202836376897292,
"learning_rate": 4.2753473156344965e-05,
"loss_nats": 0.9844655990600586,
"step": 6940,
"unclipped_grad_norm": 0.7888393998146057
},
{
"bits_per_byte": 1.4300773617257627,
"learning_rate": 4.251774902048942e-05,
"loss_nats": 0.9912540912628174,
"step": 6950,
"unclipped_grad_norm": 0.8240157961845398
},
{
"bits_per_byte": 1.4910837355704216,
"learning_rate": 4.228411776836306e-05,
"loss_nats": 1.0335404872894287,
"step": 6960,
"unclipped_grad_norm": 0.833236575126648
},
{
"bits_per_byte": 1.496710663941351,
"learning_rate": 4.2052583392079427e-05,
"loss_nats": 1.0374407768249512,
"step": 6970,
"unclipped_grad_norm": 0.828235387802124
},
{
"bits_per_byte": 1.4445393828368163,
"learning_rate": 4.182314984792214e-05,
"loss_nats": 1.0012784004211426,
"step": 6980,
"unclipped_grad_norm": 0.8125858902931213
},
{
"bits_per_byte": 1.4867339503647994,
"learning_rate": 4.159582105627748e-05,
"loss_nats": 1.0305254459381104,
"step": 6990,
"unclipped_grad_norm": 0.8594081401824951
},
{
"bits_per_byte": 1.533073471761227,
"learning_rate": 4.137060090156724e-05,
"loss_nats": 1.0626455545425415,
"step": 7000,
"unclipped_grad_norm": 0.8734340071678162
},
{
"bits_per_byte": 1.532040715942796,
"learning_rate": 4.114749323218249e-05,
"loss_nats": 1.061929702758789,
"step": 7010,
"unclipped_grad_norm": 0.8863697648048401
},
{
"bits_per_byte": 1.459764662952399,
"learning_rate": 4.0926501860417706e-05,
"loss_nats": 1.0118317604064941,
"step": 7020,
"unclipped_grad_norm": 0.8206458687782288
},
{
"bits_per_byte": 1.5177566968753855,
"learning_rate": 4.070763056240571e-05,
"loss_nats": 1.052028775215149,
"step": 7030,
"unclipped_grad_norm": 0.8729474544525146
},
{
"bits_per_byte": 1.50918250181662,
"learning_rate": 4.049088307805306e-05,
"loss_nats": 1.0460855960845947,
"step": 7040,
"unclipped_grad_norm": 0.8456775546073914
},
{
"bits_per_byte": 1.4815705152379453,
"learning_rate": 4.027626311097629e-05,
"loss_nats": 1.0269464254379272,
"step": 7050,
"unclipped_grad_norm": 0.8360646367073059
},
{
"bits_per_byte": 1.4998292253494463,
"learning_rate": 4.006377432843846e-05,
"loss_nats": 1.0396023988723755,
"step": 7060,
"unclipped_grad_norm": 0.8408570289611816
},
{
"bits_per_byte": 1.453465282416596,
"learning_rate": 3.985342036128656e-05,
"loss_nats": 1.0074653625488281,
"step": 7070,
"unclipped_grad_norm": 0.8465698957443237
},
{
"bits_per_byte": 1.431578598285144,
"learning_rate": 3.964520480388956e-05,
"loss_nats": 0.9922946691513062,
"step": 7080,
"unclipped_grad_norm": 0.8291451930999756
},
{
"bits_per_byte": 1.4576069686146742,
"learning_rate": 3.943913121407678e-05,
"loss_nats": 1.01033616065979,
"step": 7090,
"unclipped_grad_norm": 0.8671156764030457
},
{
"bits_per_byte": 1.4654204844086733,
"learning_rate": 3.923520311307735e-05,
"loss_nats": 1.0157520771026611,
"step": 7100,
"unclipped_grad_norm": 0.831190288066864
},
{
"bits_per_byte": 1.478559958934592,
"learning_rate": 3.9033423985459816e-05,
"loss_nats": 1.0248596668243408,
"step": 7110,
"unclipped_grad_norm": 0.8307176828384399
},
{
"bits_per_byte": 1.5234764958777807,
"learning_rate": 3.88337972790728e-05,
"loss_nats": 1.0559934377670288,
"step": 7120,
"unclipped_grad_norm": 0.8790873289108276
},
{
"bits_per_byte": 1.486035012871693,
"learning_rate": 3.8636326404985845e-05,
"loss_nats": 1.030040979385376,
"step": 7130,
"unclipped_grad_norm": 0.8940773606300354
},
{
"bits_per_byte": 1.4639995637472447,
"learning_rate": 3.844101473743143e-05,
"loss_nats": 1.0147671699523926,
"step": 7140,
"unclipped_grad_norm": 0.8168482184410095
},
{
"bits_per_byte": 1.430851627619947,
"learning_rate": 3.824786561374701e-05,
"loss_nats": 0.991790771484375,
"step": 7150,
"unclipped_grad_norm": 0.7878822088241577
},
{
"bits_per_byte": 1.5069802639723218,
"learning_rate": 3.805688233431824e-05,
"loss_nats": 1.044559121131897,
"step": 7160,
"unclipped_grad_norm": 0.8418039679527283
},
{
"bits_per_byte": 1.4531985373251077,
"learning_rate": 3.786806816252236e-05,
"loss_nats": 1.0072804689407349,
"step": 7170,
"unclipped_grad_norm": 0.815416157245636
},
{
"bits_per_byte": 1.4725130489302556,
"learning_rate": 3.768142632467269e-05,
"loss_nats": 1.0206682682037354,
"step": 7180,
"unclipped_grad_norm": 0.8564274907112122
},
{
"bits_per_byte": 1.437259873173746,
"learning_rate": 3.7496960009963236e-05,
"loss_nats": 0.9962326288223267,
"step": 7190,
"unclipped_grad_norm": 0.8627841472625732
},
{
"bits_per_byte": 1.4854186270510006,
"learning_rate": 3.7314672370414325e-05,
"loss_nats": 1.029613733291626,
"step": 7200,
"unclipped_grad_norm": 0.8707078695297241
},
{
"bits_per_byte": 1.5339277095880628,
"learning_rate": 3.713456652081882e-05,
"loss_nats": 1.0632376670837402,
"step": 7210,
"unclipped_grad_norm": 0.8451290726661682
},
{
"bits_per_byte": 1.5149174352922914,
"learning_rate": 3.695664553868873e-05,
"loss_nats": 1.050060749053955,
"step": 7220,
"unclipped_grad_norm": 0.8569709062576294
},
{
"bits_per_byte": 1.4518221601702528,
"learning_rate": 3.678091246420277e-05,
"loss_nats": 1.00632643699646,
"step": 7230,
"unclipped_grad_norm": 0.8588427901268005
},
{
"bits_per_byte": 1.4720953030713098,
"learning_rate": 3.660737030015427e-05,
"loss_nats": 1.0203787088394165,
"step": 7240,
"unclipped_grad_norm": 0.8185886144638062
},
{
"bits_per_byte": 1.4930615360553439,
"learning_rate": 3.6436022011900056e-05,
"loss_nats": 1.0349113941192627,
"step": 7250,
"unclipped_grad_norm": 0.9033716917037964
},
{
"bits_per_byte": 1.392136183127593,
"learning_rate": 3.626687052730954e-05,
"loss_nats": 0.9649552702903748,
"step": 7260,
"unclipped_grad_norm": 0.8070151209831238
},
{
"bits_per_byte": 1.419755822934231,
"learning_rate": 3.6099918736714955e-05,
"loss_nats": 0.9840997457504272,
"step": 7270,
"unclipped_grad_norm": 0.8482468724250793
},
{
"bits_per_byte": 1.499443640246212,
"learning_rate": 3.5935169492861716e-05,
"loss_nats": 1.0393351316452026,
"step": 7280,
"unclipped_grad_norm": 0.9195974469184875
},
{
"bits_per_byte": 1.424472533125469,
"learning_rate": 3.5772625610859886e-05,
"loss_nats": 0.9873691201210022,
"step": 7290,
"unclipped_grad_norm": 0.84366774559021
},
{
"bits_per_byte": 1.449468749575846,
"learning_rate": 3.5612289868135887e-05,
"loss_nats": 1.004695177078247,
"step": 7300,
"unclipped_grad_norm": 0.8698141574859619
},
{
"bits_per_byte": 1.4526158601039845,
"learning_rate": 3.545416500438522e-05,
"loss_nats": 1.0068765878677368,
"step": 7310,
"unclipped_grad_norm": 0.8154390454292297
},
{
"bits_per_byte": 1.40692677709312,
"learning_rate": 3.52982537215255e-05,
"loss_nats": 0.9752073287963867,
"step": 7320,
"unclipped_grad_norm": 0.835978627204895
},
{
"bits_per_byte": 1.4261922736427717,
"learning_rate": 3.5144558683650396e-05,
"loss_nats": 0.9885611534118652,
"step": 7330,
"unclipped_grad_norm": 0.8695172071456909
},
{
"bits_per_byte": 1.392467249730504,
"learning_rate": 3.499308251698404e-05,
"loss_nats": 0.96518474817276,
"step": 7340,
"unclipped_grad_norm": 0.8504277467727661
},
{
"bits_per_byte": 1.4759263886019296,
"learning_rate": 3.484382780983616e-05,
"loss_nats": 1.0230342149734497,
"step": 7350,
"unclipped_grad_norm": 0.8514381051063538
},
{
"bits_per_byte": 1.4321242992363319,
"learning_rate": 3.469679711255795e-05,
"loss_nats": 0.9926729202270508,
"step": 7360,
"unclipped_grad_norm": 0.8621233105659485
},
{
"bits_per_byte": 1.4814723131443026,
"learning_rate": 3.4551992937498284e-05,
"loss_nats": 1.0268783569335938,
"step": 7370,
"unclipped_grad_norm": 0.8794771432876587
},
{
"bits_per_byte": 1.401024074541532,
"learning_rate": 3.440941775896104e-05,
"loss_nats": 0.9711158871650696,
"step": 7380,
"unclipped_grad_norm": 0.8035550117492676
},
{
"bits_per_byte": 1.474961909895891,
"learning_rate": 3.4269074013162596e-05,
"loss_nats": 1.022365689277649,
"step": 7390,
"unclipped_grad_norm": 0.8444681763648987
},
{
"bits_per_byte": 1.4183169300857874,
"learning_rate": 3.41309640981904e-05,
"loss_nats": 0.9831023812294006,
"step": 7400,
"unclipped_grad_norm": 0.880998969078064
},
{
"bits_per_byte": 1.4093773578852642,
"learning_rate": 3.3995090373961795e-05,
"loss_nats": 0.9769059419631958,
"step": 7410,
"unclipped_grad_norm": 0.8176257014274597
},
{
"bits_per_byte": 1.476101638923159,
"learning_rate": 3.3861455162183876e-05,
"loss_nats": 1.023155689239502,
"step": 7420,
"unclipped_grad_norm": 0.8866280913352966
},
{
"bits_per_byte": 1.489828262219123,
"learning_rate": 3.3730060746313664e-05,
"loss_nats": 1.032670259475708,
"step": 7430,
"unclipped_grad_norm": 0.8741231560707092
},
{
"bits_per_byte": 1.5070485410847143,
"learning_rate": 3.3600909371519225e-05,
"loss_nats": 1.0446064472198486,
"step": 7440,
"unclipped_grad_norm": 0.8906005620956421
},
{
"bits_per_byte": 1.4470038942236803,
"learning_rate": 3.3474003244641194e-05,
"loss_nats": 1.0029866695404053,
"step": 7450,
"unclipped_grad_norm": 0.833862841129303
},
{
"bits_per_byte": 1.4545604679372903,
"learning_rate": 3.334934453415514e-05,
"loss_nats": 1.0082244873046875,
"step": 7460,
"unclipped_grad_norm": 0.851222813129425
},
{
"bits_per_byte": 1.4397255884391662,
"learning_rate": 3.3226935370134486e-05,
"loss_nats": 0.9979417324066162,
"step": 7470,
"unclipped_grad_norm": 0.8808716535568237
},
{
"bits_per_byte": 1.469680494670545,
"learning_rate": 3.31067778442141e-05,
"loss_nats": 1.018704891204834,
"step": 7480,
"unclipped_grad_norm": 0.9037331938743591
},
{
"bits_per_byte": 1.5297206699826567,
"learning_rate": 3.298887400955457e-05,
"loss_nats": 1.060321569442749,
"step": 7490,
"unclipped_grad_norm": 0.8992816209793091
},
{
"bits_per_byte": 1.48344512613235,
"learning_rate": 3.287322588080714e-05,
"loss_nats": 1.0282458066940308,
"step": 7500,
"unclipped_grad_norm": 0.8851720690727234
},
{
"bits_per_byte": 1.4271278592634655,
"learning_rate": 3.275983543407924e-05,
"loss_nats": 0.9892096519470215,
"step": 7510,
"unclipped_grad_norm": 0.8391884565353394
},
{
"bits_per_byte": 1.4636686691269847,
"learning_rate": 3.2648704606900735e-05,
"loss_nats": 1.0145378112792969,
"step": 7520,
"unclipped_grad_norm": 0.8814444541931152
},
{
"bits_per_byte": 1.5040419403823309,
"learning_rate": 3.253983529819086e-05,
"loss_nats": 1.0425224304199219,
"step": 7530,
"unclipped_grad_norm": 0.8750982880592346
},
{
"bits_per_byte": 1.448819515068839,
"learning_rate": 3.243322936822571e-05,
"loss_nats": 1.0042451620101929,
"step": 7540,
"unclipped_grad_norm": 0.8588963150978088
},
{
"bits_per_byte": 1.4846842611318165,
"learning_rate": 3.232888863860648e-05,
"loss_nats": 1.0291047096252441,
"step": 7550,
"unclipped_grad_norm": 0.8553798198699951
},
{
"bits_per_byte": 1.4456509067093426,
"learning_rate": 3.222681489222838e-05,
"loss_nats": 1.0020488500595093,
"step": 7560,
"unclipped_grad_norm": 0.8477172255516052
},
{
"bits_per_byte": 1.4068580700241002,
"learning_rate": 3.212700987325009e-05,
"loss_nats": 0.9751597046852112,
"step": 7570,
"unclipped_grad_norm": 0.8558363914489746
},
{
"bits_per_byte": 1.5845748804406512,
"learning_rate": 3.202947528706403e-05,
"loss_nats": 1.0983436107635498,
"step": 7580,
"unclipped_grad_norm": 0.9179992079734802
},
{
"bits_per_byte": 1.435446746077025,
"learning_rate": 3.1934212800267116e-05,
"loss_nats": 0.9949758648872375,
"step": 7590,
"unclipped_grad_norm": 0.8465545177459717
},
{
"bits_per_byte": 1.4142041369730542,
"learning_rate": 3.1841224040632484e-05,
"loss_nats": 0.9802516102790833,
"step": 7600,
"unclipped_grad_norm": 0.8474127054214478
},
{
"bits_per_byte": 1.441303443269507,
"learning_rate": 3.175051059708143e-05,
"loss_nats": 0.9990354180335999,
"step": 7610,
"unclipped_grad_norm": 0.8852339386940002
},
{
"bits_per_byte": 1.4014744111128163,
"learning_rate": 3.1662074019656414e-05,
"loss_nats": 0.9714280366897583,
"step": 7620,
"unclipped_grad_norm": 0.8255653977394104
},
{
"bits_per_byte": 1.3991135192730972,
"learning_rate": 3.157591581949455e-05,
"loss_nats": 0.96979159116745,
"step": 7630,
"unclipped_grad_norm": 0.8985438942909241
},
{
"bits_per_byte": 1.4200022740729177,
"learning_rate": 3.149203746880175e-05,
"loss_nats": 0.9842705726623535,
"step": 7640,
"unclipped_grad_norm": 0.8505365252494812
},
{
"bits_per_byte": 1.5509151239443857,
"learning_rate": 3.141044040082763e-05,
"loss_nats": 1.075012445449829,
"step": 7650,
"unclipped_grad_norm": 0.8840228915214539
},
{
"bits_per_byte": 1.4633312391659918,
"learning_rate": 3.133112600984091e-05,
"loss_nats": 1.0143039226531982,
"step": 7660,
"unclipped_grad_norm": 0.9004685878753662
},
{
"bits_per_byte": 1.5004719245157205,
"learning_rate": 3.125409565110571e-05,
"loss_nats": 1.0400478839874268,
"step": 7670,
"unclipped_grad_norm": 0.9166978597640991
},
{
"bits_per_byte": 1.4376227565670667,
"learning_rate": 3.117935064085835e-05,
"loss_nats": 0.9964841604232788,
"step": 7680,
"unclipped_grad_norm": 0.8822170495986938
},
{
"bits_per_byte": 1.4453855374790612,
"learning_rate": 3.110689225628479e-05,
"loss_nats": 1.0018649101257324,
"step": 7690,
"unclipped_grad_norm": 0.8656983375549316
},
{
"bits_per_byte": 1.396036663657836,
"learning_rate": 3.1036721735498926e-05,
"loss_nats": 0.9676588773727417,
"step": 7700,
"unclipped_grad_norm": 0.840412974357605
},
{
"bits_per_byte": 1.4832948132954957,
"learning_rate": 3.096884027752132e-05,
"loss_nats": 1.0281416177749634,
"step": 7710,
"unclipped_grad_norm": 0.8699836730957031
},
{
"bits_per_byte": 1.4846416094344024,
"learning_rate": 3.090324904225885e-05,
"loss_nats": 1.0290751457214355,
"step": 7720,
"unclipped_grad_norm": 0.8882943391799927
},
{
"bits_per_byte": 1.4868813394965887,
"learning_rate": 3.083994915048472e-05,
"loss_nats": 1.0306276082992554,
"step": 7730,
"unclipped_grad_norm": 0.8993004560470581
},
{
"bits_per_byte": 1.4329027787054622,
"learning_rate": 3.077894168381942e-05,
"loss_nats": 0.9932125210762024,
"step": 7740,
"unclipped_grad_norm": 0.8985061049461365
},
{
"bits_per_byte": 1.457792709877606,
"learning_rate": 3.0720227684712254e-05,
"loss_nats": 1.0104649066925049,
"step": 7750,
"unclipped_grad_norm": 0.8967045545578003
},
{
"bits_per_byte": 1.475737723633933,
"learning_rate": 3.0663808156423456e-05,
"loss_nats": 1.0229034423828125,
"step": 7760,
"unclipped_grad_norm": 0.8843175768852234
},
{
"bits_per_byte": 1.4006856986759595,
"learning_rate": 3.060968406300711e-05,
"loss_nats": 0.9708813428878784,
"step": 7770,
"unclipped_grad_norm": 0.8820574879646301
},
{
"bits_per_byte": 1.4787585988963383,
"learning_rate": 3.055785632929463e-05,
"loss_nats": 1.024997353553772,
"step": 7780,
"unclipped_grad_norm": 0.8838117718696594
},
{
"bits_per_byte": 1.4172842602586817,
"learning_rate": 3.0508325840878984e-05,
"loss_nats": 0.982386589050293,
"step": 7790,
"unclipped_grad_norm": 0.8568381071090698
},
{
"bits_per_byte": 1.4471380406913532,
"learning_rate": 3.046109344409957e-05,
"loss_nats": 1.0030796527862549,
"step": 7800,
"unclipped_grad_norm": 0.8466761708259583
},
{
"bits_per_byte": 1.4200249757828314,
"learning_rate": 3.0416159946027727e-05,
"loss_nats": 0.9842863082885742,
"step": 7810,
"unclipped_grad_norm": 0.9039835929870605
},
{
"bits_per_byte": 1.410308987904988,
"learning_rate": 3.0373526114452946e-05,
"loss_nats": 0.9775516986846924,
"step": 7820,
"unclipped_grad_norm": 0.8540316224098206
},
{
"bits_per_byte": 1.4409317887609925,
"learning_rate": 3.0333192677869803e-05,
"loss_nats": 0.9987778067588806,
"step": 7830,
"unclipped_grad_norm": 0.9217174649238586
},
{
"bits_per_byte": 1.4393295123942291,
"learning_rate": 3.029516032546544e-05,
"loss_nats": 0.9976671934127808,
"step": 7840,
"unclipped_grad_norm": 0.894000232219696
},
{
"bits_per_byte": 1.3879004224194929,
"learning_rate": 3.02594297071078e-05,
"loss_nats": 0.9620192646980286,
"step": 7850,
"unclipped_grad_norm": 0.856924295425415
},
{
"bits_per_byte": 1.4481823193473922,
"learning_rate": 3.0226001433334603e-05,
"loss_nats": 1.0038034915924072,
"step": 7860,
"unclipped_grad_norm": 0.8965994119644165
},
{
"bits_per_byte": 1.4733294505739012,
"learning_rate": 3.019487607534278e-05,
"loss_nats": 1.021234154701233,
"step": 7870,
"unclipped_grad_norm": 0.9309923648834229
},
{
"bits_per_byte": 1.3911279348369099,
"learning_rate": 3.016605416497886e-05,
"loss_nats": 0.9642564058303833,
"step": 7880,
"unclipped_grad_norm": 0.8823378086090088
},
{
"bits_per_byte": 1.46931073197119,
"learning_rate": 3.0139536194729734e-05,
"loss_nats": 1.0184485912322998,
"step": 7890,
"unclipped_grad_norm": 0.9113782048225403
},
{
"bits_per_byte": 1.449501254296859,
"learning_rate": 3.0115322617714362e-05,
"loss_nats": 1.0047177076339722,
"step": 7900,
"unclipped_grad_norm": 0.8579812049865723
},
{
"bits_per_byte": 1.457684704772864,
"learning_rate": 3.0093413847675965e-05,
"loss_nats": 1.010390043258667,
"step": 7910,
"unclipped_grad_norm": 0.8884916305541992
},
{
"bits_per_byte": 1.4556867823177908,
"learning_rate": 3.0073810258974985e-05,
"loss_nats": 1.0090051889419556,
"step": 7920,
"unclipped_grad_norm": 0.8975357413291931
},
{
"bits_per_byte": 1.4187827450956494,
"learning_rate": 3.0056512186582658e-05,
"loss_nats": 0.9834252595901489,
"step": 7930,
"unclipped_grad_norm": 0.844153106212616
},
{
"bits_per_byte": 1.4233246349222852,
"learning_rate": 3.0041519926075316e-05,
"loss_nats": 0.9865734577178955,
"step": 7940,
"unclipped_grad_norm": 0.9031311273574829
},
{
"bits_per_byte": 1.4939792354803478,
"learning_rate": 3.0028833733629313e-05,
"loss_nats": 1.0355474948883057,
"step": 7950,
"unclipped_grad_norm": 0.910707950592041
},
{
"bits_per_byte": 1.4252896226997183,
"learning_rate": 3.0018453826016686e-05,
"loss_nats": 0.987935483455658,
"step": 7960,
"unclipped_grad_norm": 0.8988999128341675
},
{
"bits_per_byte": 1.446669559950403,
"learning_rate": 3.001038038060143e-05,
"loss_nats": 1.0027549266815186,
"step": 7970,
"unclipped_grad_norm": 0.9068711400032043
},
{
"bits_per_byte": 1.484100552014788,
"learning_rate": 3.0004613535336433e-05,
"loss_nats": 1.0287001132965088,
"step": 7980,
"unclipped_grad_norm": 0.9153758883476257
},
{
"bits_per_byte": 1.5191428770413398,
"learning_rate": 3.000115338876117e-05,
"loss_nats": 1.0529896020889282,
"step": 7990,
"unclipped_grad_norm": 0.9229548573493958
},
{
"bits_per_byte": 1.419257675186007,
"learning_rate": 3e-05,
"loss_nats": 0.9837544560432434,
"step": 8000,
"unclipped_grad_norm": 0.8808227777481079
}
]
}
Reference in New Issue View Git Blame Copy Permalink