wuyang
1133 lines
32 KiB
JSON
1133 lines
32 KiB
JSON
{
|
|
"architecture": "baseline",
|
|
"batch_size": 32,
|
|
"canonical_sha256_without_self": "5958d579e71547e6f80ee5807def35862d1fe1f475d705f6d6c7f20874d1f5b2",
|
|
"depth": 32,
|
|
"diagnostics": [
|
|
{
|
|
"activation_grad_rms_by_block": [
|
|
0.0002768727717921138,
|
|
0.0002554995589889586,
|
|
0.00023783760843798518,
|
|
0.00022115583124104887,
|
|
0.00020655356638599187,
|
|
0.00019376208365429193,
|
|
0.00018488570640329272,
|
|
0.00017590694187674671,
|
|
0.0001684112212387845,
|
|
0.0001647331955609843,
|
|
0.00015843863366171718,
|
|
0.0001538397918920964,
|
|
0.00015132295084185898,
|
|
0.00014730033581145108,
|
|
0.00014253577683120966,
|
|
0.0001394751889165491,
|
|
0.0001364792260574177,
|
|
0.0001337067224085331,
|
|
0.00013106675760354847,
|
|
0.00012915431580040604,
|
|
0.00012709009752143174,
|
|
0.00012574320135172457,
|
|
0.0001242366706719622,
|
|
0.0001228937617270276,
|
|
0.00012132258416386321,
|
|
0.00011988351616309956,
|
|
0.00011879197700181976,
|
|
0.00011764531518565491,
|
|
0.00011639683361863717,
|
|
0.00011549647024367005,
|
|
0.00011481995898066089,
|
|
0.00011415599146857858
|
|
],
|
|
"activation_grad_statistics": {
|
|
"first_quartile_mean": 0.0002190592585975537,
|
|
"first_to_last_ratio": 1.8672887091156776,
|
|
"imbalance_abs_log_ratio": 0.6244874905881238,
|
|
"last_quartile_mean": 0.00011731408085324802,
|
|
"mean": 0.0001546067051094724,
|
|
"normalized": [
|
|
1.7908199492128654,
|
|
1.6525774791465027,
|
|
1.538339545297078,
|
|
1.4304413969915148,
|
|
1.3359935860462033,
|
|
1.2532579587482624,
|
|
1.1958453307208163,
|
|
1.1377704592578457,
|
|
1.0892879524179597,
|
|
1.065498391187767,
|
|
1.0247850088360109,
|
|
0.9950395863049215,
|
|
0.978760596021445,
|
|
0.9527422223192203,
|
|
0.9219249367631522,
|
|
0.9021289782858439,
|
|
0.8827510162691898,
|
|
0.8648183939620171,
|
|
0.8477430361816715,
|
|
0.835373315206192,
|
|
0.8220219002238166,
|
|
0.8133101424203405,
|
|
0.80356586464988,
|
|
0.7948798963150414,
|
|
0.7847174809007043,
|
|
0.775409553409819,
|
|
0.7683494510649245,
|
|
0.7609328140222236,
|
|
0.7528576043077825,
|
|
0.7470340316863389,
|
|
0.7426583400723811,
|
|
0.7383637817502684
|
|
],
|
|
"population_cv": 0.27888268733798954
|
|
},
|
|
"activation_output_rms_by_block": [
|
|
0.028270306065678596,
|
|
0.0284538883715868,
|
|
0.02867252007126808,
|
|
0.02881420962512493,
|
|
0.029061447829008102,
|
|
0.02929086796939373,
|
|
0.029533835127949715,
|
|
0.029811719432473183,
|
|
0.030115574598312378,
|
|
0.030315518379211426,
|
|
0.030812423676252365,
|
|
0.03120259940624237,
|
|
0.03179442882537842,
|
|
0.03198441118001938,
|
|
0.03240228444337845,
|
|
0.0329107940196991,
|
|
0.033366311341524124,
|
|
0.033811621367931366,
|
|
0.03420914337038994,
|
|
0.03470747172832489,
|
|
0.03500960022211075,
|
|
0.03567802160978317,
|
|
0.035978782922029495,
|
|
0.036836784332990646,
|
|
0.03769209608435631,
|
|
0.038124579936265945,
|
|
0.038542140275239944,
|
|
0.03904740884900093,
|
|
0.0400773324072361,
|
|
0.04093807935714722,
|
|
0.04163828864693642,
|
|
0.042313966900110245
|
|
],
|
|
"activation_output_statistics": {
|
|
"first_quartile_mean": 0.028988599311560392,
|
|
"first_to_last_ratio": 0.728416493900548,
|
|
"imbalance_abs_log_ratio": 0.31688228737002583,
|
|
"last_quartile_mean": 0.03979673655703664,
|
|
"mean": 0.03379432682413608,
|
|
"normalized": [
|
|
0.8365399971656723,
|
|
0.8419723381282118,
|
|
0.8484418174825138,
|
|
0.8526345198433033,
|
|
0.8599504875549783,
|
|
0.8667392051281828,
|
|
0.873928789339177,
|
|
0.8821515986281313,
|
|
0.8911429055838948,
|
|
0.8970593951159853,
|
|
0.9117632032322649,
|
|
0.9233088017589188,
|
|
0.9408214873116122,
|
|
0.9464432106153376,
|
|
0.9588084003566117,
|
|
0.9738555879797565,
|
|
0.9873346942272493,
|
|
1.0005117587897308,
|
|
1.0122747391422395,
|
|
1.0270206567197142,
|
|
1.0359608701276664,
|
|
1.055739970659861,
|
|
1.0646397281194921,
|
|
1.0900286466627183,
|
|
1.1153379761196018,
|
|
1.1281355043603702,
|
|
1.1404914344295491,
|
|
1.1554427183059932,
|
|
1.1859189447920195,
|
|
1.2113891059344621,
|
|
1.2321088348236646,
|
|
1.2521026715611152
|
|
],
|
|
"population_cv": 0.12455242949918699
|
|
},
|
|
"bits_per_byte": 7.95915895791699,
|
|
"branch_output_rms_by_sublayer": [
|
|
0.0022265268489718437,
|
|
0.002717922907322645,
|
|
0.0021652730647474527,
|
|
0.002733128145337105,
|
|
0.0025650423485785723,
|
|
0.002695153234526515,
|
|
0.002537536434829235,
|
|
0.002696225419640541,
|
|
0.0027307753916829824,
|
|
0.0027451792266219854,
|
|
0.0029185698367655277,
|
|
0.0026901187375187874,
|
|
0.002936159959062934,
|
|
0.0027115964330732822,
|
|
0.0030124466866254807,
|
|
0.0027617362793534994,
|
|
0.0031113612931221724,
|
|
0.002723218873143196,
|
|
0.0034121612552553415,
|
|
0.0027921765577048063,
|
|
0.0034701526165008545,
|
|
0.002744668396189809,
|
|
0.0039346772246062756,
|
|
0.0027058592531830072,
|
|
0.004295050632208586,
|
|
0.0026920263189822435,
|
|
0.004176756367087364,
|
|
0.0027182905469089746,
|
|
0.004023758694529533,
|
|
0.0027150253299623728,
|
|
0.004447397775948048,
|
|
0.0027387896552681923,
|
|
0.004260339308530092,
|
|
0.00275799329392612,
|
|
0.005327676888555288,
|
|
0.002721488941460848,
|
|
0.00474865036085248,
|
|
0.0027454239316284657,
|
|
0.00498866755515337,
|
|
0.002719792304560542,
|
|
0.0052231852896511555,
|
|
0.0027601351030170918,
|
|
0.005184618756175041,
|
|
0.002755985129624605,
|
|
0.0052438946440815926,
|
|
0.0027470553759485483,
|
|
0.0059704589657485485,
|
|
0.0027777766808867455,
|
|
0.0060942270793020725,
|
|
0.0026787330862134695,
|
|
0.005698176100850105,
|
|
0.002644060645252466,
|
|
0.006003713235259056,
|
|
0.002661966485902667,
|
|
0.006405353546142578,
|
|
0.002804658142849803,
|
|
0.006667613051831722,
|
|
0.0026854604948312044,
|
|
0.006290186662226915,
|
|
0.0027102255262434483,
|
|
0.0072579155676066875,
|
|
0.0027289679273962975,
|
|
0.006407012697309256,
|
|
0.0026884053368121386
|
|
],
|
|
"capture": {
|
|
"all_gradients_finite": true,
|
|
"all_gradients_present": true,
|
|
"count": 32,
|
|
"dtypes": [
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32"
|
|
],
|
|
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
|
"shape": [
|
|
16,
|
|
256,
|
|
192
|
|
],
|
|
"storage_unique": true
|
|
},
|
|
"core_parameter_grad_rms_by_block": [
|
|
0.005888019831616423,
|
|
0.005779358340731243,
|
|
0.0060348395662513914,
|
|
0.005586875351280001,
|
|
0.005559071516712478,
|
|
0.0054082748429522395,
|
|
0.005481135467826832,
|
|
0.0053805923812947415,
|
|
0.004971746645687392,
|
|
0.005439263395635174,
|
|
0.005139424262259116,
|
|
0.005696496994849179,
|
|
0.005782818048228185,
|
|
0.0054294510739720785,
|
|
0.005067071208887471,
|
|
0.005218073376917496,
|
|
0.005181036474299639,
|
|
0.005605812513830226,
|
|
0.005170579144406511,
|
|
0.004834742615880465,
|
|
0.005150211089068095,
|
|
0.00476651480071292,
|
|
0.004776402486441839,
|
|
0.00519066249960807,
|
|
0.0053994045182471796,
|
|
0.004795508114414784,
|
|
0.004895792813412353,
|
|
0.00496559115560922,
|
|
0.00502742614163479,
|
|
0.004657478744424888,
|
|
0.005287606581362069,
|
|
0.004870555415551657
|
|
],
|
|
"core_parameter_grad_statistics": {
|
|
"first_quartile_mean": 0.005639770912333169,
|
|
"first_to_last_ratio": 1.130799174678946,
|
|
"imbalance_abs_log_ratio": 0.12292461698112994,
|
|
"last_quartile_mean": 0.0049874204355821175,
|
|
"mean": 0.005263682419187692,
|
|
"normalized": [
|
|
1.1186122875029152,
|
|
1.0979686615711766,
|
|
1.146505257280075,
|
|
1.061400537941684,
|
|
1.0561183357962487,
|
|
1.0274698228824493,
|
|
1.0413119621059315,
|
|
1.0222106792919112,
|
|
0.9445377303850805,
|
|
1.0333570611721248,
|
|
0.976393302818647,
|
|
1.0822265747043835,
|
|
1.0986259404914112,
|
|
1.0314929058371969,
|
|
0.962647592570647,
|
|
0.9913351455049914,
|
|
0.9842988352437859,
|
|
1.0649982402804865,
|
|
0.9823121405573801,
|
|
0.9185095586801336,
|
|
0.9784425956805525,
|
|
0.9055475655859389,
|
|
0.9074260386664719,
|
|
0.9861275977985596,
|
|
1.0257846291342996,
|
|
0.9110557462459602,
|
|
0.9301079403205877,
|
|
0.9433683038148654,
|
|
0.9551157804103686,
|
|
0.884832779319472,
|
|
1.0045451378463044,
|
|
0.9253133125579595
|
|
],
|
|
"population_cv": 0.06750436074287791
|
|
},
|
|
"depth_weights": [],
|
|
"layer_input_rms_by_sublayer": [
|
|
0.028027428314089775,
|
|
0.028122643008828163,
|
|
0.028270306065678596,
|
|
0.028331812471151352,
|
|
0.0284538883715868,
|
|
0.02856677956879139,
|
|
0.02867252007126808,
|
|
0.02870391495525837,
|
|
0.02881420962512493,
|
|
0.028933145105838776,
|
|
0.029061447829008102,
|
|
0.029177924618124962,
|
|
0.02929086796939373,
|
|
0.029417123645544052,
|
|
0.029533835127949715,
|
|
0.02969702146947384,
|
|
0.029811719432473183,
|
|
0.02999095991253853,
|
|
0.030115574598312378,
|
|
0.0302139800041914,
|
|
0.030315518379211426,
|
|
0.03069150820374489,
|
|
0.030812423676252365,
|
|
0.03109365701675415,
|
|
0.03120259940624237,
|
|
0.03165328875184059,
|
|
0.03179442882537842,
|
|
0.03191109746694565,
|
|
0.03198441118001938,
|
|
0.03227011114358902,
|
|
0.03240228444337845,
|
|
0.03283532336354256,
|
|
0.0329107940196991,
|
|
0.03322411701083183,
|
|
0.033366311341524124,
|
|
0.0337243415415287,
|
|
0.033811621367931366,
|
|
0.03413747623562813,
|
|
0.03420914337038994,
|
|
0.03456577658653259,
|
|
0.03470747172832489,
|
|
0.03486170619726181,
|
|
0.03500960022211075,
|
|
0.03554223105311394,
|
|
0.03567802160978317,
|
|
0.03587620332837105,
|
|
0.035978782922029495,
|
|
0.036742135882377625,
|
|
0.036836784332990646,
|
|
0.03761643171310425,
|
|
0.03769209608435631,
|
|
0.03803960233926773,
|
|
0.038124579936265945,
|
|
0.03846931457519531,
|
|
0.038542140275239944,
|
|
0.03895273432135582,
|
|
0.03904740884900093,
|
|
0.03994588926434517,
|
|
0.0400773324072361,
|
|
0.04087882116436958,
|
|
0.04093807935714722,
|
|
0.04153488203883171,
|
|
0.04163828864693642,
|
|
0.042190924286842346
|
|
],
|
|
"loss_nats": 5.516868591308594,
|
|
"loss_scale": 1.0,
|
|
"output_weights": null,
|
|
"step": 0,
|
|
"stream_state_rms_by_sublayer": [
|
|
0.028122643008828163,
|
|
0.028270306065678596,
|
|
0.028331812471151352,
|
|
0.0284538883715868,
|
|
0.02856677956879139,
|
|
0.02867252007126808,
|
|
0.02870391495525837,
|
|
0.02881420962512493,
|
|
0.028933145105838776,
|
|
0.029061447829008102,
|
|
0.029177924618124962,
|
|
0.02929086796939373,
|
|
0.029417123645544052,
|
|
0.029533835127949715,
|
|
0.02969702146947384,
|
|
0.029811719432473183,
|
|
0.02999095991253853,
|
|
0.030115574598312378,
|
|
0.0302139800041914,
|
|
0.030315518379211426,
|
|
0.03069150820374489,
|
|
0.030812423676252365,
|
|
0.03109365701675415,
|
|
0.03120259940624237,
|
|
0.03165328875184059,
|
|
0.03179442882537842,
|
|
0.03191109746694565,
|
|
0.03198441118001938,
|
|
0.03227011114358902,
|
|
0.03240228444337845,
|
|
0.03283532336354256,
|
|
0.0329107940196991,
|
|
0.03322411701083183,
|
|
0.033366311341524124,
|
|
0.0337243415415287,
|
|
0.033811621367931366,
|
|
0.03413747623562813,
|
|
0.03420914337038994,
|
|
0.03456577658653259,
|
|
0.03470747172832489,
|
|
0.03486170619726181,
|
|
0.03500960022211075,
|
|
0.03554223105311394,
|
|
0.03567802160978317,
|
|
0.03587620332837105,
|
|
0.035978782922029495,
|
|
0.036742135882377625,
|
|
0.036836784332990646,
|
|
0.03761643171310425,
|
|
0.03769209608435631,
|
|
0.03803960233926773,
|
|
0.038124579936265945,
|
|
0.03846931457519531,
|
|
0.038542140275239944,
|
|
0.03895273432135582,
|
|
0.03904740884900093,
|
|
0.03994588926434517,
|
|
0.0400773324072361,
|
|
0.04087882116436958,
|
|
0.04093807935714722,
|
|
0.04153488203883171,
|
|
0.04163828864693642,
|
|
0.042190924286842346,
|
|
0.042313966900110245
|
|
]
|
|
},
|
|
{
|
|
"activation_grad_rms_by_block": [
|
|
4.0331513446290046e-05,
|
|
3.985461080446839e-05,
|
|
3.939039379474707e-05,
|
|
3.8955986383371055e-05,
|
|
3.865906546707265e-05,
|
|
3.843139347736724e-05,
|
|
3.827241380349733e-05,
|
|
3.811378701357171e-05,
|
|
3.791215567616746e-05,
|
|
3.7825848266948014e-05,
|
|
3.78328186343424e-05,
|
|
3.776324956561439e-05,
|
|
3.772067429963499e-05,
|
|
3.7761601561214775e-05,
|
|
3.77303222194314e-05,
|
|
3.774304786929861e-05,
|
|
3.785186345339753e-05,
|
|
3.794040094362572e-05,
|
|
3.797327008214779e-05,
|
|
3.8049773138482124e-05,
|
|
3.8102883991086856e-05,
|
|
3.8117221265565604e-05,
|
|
3.811223359662108e-05,
|
|
3.81382014893461e-05,
|
|
3.817927427007817e-05,
|
|
3.815999662037939e-05,
|
|
3.819023913820274e-05,
|
|
3.816752723651007e-05,
|
|
3.8173260691110045e-05,
|
|
3.818520053755492e-05,
|
|
3.816522075794637e-05,
|
|
3.822705912170932e-05
|
|
],
|
|
"activation_grad_statistics": {
|
|
"first_quartile_mean": 3.9001145523798186e-05,
|
|
"first_to_last_ratio": 1.0214812032742024,
|
|
"imbalance_abs_log_ratio": 0.021253734004134984,
|
|
"last_quartile_mean": 3.818097229668638e-05,
|
|
"mean": 3.825101589427504e-05,
|
|
"normalized": [
|
|
1.0543906482840992,
|
|
1.0419229364946974,
|
|
1.0297868664095418,
|
|
1.0184301115307508,
|
|
1.0106676793611298,
|
|
1.0047156285624088,
|
|
1.000559407605838,
|
|
0.9964124121282784,
|
|
0.9911411446157616,
|
|
0.9888848016873021,
|
|
0.9890670286747801,
|
|
0.9872482777971484,
|
|
0.9861352285098545,
|
|
0.987205193859086,
|
|
0.9863874550081799,
|
|
0.9867201429007677,
|
|
0.9895649192172892,
|
|
0.991879563368778,
|
|
0.9927388644292499,
|
|
0.9947388912140491,
|
|
0.9961273733592431,
|
|
0.9965021941095827,
|
|
0.9963718009990231,
|
|
0.9970506821246067,
|
|
0.99812445179508,
|
|
0.9976204743385841,
|
|
0.998411107400643,
|
|
0.9978173479628427,
|
|
0.9979672382197662,
|
|
0.9982793827776487,
|
|
0.9977570494711616,
|
|
0.9993736957828273
|
|
],
|
|
"population_cv": 0.01543848086298159
|
|
},
|
|
"activation_output_rms_by_block": [
|
|
0.028325499966740608,
|
|
0.028658077120780945,
|
|
0.029205938801169395,
|
|
0.029751326888799667,
|
|
0.03091570734977722,
|
|
0.032307714223861694,
|
|
0.034374311566352844,
|
|
0.03687922656536102,
|
|
0.03944294899702072,
|
|
0.04215770959854126,
|
|
0.04655195400118828,
|
|
0.050798118114471436,
|
|
0.05539363995194435,
|
|
0.05819777771830559,
|
|
0.062257908284664154,
|
|
0.0653507262468338,
|
|
0.06877651810646057,
|
|
0.07320939004421234,
|
|
0.07733170688152313,
|
|
0.08104265481233597,
|
|
0.08443007618188858,
|
|
0.08700668811798096,
|
|
0.09079501777887344,
|
|
0.093939870595932,
|
|
0.09777236729860306,
|
|
0.10033779591321945,
|
|
0.10455679148435593,
|
|
0.10687003284692764,
|
|
0.11059616506099701,
|
|
0.11359578371047974,
|
|
0.11617089062929153,
|
|
0.11852118372917175
|
|
],
|
|
"activation_output_statistics": {
|
|
"first_quartile_mean": 0.031302225310355425,
|
|
"first_to_last_ratio": 0.2883599077004872,
|
|
"imbalance_abs_log_ratio": 1.2435458995301245,
|
|
"last_quartile_mean": 0.10855262633413076,
|
|
"mean": 0.06861004745587707,
|
|
"normalized": [
|
|
0.4128476953023048,
|
|
0.41769504880769653,
|
|
0.4256802011389317,
|
|
0.43362930054716353,
|
|
0.4506002910092582,
|
|
0.4708889648362173,
|
|
0.5010098788877659,
|
|
0.5375193274582407,
|
|
0.5748859016951763,
|
|
0.614453876098144,
|
|
0.6785005363992164,
|
|
0.7403889080114624,
|
|
0.8073692120322151,
|
|
0.8482398697615304,
|
|
0.9074167792217611,
|
|
0.9524949868145868,
|
|
1.002426330497593,
|
|
1.0670359919411672,
|
|
1.1271192740575633,
|
|
1.1812068030481144,
|
|
1.2305788920520035,
|
|
1.2681333324238657,
|
|
1.323348709782827,
|
|
1.3691853318764207,
|
|
1.4250444493786454,
|
|
1.4624358914450017,
|
|
1.5239282781664902,
|
|
1.5576440595767767,
|
|
1.61195290138986,
|
|
1.6556727173747092,
|
|
1.693205221931973,
|
|
1.7274610370353176
|
|
],
|
|
"population_cv": 0.4388810528199641
|
|
},
|
|
"bits_per_byte": 6.713833958880599,
|
|
"branch_output_rms_by_sublayer": [
|
|
0.002437001559883356,
|
|
0.0027646415401250124,
|
|
0.0026093844790011644,
|
|
0.002773481421172619,
|
|
0.0033957704436033964,
|
|
0.0027507112827152014,
|
|
0.003547805128619075,
|
|
0.0027288019191473722,
|
|
0.004703273996710777,
|
|
0.0028269628528505564,
|
|
0.005306650884449482,
|
|
0.002743088873103261,
|
|
0.006675844080746174,
|
|
0.002795359119772911,
|
|
0.006955645978450775,
|
|
0.002845237497240305,
|
|
0.006907123140990734,
|
|
0.002933910582214594,
|
|
0.00843191146850586,
|
|
0.0027949840296059847,
|
|
0.00897424016147852,
|
|
0.0029047494754195213,
|
|
0.009036343544721603,
|
|
0.0032153772190213203,
|
|
0.010582618415355682,
|
|
0.0028866741340607405,
|
|
0.009517580270767212,
|
|
0.0026635508984327316,
|
|
0.008616920560598373,
|
|
0.003158343257382512,
|
|
0.009494570083916187,
|
|
0.0030213629361242056,
|
|
0.009422268718481064,
|
|
0.002808293094858527,
|
|
0.010874342173337936,
|
|
0.0031132514122873545,
|
|
0.01018539723008871,
|
|
0.002876743907108903,
|
|
0.008500457741320133,
|
|
0.0031567441765218973,
|
|
0.01006424892693758,
|
|
0.0032528946176171303,
|
|
0.009540545754134655,
|
|
0.0029259754810482264,
|
|
0.010095320641994476,
|
|
0.002696031704545021,
|
|
0.009177232161164284,
|
|
0.002929815324023366,
|
|
0.011962441727519035,
|
|
0.0029209970962256193,
|
|
0.008841389790177345,
|
|
0.0027309327851980925,
|
|
0.010780212469398975,
|
|
0.0031468512024730444,
|
|
0.01038750447332859,
|
|
0.0032646870240569115,
|
|
0.009689838625490665,
|
|
0.0026016440242528915,
|
|
0.009929094463586807,
|
|
0.003191312775015831,
|
|
0.010107175447046757,
|
|
0.0028179220389574766,
|
|
0.009526536799967289,
|
|
0.003107903292402625
|
|
],
|
|
"capture": {
|
|
"all_gradients_finite": true,
|
|
"all_gradients_present": true,
|
|
"count": 32,
|
|
"dtypes": [
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32"
|
|
],
|
|
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
|
"shape": [
|
|
16,
|
|
256,
|
|
192
|
|
],
|
|
"storage_unique": true
|
|
},
|
|
"core_parameter_grad_rms_by_block": [
|
|
0.00017190496538337113,
|
|
0.0001735542784589828,
|
|
0.00019824933045045772,
|
|
0.0002107982497012391,
|
|
0.0002450550613783958,
|
|
0.00026804391005753116,
|
|
0.0003238054491237347,
|
|
0.00031010311411301124,
|
|
0.00031952489285031816,
|
|
0.00034636354958654015,
|
|
0.00034728721218647997,
|
|
0.00036779944805402854,
|
|
0.00040306175531293955,
|
|
0.00038015703078453524,
|
|
0.0003565887583279226,
|
|
0.00036832720297469803,
|
|
0.00036821166426922845,
|
|
0.000375260819686975,
|
|
0.00038062964117533765,
|
|
0.00035620909008044117,
|
|
0.0003690236577410054,
|
|
0.00035091389469732654,
|
|
0.00035793360754103896,
|
|
0.000359518412683121,
|
|
0.00040508202264603015,
|
|
0.0003592735021571915,
|
|
0.00038012959666749183,
|
|
0.00035476778007974936,
|
|
0.0003492989830982136,
|
|
0.0003659888269167611,
|
|
0.0003625696460584541,
|
|
0.0003779058324425446
|
|
],
|
|
"core_parameter_grad_statistics": {
|
|
"first_quartile_mean": 0.00023768929483334046,
|
|
"first_to_last_ratio": 0.6434869511235988,
|
|
"imbalance_abs_log_ratio": 0.44085353012576206,
|
|
"last_quartile_mean": 0.0003693770237583045,
|
|
"mean": 0.00033322941208390925,
|
|
"normalized": [
|
|
0.5158757275005617,
|
|
0.5208252097965493,
|
|
0.5949334700399654,
|
|
0.632591968346896,
|
|
0.7353944534664587,
|
|
0.8043825074781695,
|
|
0.9717192942206389,
|
|
0.9305994695177908,
|
|
0.9588736205850275,
|
|
1.0394147005826835,
|
|
1.0421865525454603,
|
|
1.1037424510457507,
|
|
1.2095623636350745,
|
|
1.140826760780676,
|
|
1.0700998933375405,
|
|
1.105326209566252,
|
|
1.1049794853538033,
|
|
1.1261335466764921,
|
|
1.1422450341192953,
|
|
1.0689605333839665,
|
|
1.107416225456362,
|
|
1.053069993140234,
|
|
1.074135699194851,
|
|
1.0788915973377284,
|
|
1.2156250557619686,
|
|
1.0781566366258335,
|
|
1.1407444327626544,
|
|
1.0646352549168638,
|
|
1.0482237474591767,
|
|
1.0983088936477277,
|
|
1.0880481521455758,
|
|
1.134071059571973
|
|
],
|
|
"population_cv": 0.19078746332772342
|
|
},
|
|
"depth_weights": [],
|
|
"layer_input_rms_by_sublayer": [
|
|
0.028024883940815926,
|
|
0.02821914665400982,
|
|
0.028325499966740608,
|
|
0.02856484428048134,
|
|
0.028658077120780945,
|
|
0.02912120893597603,
|
|
0.029205938801169395,
|
|
0.02964756451547146,
|
|
0.029751326888799667,
|
|
0.030729545280337334,
|
|
0.03091570734977722,
|
|
0.03217839077115059,
|
|
0.032307714223861694,
|
|
0.03411358222365379,
|
|
0.034374311566352844,
|
|
0.036541424691677094,
|
|
0.03687922656536102,
|
|
0.03906221315264702,
|
|
0.03944294899702072,
|
|
0.04181833937764168,
|
|
0.04215770959854126,
|
|
0.04607149586081505,
|
|
0.04655195400118828,
|
|
0.0500124990940094,
|
|
0.050798118114471436,
|
|
0.05468229204416275,
|
|
0.05539363995194435,
|
|
0.05778742954134941,
|
|
0.05819777771830559,
|
|
0.06149948015809059,
|
|
0.062257908284664154,
|
|
0.06454319506883621,
|
|
0.0653507262468338,
|
|
0.06803742796182632,
|
|
0.06877651810646057,
|
|
0.07261326909065247,
|
|
0.07320939004421234,
|
|
0.07679533213376999,
|
|
0.07733170688152313,
|
|
0.08014968782663345,
|
|
0.08104265481233597,
|
|
0.0835338607430458,
|
|
0.08443007618188858,
|
|
0.08647285401821136,
|
|
0.08700668811798096,
|
|
0.09021914005279541,
|
|
0.09079501777887344,
|
|
0.09300754219293594,
|
|
0.093939870595932,
|
|
0.0970776379108429,
|
|
0.09777236729860306,
|
|
0.0996294915676117,
|
|
0.10033779591321945,
|
|
0.10360711812973022,
|
|
0.10455679148435593,
|
|
0.10598249733448029,
|
|
0.10687003284692764,
|
|
0.10993104428052902,
|
|
0.11059616506099701,
|
|
0.11287258565425873,
|
|
0.11359578371047974,
|
|
0.11536551266908646,
|
|
0.11617089062929153,
|
|
0.11782468110322952
|
|
],
|
|
"loss_nats": 4.653675079345703,
|
|
"loss_scale": 1.0,
|
|
"output_weights": null,
|
|
"step": 20,
|
|
"stream_state_rms_by_sublayer": [
|
|
0.02821914665400982,
|
|
0.028325499966740608,
|
|
0.02856484428048134,
|
|
0.028658077120780945,
|
|
0.02912120893597603,
|
|
0.029205938801169395,
|
|
0.02964756451547146,
|
|
0.029751326888799667,
|
|
0.030729545280337334,
|
|
0.03091570734977722,
|
|
0.03217839077115059,
|
|
0.032307714223861694,
|
|
0.03411358222365379,
|
|
0.034374311566352844,
|
|
0.036541424691677094,
|
|
0.03687922656536102,
|
|
0.03906221315264702,
|
|
0.03944294899702072,
|
|
0.04181833937764168,
|
|
0.04215770959854126,
|
|
0.04607149586081505,
|
|
0.04655195400118828,
|
|
0.0500124990940094,
|
|
0.050798118114471436,
|
|
0.05468229204416275,
|
|
0.05539363995194435,
|
|
0.05778742954134941,
|
|
0.05819777771830559,
|
|
0.06149948015809059,
|
|
0.062257908284664154,
|
|
0.06454319506883621,
|
|
0.0653507262468338,
|
|
0.06803742796182632,
|
|
0.06877651810646057,
|
|
0.07261326909065247,
|
|
0.07320939004421234,
|
|
0.07679533213376999,
|
|
0.07733170688152313,
|
|
0.08014968782663345,
|
|
0.08104265481233597,
|
|
0.0835338607430458,
|
|
0.08443007618188858,
|
|
0.08647285401821136,
|
|
0.08700668811798096,
|
|
0.09021914005279541,
|
|
0.09079501777887344,
|
|
0.09300754219293594,
|
|
0.093939870595932,
|
|
0.0970776379108429,
|
|
0.09777236729860306,
|
|
0.0996294915676117,
|
|
0.10033779591321945,
|
|
0.10360711812973022,
|
|
0.10455679148435593,
|
|
0.10598249733448029,
|
|
0.10687003284692764,
|
|
0.10993104428052902,
|
|
0.11059616506099701,
|
|
0.11287258565425873,
|
|
0.11359578371047974,
|
|
0.11536551266908646,
|
|
0.11617089062929153,
|
|
0.11782468110322952,
|
|
0.11852118372917175
|
|
]
|
|
}
|
|
],
|
|
"environment": {
|
|
"autocast": "cuda-bfloat16-forward-fp32-cross-entropy",
|
|
"compile": false,
|
|
"compute_capability": [
|
|
12,
|
|
0
|
|
],
|
|
"cublas_workspace_config": ":4096:8",
|
|
"cuda": "12.8",
|
|
"deterministic_algorithms": true,
|
|
"gpu": "NVIDIA GeForce RTX 5090",
|
|
"python": "3.10.14",
|
|
"torch": "2.11.0+cu128"
|
|
},
|
|
"evaluations": [
|
|
{
|
|
"bits_per_byte": 7.95552135286859,
|
|
"cross_entropy_nats": 5.514347195625305,
|
|
"step": 0
|
|
},
|
|
{
|
|
"bits_per_byte": 6.717222361067899,
|
|
"cross_entropy_nats": 4.656023740768433,
|
|
"step": 20
|
|
}
|
|
],
|
|
"gradient_gate": {
|
|
"first_to_last_ratio_abs_delta": 0.0,
|
|
"max_abs_scale_ratio_error": 0.0,
|
|
"normalized_spectrum_max_abs_delta": 0.0,
|
|
"passed": true,
|
|
"per_block_scale_ratios": [
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0
|
|
],
|
|
"population_cv_abs_delta": 0.0,
|
|
"thresholds": {
|
|
"scale_ratio_abs": 1e-05,
|
|
"shape_abs": 1e-06
|
|
}
|
|
},
|
|
"hashes": {
|
|
"final_mixer_parameters": null,
|
|
"final_model_state": "450392cd5ca5eb90de17c89ee83bb7c238615fdc46da5fcb7c6b648eec8c8858",
|
|
"final_optimizer_state": "88ff74de5c6dcffa477d815ee1b7fd6ced81b387b2327a5bf0fd7a221cea9a6d",
|
|
"final_public_parameters": "450392cd5ca5eb90de17c89ee83bb7c238615fdc46da5fcb7c6b648eec8c8858",
|
|
"initial_mixer_parameters": null,
|
|
"initial_public_parameter_elements": 18985152,
|
|
"initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670",
|
|
"initial_public_parameter_tensors": 227,
|
|
"initial_public_parameters": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58"
|
|
},
|
|
"manifest": {
|
|
"diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716",
|
|
"file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371",
|
|
"formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
|
|
"input_gate_tensor_hashes": {
|
|
"0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434",
|
|
"1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94",
|
|
"7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc"
|
|
},
|
|
"path": "experiments/k3/attnres_gradient/manifest.json",
|
|
"validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338"
|
|
},
|
|
"model": {
|
|
"attnres_aggregation_groups": 8,
|
|
"context": 256,
|
|
"d_ff": 768,
|
|
"d_head": 32,
|
|
"d_model": 192,
|
|
"heads": 6,
|
|
"layers": 32,
|
|
"parameters": {
|
|
"core": 18985152,
|
|
"embedding": 98304,
|
|
"mixer": 0,
|
|
"total": 18985152
|
|
},
|
|
"sublayers": 64,
|
|
"sublayers_per_attnres_group": 8,
|
|
"transformer_blocks_per_attnres_group": 4,
|
|
"vocabulary": 256
|
|
},
|
|
"optimizer": {
|
|
"betas": [
|
|
0.9,
|
|
0.95
|
|
],
|
|
"epsilon": 1e-08,
|
|
"grad_clip": 1.0,
|
|
"min_lr": 3e-05,
|
|
"name": "AdamW",
|
|
"peak_lr": 0.0003,
|
|
"warmup_steps": 400,
|
|
"weight_decay_ndim_ge_2": 0.1
|
|
},
|
|
"protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
|
|
"run_kind": "smoke",
|
|
"schema_version": 1,
|
|
"seed": 2026073001,
|
|
"steps": 20,
|
|
"target_bytes_seen": 163840,
|
|
"timing": {
|
|
"mean_ms": null,
|
|
"measured_steps": 0,
|
|
"median_ms": null,
|
|
"p95_ms": null,
|
|
"peak_allocated_bytes": 3185052160,
|
|
"peak_reserved_bytes": 6325010432,
|
|
"warmup_steps_excluded": 20
|
|
},
|
|
"training_history": [
|
|
{
|
|
"bits_per_byte": 7.958522622108798,
|
|
"learning_rate": 7.499999999999999e-07,
|
|
"loss_nats": 5.516427516937256,
|
|
"step": 1,
|
|
"unclipped_grad_norm": 22.09486961364746
|
|
},
|
|
{
|
|
"bits_per_byte": 7.238571600789322,
|
|
"learning_rate": 7.499999999999999e-06,
|
|
"loss_nats": 5.017395496368408,
|
|
"step": 10,
|
|
"unclipped_grad_norm": 11.172341346740723
|
|
},
|
|
{
|
|
"bits_per_byte": 6.729949421197048,
|
|
"learning_rate": 1.4999999999999999e-05,
|
|
"loss_nats": 4.6648454666137695,
|
|
"step": 20,
|
|
"unclipped_grad_norm": 3.462282180786133
|
|
}
|
|
]
|
|
}
|