wuyang
3557 lines
94 KiB
JSON
3557 lines
94 KiB
JSON
{
|
|
"architecture": "block",
|
|
"batch_size": 32,
|
|
"canonical_sha256_without_self": "81b011e2ab02c49abb6b2686b3e0701d71ab6e7f46cbb7c426eec91702836b7f",
|
|
"depth": 32,
|
|
"diagnostics": [
|
|
{
|
|
"activation_grad_rms_by_block": [
|
|
0.0002677410375326872,
|
|
0.0002473295899108052,
|
|
0.00023034414334688336,
|
|
0.00021438254043459892,
|
|
0.00020044577831868082,
|
|
0.00018819731485564262,
|
|
0.00017973134526982903,
|
|
0.000171217747265473,
|
|
0.0001640921109355986,
|
|
0.00016061548376455903,
|
|
0.00015463090676348656,
|
|
0.00015024171443656087,
|
|
0.0001478576596127823,
|
|
0.00014403100067283958,
|
|
0.00013951722939964384,
|
|
0.00013661097909789532,
|
|
0.00013379570737015456,
|
|
0.0001311724045081064,
|
|
0.0001286817860091105,
|
|
0.00012687484559137374,
|
|
0.0001249351626029238,
|
|
0.00012369138130452484,
|
|
0.00012227990373503417,
|
|
0.00012101328320568427,
|
|
0.00011953213106608018,
|
|
0.0001181897969217971,
|
|
0.0001171672047348693,
|
|
0.00011609455395955592,
|
|
0.00011492414341773838,
|
|
0.0001140913664130494,
|
|
0.00011346775136189535,
|
|
0.00011285000800853595
|
|
],
|
|
"activation_grad_statistics": {
|
|
"first_quartile_mean": 0.00021242368711682502,
|
|
"first_to_last_ratio": 1.8345658968465297,
|
|
"imbalance_abs_log_ratio": 0.6068078850437921,
|
|
"last_quartile_mean": 0.0001157896194854402,
|
|
"mean": 0.0001511171253696375,
|
|
"normalized": [
|
|
1.7717451736709768,
|
|
1.636674793183293,
|
|
1.5242755762025908,
|
|
1.4186515254985965,
|
|
1.3264266232459345,
|
|
1.2453738409548603,
|
|
1.1893512719369177,
|
|
1.1330135273991528,
|
|
1.085860457802012,
|
|
1.0628542818802846,
|
|
1.023252039669589,
|
|
0.9942070699734702,
|
|
0.9784308644775869,
|
|
0.9531083940389613,
|
|
0.9232390376562555,
|
|
0.9040072643238835,
|
|
0.8853775311228679,
|
|
0.8680181295617843,
|
|
0.8515367513400655,
|
|
0.8395795332993111,
|
|
0.8267439067367663,
|
|
0.8185133286645813,
|
|
0.8091730400239778,
|
|
0.800791325997512,
|
|
0.7909899739933552,
|
|
0.7821072339266707,
|
|
0.7753403490721149,
|
|
0.7682422073314642,
|
|
0.7604971517068638,
|
|
0.7549863467424895,
|
|
0.7508596466770359,
|
|
0.7467718018887739
|
|
],
|
|
"population_cv": 0.2714186609524326
|
|
},
|
|
"activation_output_rms_by_block": [
|
|
0.0034885562490671873,
|
|
0.004831995815038681,
|
|
0.005952076520770788,
|
|
0.006925336085259914,
|
|
0.00385080324485898,
|
|
0.005542940925806761,
|
|
0.006729288958013058,
|
|
0.007927131839096546,
|
|
0.004081308841705322,
|
|
0.005811207927763462,
|
|
0.007534808944910765,
|
|
0.008581922389566898,
|
|
0.00499920267611742,
|
|
0.007102519739419222,
|
|
0.008542952127754688,
|
|
0.010447698645293713,
|
|
0.005041283555328846,
|
|
0.007636935915797949,
|
|
0.009387043304741383,
|
|
0.011020516976714134,
|
|
0.005807018838822842,
|
|
0.008367395959794521,
|
|
0.01024632342159748,
|
|
0.012378672137856483,
|
|
0.006453148555010557,
|
|
0.009127611294388771,
|
|
0.011037957854568958,
|
|
0.012611051090061665,
|
|
0.007010402157902718,
|
|
0.009794782847166061,
|
|
0.012445803731679916,
|
|
0.014087481424212456
|
|
],
|
|
"activation_output_statistics": {
|
|
"first_quartile_mean": 0.0056560162047389895,
|
|
"first_to_last_ratio": 0.5480088979804595,
|
|
"imbalance_abs_log_ratio": 0.6014637549753238,
|
|
"last_quartile_mean": 0.010321029869373888,
|
|
"mean": 0.007962599374877755,
|
|
"normalized": [
|
|
0.4381177659237371,
|
|
0.606836484864952,
|
|
0.747504205668035,
|
|
0.8697330807712821,
|
|
0.4836113263475722,
|
|
0.6961220406611074,
|
|
0.845112084785299,
|
|
0.9955457339856744,
|
|
0.512559862622497,
|
|
0.7298129234151854,
|
|
0.9462750278110585,
|
|
1.077778999737589,
|
|
0.6278355146046978,
|
|
0.8919850673170736,
|
|
1.072884836415099,
|
|
1.3120964842531853,
|
|
0.6331203314377778,
|
|
0.9591008609440675,
|
|
1.1788918244911069,
|
|
1.384035094304029,
|
|
0.7292868277593074,
|
|
1.0508372411895934,
|
|
1.2868063479276561,
|
|
1.554601902603762,
|
|
0.8104324041933391,
|
|
1.1463105029730196,
|
|
1.3862254440922965,
|
|
1.5837857082010076,
|
|
0.8804162846646224,
|
|
1.2300986632667847,
|
|
1.5630327667805075,
|
|
1.7692063559870779
|
|
],
|
|
"population_cv": 0.3451071044649734
|
|
},
|
|
"bits_per_byte": 7.959341947457508,
|
|
"branch_output_rms_by_sublayer": [
|
|
0.0022265110164880753,
|
|
0.0027073738165199757,
|
|
0.0021613112185150385,
|
|
0.002722575329244137,
|
|
0.0025601957459002733,
|
|
0.0026850115973502398,
|
|
0.0025323182344436646,
|
|
0.0026861224323511124,
|
|
0.0027253651060163975,
|
|
0.00271838391199708,
|
|
0.002904455177485943,
|
|
0.0026645169127732515,
|
|
0.0029208383057266474,
|
|
0.0026859452482312918,
|
|
0.00299624796025455,
|
|
0.0027359756641089916,
|
|
0.0030937574338167906,
|
|
0.0026767742820084095,
|
|
0.0033804215490818024,
|
|
0.0027451682835817337,
|
|
0.0034370317589491606,
|
|
0.0026999011170119047,
|
|
0.0038969130255281925,
|
|
0.002663193503394723,
|
|
0.004255065228790045,
|
|
0.002626648638397455,
|
|
0.004119740333408117,
|
|
0.0026531771291047335,
|
|
0.003970506135374308,
|
|
0.0026517712976783514,
|
|
0.004385598469525576,
|
|
0.002676903735846281,
|
|
0.0041999719105660915,
|
|
0.002670086920261383,
|
|
0.005232699681073427,
|
|
0.0026355870068073273,
|
|
0.004664790350943804,
|
|
0.0026610493659973145,
|
|
0.004898969549685717,
|
|
0.0026388936676084995,
|
|
0.005131533369421959,
|
|
0.0026469926815479994,
|
|
0.005069608334451914,
|
|
0.00264862016774714,
|
|
0.0051318020559847355,
|
|
0.002641494618728757,
|
|
0.0058372304774820805,
|
|
0.002674198243767023,
|
|
0.0059690922498703,
|
|
0.002557571977376938,
|
|
0.00554921617731452,
|
|
0.00252733426168561,
|
|
0.0058550527319312096,
|
|
0.0025468147359788418,
|
|
0.006255472078919411,
|
|
0.0026856842450797558,
|
|
0.006502739619463682,
|
|
0.002551089273765683,
|
|
0.006122888531535864,
|
|
0.0025778315030038357,
|
|
0.007071305997669697,
|
|
0.0026001501828432083,
|
|
0.006241494789719582,
|
|
0.0025662274565547705
|
|
],
|
|
"capture": {
|
|
"all_gradients_finite": true,
|
|
"all_gradients_present": true,
|
|
"count": 32,
|
|
"dtypes": [
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32"
|
|
],
|
|
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
|
"shape": [
|
|
16,
|
|
256,
|
|
192
|
|
],
|
|
"storage_unique": true
|
|
},
|
|
"core_parameter_grad_rms_by_block": [
|
|
0.0056993408498569815,
|
|
0.005586029321860134,
|
|
0.005830344684528519,
|
|
0.00540288712216694,
|
|
0.00538025811909819,
|
|
0.0052226633069546126,
|
|
0.005297605878918106,
|
|
0.0052023753769565,
|
|
0.0048107019260181325,
|
|
0.005248672753719287,
|
|
0.004962353375385338,
|
|
0.005501319898161136,
|
|
0.005588445553926757,
|
|
0.005229606380765812,
|
|
0.004884302670286647,
|
|
0.005031817046498285,
|
|
0.005000470961685071,
|
|
0.0053937604587735305,
|
|
0.004980398173225761,
|
|
0.004658922297714218,
|
|
0.00496598185096497,
|
|
0.004578385934052906,
|
|
0.004596161734371138,
|
|
0.004990442293511754,
|
|
0.005201771295872185,
|
|
0.00459813521112035,
|
|
0.004706502430387731,
|
|
0.004781878207244352,
|
|
0.0048378722082161795,
|
|
0.004473110720597102,
|
|
0.005084432926394394,
|
|
0.004685705425265166
|
|
],
|
|
"core_parameter_grad_statistics": {
|
|
"first_quartile_mean": 0.005452688082542498,
|
|
"first_to_last_ratio": 1.1368823875795573,
|
|
"imbalance_abs_log_ratio": 0.12828976841498158,
|
|
"last_quartile_mean": 0.004796176053137182,
|
|
"mean": 0.005075395512328069,
|
|
"normalized": [
|
|
1.122935313319594,
|
|
1.1006096585560166,
|
|
1.1487468652180286,
|
|
1.064525337787646,
|
|
1.060066768398564,
|
|
1.0290160233362764,
|
|
1.0437818818356701,
|
|
1.025018713185997,
|
|
0.947847692723257,
|
|
1.0341406380981206,
|
|
0.9777274230809889,
|
|
1.0839194472230789,
|
|
1.1010857262951226,
|
|
1.0303840100861434,
|
|
0.9623491722808084,
|
|
0.9914137793352396,
|
|
0.9852376922229987,
|
|
1.0627271206100406,
|
|
0.9812827711906272,
|
|
0.9179427074003902,
|
|
0.9784423379227619,
|
|
0.9020747098294245,
|
|
0.9055770576316904,
|
|
0.9832617539638114,
|
|
1.024899691706223,
|
|
0.905965889742295,
|
|
0.9273173724009683,
|
|
0.9421685848185097,
|
|
0.9532010257062827,
|
|
0.8813324419214177,
|
|
1.00178063247374,
|
|
0.9232197596982638
|
|
],
|
|
"population_cv": 0.06921897980134169
|
|
},
|
|
"depth_weights": [
|
|
{
|
|
"entropy_mean": 0.0,
|
|
"mean_weights": [
|
|
1.0
|
|
],
|
|
"sources": 1
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
}
|
|
],
|
|
"layer_input_rms_by_sublayer": [
|
|
0.02802751213312149,
|
|
0.014061377383768559,
|
|
0.01413465291261673,
|
|
0.014165290631353855,
|
|
0.014225807972252369,
|
|
0.014281988143920898,
|
|
0.01433455292135477,
|
|
0.014350161887705326,
|
|
0.01440478302538395,
|
|
0.00966146495193243,
|
|
0.009703479707241058,
|
|
0.009741888381540775,
|
|
0.00977881159633398,
|
|
0.009820535778999329,
|
|
0.009858808480203152,
|
|
0.00991273857653141,
|
|
0.009950309991836548,
|
|
0.0074927168898284435,
|
|
0.00752287870272994,
|
|
0.007546746172010899,
|
|
0.007571155205368996,
|
|
0.00766373984515667,
|
|
0.007693085819482803,
|
|
0.007762390188872814,
|
|
0.007788818795233965,
|
|
0.006325984839349985,
|
|
0.006353048607707024,
|
|
0.006375366821885109,
|
|
0.006389216054230928,
|
|
0.006445344537496567,
|
|
0.0064707002602517605,
|
|
0.006555322092026472,
|
|
0.00656962301582098,
|
|
0.005531130358576775,
|
|
0.005553483963012695,
|
|
0.005611324217170477,
|
|
0.0056249531917274,
|
|
0.0056777093559503555,
|
|
0.005688921548426151,
|
|
0.00574630219489336,
|
|
0.005768724251538515,
|
|
0.004946626722812653,
|
|
0.004966245498508215,
|
|
0.0050392793491482735,
|
|
0.005057469941675663,
|
|
0.005084683652967215,
|
|
0.005098348017781973,
|
|
0.005203457549214363,
|
|
0.005216081161051989,
|
|
0.004667261149734259,
|
|
0.0046758754178881645,
|
|
0.004717131145298481,
|
|
0.004727107938379049,
|
|
0.004768412560224533,
|
|
0.004777040798217058,
|
|
0.004826710093766451,
|
|
0.004837613552808762,
|
|
0.0044052014127373695,
|
|
0.0044186655431985855,
|
|
0.004504534415900707,
|
|
0.004510571248829365,
|
|
0.004574235528707504,
|
|
0.0045849340967834,
|
|
0.004644096828997135
|
|
],
|
|
"loss_nats": 5.516995429992676,
|
|
"loss_scale": 1.0,
|
|
"output_weights": {
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
},
|
|
"step": 0,
|
|
"stream_state_rms_by_sublayer": [
|
|
0.0022265110164880753,
|
|
0.0034885562490671873,
|
|
0.0040087271481752396,
|
|
0.004831995815038681,
|
|
0.005379301495850086,
|
|
0.005952076520770788,
|
|
0.006410556845366955,
|
|
0.006925336085259914,
|
|
0.0027253651060163975,
|
|
0.00385080324485898,
|
|
0.004838948138058186,
|
|
0.005542940925806761,
|
|
0.006175771821290255,
|
|
0.006729288958013058,
|
|
0.00742313964292407,
|
|
0.007927131839096546,
|
|
0.0030937574338167906,
|
|
0.004081308841705322,
|
|
0.005150026176124811,
|
|
0.005811207927763462,
|
|
0.007019025273621082,
|
|
0.007534808944910765,
|
|
0.008143599145114422,
|
|
0.008581922389566898,
|
|
0.004255065228790045,
|
|
0.00499920267611742,
|
|
0.0066224560141563416,
|
|
0.007102519739419222,
|
|
0.008129785768687725,
|
|
0.008542952127754688,
|
|
0.010099578648805618,
|
|
0.010447698645293713,
|
|
0.0041999719105660915,
|
|
0.005041283555328846,
|
|
0.00719039561226964,
|
|
0.007636935915797949,
|
|
0.009030314162373543,
|
|
0.009387043304741383,
|
|
0.010736005380749702,
|
|
0.011020516976714134,
|
|
0.005131533369421959,
|
|
0.005807018838822842,
|
|
0.00787449348717928,
|
|
0.008367395959794521,
|
|
0.009912566281855106,
|
|
0.01024632342159748,
|
|
0.012113703414797783,
|
|
0.012378672137856483,
|
|
0.0059690922498703,
|
|
0.006453148555010557,
|
|
0.00876442901790142,
|
|
0.009127611294388771,
|
|
0.010806876234710217,
|
|
0.011037957854568958,
|
|
0.012353328987956047,
|
|
0.012611051090061665,
|
|
0.006502739619463682,
|
|
0.007010402157902718,
|
|
0.009470963850617409,
|
|
0.009794782847166061,
|
|
0.012123330496251583,
|
|
0.012445803731679916,
|
|
0.01384514570236206,
|
|
0.014087481424212456
|
|
]
|
|
},
|
|
{
|
|
"activation_grad_rms_by_block": [
|
|
3.9337450289167464e-05,
|
|
3.887022103299387e-05,
|
|
3.8419901102315634e-05,
|
|
3.799897967837751e-05,
|
|
3.775028380914591e-05,
|
|
3.7539852201007307e-05,
|
|
3.739175735972822e-05,
|
|
3.724253110704012e-05,
|
|
3.720997119671665e-05,
|
|
3.713165278895758e-05,
|
|
3.7142541259527206e-05,
|
|
3.707727591972798e-05,
|
|
3.688051219796762e-05,
|
|
3.692117388709448e-05,
|
|
3.689077857416123e-05,
|
|
3.6904009903082624e-05,
|
|
3.717117215273902e-05,
|
|
3.7258152588037774e-05,
|
|
3.729091622517444e-05,
|
|
3.7364865420386195e-05,
|
|
3.74124720110558e-05,
|
|
3.742674380191602e-05,
|
|
3.742256740224548e-05,
|
|
3.74479714082554e-05,
|
|
3.765593282878399e-05,
|
|
3.763797212741338e-05,
|
|
3.7667807191610336e-05,
|
|
3.764635039260611e-05,
|
|
3.751127951545641e-05,
|
|
3.7523430364672095e-05,
|
|
3.7504440115299076e-05,
|
|
3.756426303880289e-05
|
|
],
|
|
"activation_grad_statistics": {
|
|
"first_quartile_mean": 3.8068872072472004e-05,
|
|
"first_to_last_ratio": 1.0127680561501509,
|
|
"imbalance_abs_log_ratio": 0.01268723177426717,
|
|
"last_quartile_mean": 3.7588934446830535e-05,
|
|
"mean": 3.750672590285831e-05,
|
|
"normalized": [
|
|
1.0488105624322075,
|
|
1.0363533498942825,
|
|
1.0243469718423952,
|
|
1.013124413386392,
|
|
1.0064937128054956,
|
|
1.0008832095404647,
|
|
0.9969347219635258,
|
|
0.9929560688260969,
|
|
0.9920879602525092,
|
|
0.9899998438980743,
|
|
0.9902901510445264,
|
|
0.9885500540824973,
|
|
0.9833039624276305,
|
|
0.9843880796932157,
|
|
0.9835776833655818,
|
|
0.9839304555311837,
|
|
0.9910535046170554,
|
|
0.9933725669506762,
|
|
0.9942461072650594,
|
|
0.9962177321785024,
|
|
0.997487013607996,
|
|
0.9978675264498043,
|
|
0.9977561757634937,
|
|
0.9984334944416348,
|
|
1.0039781378495187,
|
|
1.0034992717011608,
|
|
1.0042947307416068,
|
|
1.0037226520413813,
|
|
1.0001214078938774,
|
|
1.0004453724341882,
|
|
0.9999390565957383,
|
|
1.001534048482227
|
|
],
|
|
"population_cv": 0.013988937740460563
|
|
},
|
|
"activation_output_rms_by_block": [
|
|
0.0037520017940551043,
|
|
0.005621022544801235,
|
|
0.0076507688499987125,
|
|
0.009552493691444397,
|
|
0.005728918593376875,
|
|
0.009368892759084702,
|
|
0.013378600589931011,
|
|
0.01761116087436676,
|
|
0.00800272449851036,
|
|
0.013154285028576851,
|
|
0.01923154480755329,
|
|
0.024512210860848427,
|
|
0.011547097004950047,
|
|
0.016813727095723152,
|
|
0.02113443799316883,
|
|
0.025982216000556946,
|
|
0.010269254446029663,
|
|
0.01641162671148777,
|
|
0.02165052480995655,
|
|
0.026260115206241608,
|
|
0.011169232428073883,
|
|
0.016121258959174156,
|
|
0.019987935200333595,
|
|
0.02486182004213333,
|
|
0.01274633314460516,
|
|
0.016767995432019234,
|
|
0.022238746285438538,
|
|
0.02633264660835266,
|
|
0.0102624436840415,
|
|
0.016213588416576385,
|
|
0.020620016381144524,
|
|
0.02528378553688526
|
|
],
|
|
"activation_output_statistics": {
|
|
"first_quartile_mean": 0.00908298246213235,
|
|
"first_to_last_ratio": 0.4829268696139592,
|
|
"imbalance_abs_log_ratio": 0.7278900454597648,
|
|
"last_quartile_mean": 0.018808194436132908,
|
|
"mean": 0.015944982071232516,
|
|
"normalized": [
|
|
0.2353092513552812,
|
|
0.35252611258450545,
|
|
0.47982298229121323,
|
|
0.5990909020009959,
|
|
0.3592928839796457,
|
|
0.5875762492068142,
|
|
0.8390477035448226,
|
|
1.1044954955540724,
|
|
0.5018961114386356,
|
|
0.8249796061112817,
|
|
1.2061189358281121,
|
|
1.537299367998203,
|
|
0.7241837559531027,
|
|
1.0544839135352808,
|
|
1.3254601290081713,
|
|
1.62949170368986,
|
|
0.6440430224476176,
|
|
1.0292659244250366,
|
|
1.3578268519359336,
|
|
1.6469203344931544,
|
|
0.7004857313841133,
|
|
1.0110553205487558,
|
|
1.2535564550050677,
|
|
1.5592253369157636,
|
|
0.7993946363603464,
|
|
1.0516158222762493,
|
|
1.394717547256516,
|
|
1.6514691889093607,
|
|
0.6436158810461574,
|
|
1.016845822977028,
|
|
1.2931978404884237,
|
|
1.5856891794504773
|
|
],
|
|
"population_cv": 0.416991705473353
|
|
},
|
|
"bits_per_byte": 6.71703146032544,
|
|
"branch_output_rms_by_sublayer": [
|
|
0.002443866338580847,
|
|
0.0027545092161744833,
|
|
0.0026208641938865185,
|
|
0.002763434313237667,
|
|
0.0034182921517640352,
|
|
0.002741520293056965,
|
|
0.0035785960499197245,
|
|
0.002719961805269122,
|
|
0.0047746808268129826,
|
|
0.0028042772319167852,
|
|
0.005376013461500406,
|
|
0.0027222761418670416,
|
|
0.006767150945961475,
|
|
0.0027788877487182617,
|
|
0.007074786350131035,
|
|
0.002833561971783638,
|
|
0.006997238378971815,
|
|
0.0029156480450183153,
|
|
0.008518846705555916,
|
|
0.0027741468511521816,
|
|
0.009081361815333366,
|
|
0.0028916175942867994,
|
|
0.00911131501197815,
|
|
0.0032094581983983517,
|
|
0.010676512494683266,
|
|
0.002878024475648999,
|
|
0.009562043473124504,
|
|
0.0026487084105610847,
|
|
0.008648304268717766,
|
|
0.003162843408063054,
|
|
0.009535389021039009,
|
|
0.0030166516080498695,
|
|
0.009431489743292332,
|
|
0.002794384490698576,
|
|
0.010901342146098614,
|
|
0.00310111534781754,
|
|
0.010227455757558346,
|
|
0.0028713408391922712,
|
|
0.008522198535501957,
|
|
0.0031550333369523287,
|
|
0.010077033191919327,
|
|
0.0032391855493187904,
|
|
0.009554130956530571,
|
|
0.00291044800542295,
|
|
0.010129180736839771,
|
|
0.0026873303577303886,
|
|
0.00920273270457983,
|
|
0.002927709138020873,
|
|
0.011994888074696064,
|
|
0.0029113248456269503,
|
|
0.008809582330286503,
|
|
0.0027162786573171616,
|
|
0.01077704131603241,
|
|
0.0031467911321669817,
|
|
0.010385709814727306,
|
|
0.003254092764109373,
|
|
0.009710066020488739,
|
|
0.0025947801768779755,
|
|
0.009926802478730679,
|
|
0.0031805087346583605,
|
|
0.0101145189255476,
|
|
0.002813765313476324,
|
|
0.009518872946500778,
|
|
0.0030965779442340136
|
|
],
|
|
"capture": {
|
|
"all_gradients_finite": true,
|
|
"all_gradients_present": true,
|
|
"count": 32,
|
|
"dtypes": [
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32"
|
|
],
|
|
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
|
"shape": [
|
|
16,
|
|
256,
|
|
192
|
|
],
|
|
"storage_unique": true
|
|
},
|
|
"core_parameter_grad_rms_by_block": [
|
|
0.00016739224645496468,
|
|
0.0001695381430192955,
|
|
0.0001941383401676919,
|
|
0.00020674849822850373,
|
|
0.00024134787186321302,
|
|
0.00026490993231531454,
|
|
0.000322331199796065,
|
|
0.0003097008723390916,
|
|
0.00031886187327784883,
|
|
0.00034582398876540076,
|
|
0.00034801623779775657,
|
|
0.00036856862737498307,
|
|
0.00040270040252095,
|
|
0.0003782002933862871,
|
|
0.0003548760100734237,
|
|
0.0003670443346435906,
|
|
0.0003676146253339179,
|
|
0.0003749331083729355,
|
|
0.00038033249912268637,
|
|
0.0003562102847256418,
|
|
0.00036931756079357426,
|
|
0.00035104688530666954,
|
|
0.0003583944935512815,
|
|
0.0003605180195562536,
|
|
0.0004079256274935599,
|
|
0.0003614699495467052,
|
|
0.00038354947558429407,
|
|
0.0003579111697278514,
|
|
0.0003529832358531257,
|
|
0.00036980562702627114,
|
|
0.00036596233134640236,
|
|
0.0003819940876588112
|
|
],
|
|
"core_parameter_grad_statistics": {
|
|
"first_quartile_mean": 0.0002345133880230175,
|
|
"first_to_last_ratio": 0.6292279841951004,
|
|
"imbalance_abs_log_ratio": 0.46326163295248246,
|
|
"last_quartile_mean": 0.00037270018802962763,
|
|
"mean": 0.0003331302454070113,
|
|
"normalized": [
|
|
0.502482883985657,
|
|
0.5089244983209419,
|
|
0.582770081205019,
|
|
0.6206236181764371,
|
|
0.7244850180695525,
|
|
0.7952142922107038,
|
|
0.9675831127319218,
|
|
0.9296690306840971,
|
|
0.9571687881064967,
|
|
1.0381044457337716,
|
|
1.044685202247421,
|
|
1.1063799593599613,
|
|
1.2088377085933444,
|
|
1.1352925728019987,
|
|
1.0652770649505088,
|
|
1.1018042933782364,
|
|
1.1035162084570687,
|
|
1.1254850423889022,
|
|
1.141693089614257,
|
|
1.069282329169577,
|
|
1.1086281293442752,
|
|
1.0537826875424299,
|
|
1.0758389503582988,
|
|
1.0822134121018658,
|
|
1.224522939954507,
|
|
1.0850709430633327,
|
|
1.1513499025454192,
|
|
1.07438809493435,
|
|
1.0595952805842004,
|
|
1.1100932206694432,
|
|
1.0985563046047577,
|
|
1.146680894111248
|
|
],
|
|
"population_cv": 0.19646899587719932
|
|
},
|
|
"depth_weights": [
|
|
{
|
|
"entropy_mean": 0.0,
|
|
"mean_weights": [
|
|
1.0
|
|
],
|
|
"sources": 1
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931430697441101,
|
|
"mean_weights": [
|
|
0.49865636229515076,
|
|
0.5013436079025269
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931370496749878,
|
|
"mean_weights": [
|
|
0.49779537320137024,
|
|
0.5022045969963074
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931354999542236,
|
|
"mean_weights": [
|
|
0.49762701988220215,
|
|
0.5023729801177979
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931448578834534,
|
|
"mean_weights": [
|
|
0.49909353256225586,
|
|
0.5009064674377441
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931440234184265,
|
|
"mean_weights": [
|
|
0.49918538331985474,
|
|
0.5008145570755005
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931461691856384,
|
|
"mean_weights": [
|
|
0.49945560097694397,
|
|
0.5005444288253784
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931438446044922,
|
|
"mean_weights": [
|
|
0.49891161918640137,
|
|
0.5010883808135986
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931411623954773,
|
|
"mean_weights": [
|
|
0.49836465716362,
|
|
0.5016353130340576
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986062288284302,
|
|
"mean_weights": [
|
|
0.3321005403995514,
|
|
0.3337631821632385,
|
|
0.3341362476348877
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0985990762710571,
|
|
"mean_weights": [
|
|
0.33337411284446716,
|
|
0.33537548780441284,
|
|
0.3312504291534424
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986008644104004,
|
|
"mean_weights": [
|
|
0.3350442051887512,
|
|
0.33362624049186707,
|
|
0.3313295841217041
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986016988754272,
|
|
"mean_weights": [
|
|
0.3313179612159729,
|
|
0.3338589072227478,
|
|
0.3348231613636017
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986084938049316,
|
|
"mean_weights": [
|
|
0.33454394340515137,
|
|
0.3326125741004944,
|
|
0.33284348249435425
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0985959768295288,
|
|
"mean_weights": [
|
|
0.3312351107597351,
|
|
0.332991361618042,
|
|
0.3357735574245453
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986058712005615,
|
|
"mean_weights": [
|
|
0.33314070105552673,
|
|
0.3320331275463104,
|
|
0.33482617139816284
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986061096191406,
|
|
"mean_weights": [
|
|
0.33257031440734863,
|
|
0.3324887454509735,
|
|
0.33494094014167786
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862926959991455,
|
|
"mean_weights": [
|
|
0.24962691962718964,
|
|
0.2496621459722519,
|
|
0.25036555528640747,
|
|
0.2503453493118286
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862905502319336,
|
|
"mean_weights": [
|
|
0.24916216731071472,
|
|
0.24970784783363342,
|
|
0.25086167454719543,
|
|
0.2502683103084564
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.386286973953247,
|
|
"mean_weights": [
|
|
0.25067827105522156,
|
|
0.24852371215820312,
|
|
0.24991214275360107,
|
|
0.25088587403297424
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862855434417725,
|
|
"mean_weights": [
|
|
0.24860653281211853,
|
|
0.24958111345767975,
|
|
0.2513706088066101,
|
|
0.2504417300224304
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.386291265487671,
|
|
"mean_weights": [
|
|
0.25055181980133057,
|
|
0.24973659217357635,
|
|
0.2504120469093323,
|
|
0.2492995262145996
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862907886505127,
|
|
"mean_weights": [
|
|
0.24893252551555634,
|
|
0.25016576051712036,
|
|
0.2503091096878052,
|
|
0.25059258937835693
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862874507904053,
|
|
"mean_weights": [
|
|
0.24930882453918457,
|
|
0.24942341446876526,
|
|
0.2499566376209259,
|
|
0.25131112337112427
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862828016281128,
|
|
"mean_weights": [
|
|
0.24865929782390594,
|
|
0.24900496006011963,
|
|
0.25141197443008423,
|
|
0.2509238123893738
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094212532043457,
|
|
"mean_weights": [
|
|
0.19921688735485077,
|
|
0.1992446780204773,
|
|
0.19889645278453827,
|
|
0.20075249671936035,
|
|
0.20188948512077332
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.609428882598877,
|
|
"mean_weights": [
|
|
0.19964422285556793,
|
|
0.19919568300247192,
|
|
0.19919705390930176,
|
|
0.20121362805366516,
|
|
0.20074939727783203
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094245910644531,
|
|
"mean_weights": [
|
|
0.20065900683403015,
|
|
0.19955331087112427,
|
|
0.2014925479888916,
|
|
0.19979336857795715,
|
|
0.19850176572799683
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094355583190918,
|
|
"mean_weights": [
|
|
0.19937855005264282,
|
|
0.20006650686264038,
|
|
0.20067161321640015,
|
|
0.1999610960483551,
|
|
0.19992223381996155
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094313859939575,
|
|
"mean_weights": [
|
|
0.1997283697128296,
|
|
0.19996783137321472,
|
|
0.19928213953971863,
|
|
0.1997123807668686,
|
|
0.20130927860736847
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094284057617188,
|
|
"mean_weights": [
|
|
0.19933226704597473,
|
|
0.19993427395820618,
|
|
0.19907569885253906,
|
|
0.2001166194677353,
|
|
0.20154115557670593
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094276905059814,
|
|
"mean_weights": [
|
|
0.19932052493095398,
|
|
0.19971224665641785,
|
|
0.2011900395154953,
|
|
0.20086491107940674,
|
|
0.19891227781772614
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094303131103516,
|
|
"mean_weights": [
|
|
0.19955268502235413,
|
|
0.19991174340248108,
|
|
0.20052853226661682,
|
|
0.20107966661453247,
|
|
0.1989273726940155
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.7917479276657104,
|
|
"mean_weights": [
|
|
0.16630326211452484,
|
|
0.1665954291820526,
|
|
0.1656438112258911,
|
|
0.16612562537193298,
|
|
0.16804365813732147,
|
|
0.16728821396827698
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7917546033859253,
|
|
"mean_weights": [
|
|
0.16606619954109192,
|
|
0.16676779091358185,
|
|
0.16676853597164154,
|
|
0.16595947742462158,
|
|
0.16733971238136292,
|
|
0.1670982539653778
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791755199432373,
|
|
"mean_weights": [
|
|
0.16713324189186096,
|
|
0.16643813252449036,
|
|
0.16707737743854523,
|
|
0.16582445800304413,
|
|
0.16707313060760498,
|
|
0.16645365953445435
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791752815246582,
|
|
"mean_weights": [
|
|
0.16595935821533203,
|
|
0.16658073663711548,
|
|
0.16717244684696198,
|
|
0.1671261042356491,
|
|
0.16733530163764954,
|
|
0.16582605242729187
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7917518615722656,
|
|
"mean_weights": [
|
|
0.1666388213634491,
|
|
0.16675034165382385,
|
|
0.16638417541980743,
|
|
0.16550619900226593,
|
|
0.16747808456420898,
|
|
0.1672423779964447
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7917499542236328,
|
|
"mean_weights": [
|
|
0.16634854674339294,
|
|
0.16704240441322327,
|
|
0.16620638966560364,
|
|
0.16553011536598206,
|
|
0.16771462559700012,
|
|
0.16715793311595917
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7917507886886597,
|
|
"mean_weights": [
|
|
0.16621501743793488,
|
|
0.1665443778038025,
|
|
0.16585618257522583,
|
|
0.16727086901664734,
|
|
0.16779379546642303,
|
|
0.16631974279880524
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7917485237121582,
|
|
"mean_weights": [
|
|
0.16604092717170715,
|
|
0.1667710244655609,
|
|
0.16534343361854553,
|
|
0.1669321358203888,
|
|
0.16766314208507538,
|
|
0.16724932193756104
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.9458963871002197,
|
|
"mean_weights": [
|
|
0.14258044958114624,
|
|
0.14276695251464844,
|
|
0.14234250783920288,
|
|
0.14281943440437317,
|
|
0.14429350197315216,
|
|
0.14340519905090332,
|
|
0.14179196953773499
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9458930492401123,
|
|
"mean_weights": [
|
|
0.14285841584205627,
|
|
0.14281482994556427,
|
|
0.14142268896102905,
|
|
0.14248794317245483,
|
|
0.1441671997308731,
|
|
0.14373424649238586,
|
|
0.1425146758556366
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459006786346436,
|
|
"mean_weights": [
|
|
0.1428324282169342,
|
|
0.14242856204509735,
|
|
0.14238804578781128,
|
|
0.14283627271652222,
|
|
0.14203999936580658,
|
|
0.14380358159542084,
|
|
0.14367109537124634
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9458978176116943,
|
|
"mean_weights": [
|
|
0.14253869652748108,
|
|
0.14282220602035522,
|
|
0.1414484828710556,
|
|
0.14307048916816711,
|
|
0.1438344419002533,
|
|
0.14343075454235077,
|
|
0.1428549587726593
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.94590425491333,
|
|
"mean_weights": [
|
|
0.14281800389289856,
|
|
0.1431872844696045,
|
|
0.1432121843099594,
|
|
0.14210283756256104,
|
|
0.14336752891540527,
|
|
0.14217621088027954,
|
|
0.14313597977161407
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9458997249603271,
|
|
"mean_weights": [
|
|
0.14271901547908783,
|
|
0.14267435669898987,
|
|
0.14293994009494781,
|
|
0.14278604090213776,
|
|
0.14186438918113708,
|
|
0.1428055316209793,
|
|
0.14421072602272034
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459013938903809,
|
|
"mean_weights": [
|
|
0.14281892776489258,
|
|
0.1429084837436676,
|
|
0.14260512590408325,
|
|
0.14224481582641602,
|
|
0.14402621984481812,
|
|
0.14324231445789337,
|
|
0.14215409755706787
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459059238433838,
|
|
"mean_weights": [
|
|
0.14244785904884338,
|
|
0.14288531243801117,
|
|
0.14282266795635223,
|
|
0.14290127158164978,
|
|
0.1435883343219757,
|
|
0.14224958419799805,
|
|
0.14310497045516968
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 2.079432487487793,
|
|
"mean_weights": [
|
|
0.12496939301490784,
|
|
0.12501314282417297,
|
|
0.12449057400226593,
|
|
0.12434226274490356,
|
|
0.12589572370052338,
|
|
0.12463213503360748,
|
|
0.12490519136190414,
|
|
0.1257515549659729
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.0794320106506348,
|
|
"mean_weights": [
|
|
0.12487496435642242,
|
|
0.12489178031682968,
|
|
0.12453664839267731,
|
|
0.12434129416942596,
|
|
0.1249510794878006,
|
|
0.12465334683656693,
|
|
0.12601137161254883,
|
|
0.12573951482772827
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.0794363021850586,
|
|
"mean_weights": [
|
|
0.12491423636674881,
|
|
0.12523731589317322,
|
|
0.12523618340492249,
|
|
0.12467047572135925,
|
|
0.12580403685569763,
|
|
0.12441615015268326,
|
|
0.12498776614665985,
|
|
0.1247338205575943
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079427480697632,
|
|
"mean_weights": [
|
|
0.12474467605352402,
|
|
0.1245410293340683,
|
|
0.12462934106588364,
|
|
0.1246577650308609,
|
|
0.12418574839830399,
|
|
0.12514574825763702,
|
|
0.12619128823280334,
|
|
0.1259044110774994
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.0794358253479004,
|
|
"mean_weights": [
|
|
0.12475918978452682,
|
|
0.12487104535102844,
|
|
0.1250094473361969,
|
|
0.1255180537700653,
|
|
0.12490293383598328,
|
|
0.12545335292816162,
|
|
0.12531054019927979,
|
|
0.12417542934417725
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079436779022217,
|
|
"mean_weights": [
|
|
0.12494880706071854,
|
|
0.12516388297080994,
|
|
0.12451333552598953,
|
|
0.124576136469841,
|
|
0.12569281458854675,
|
|
0.12472400069236755,
|
|
0.1254362016916275,
|
|
0.12494482100009918
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079434871673584,
|
|
"mean_weights": [
|
|
0.12489444017410278,
|
|
0.12512432038784027,
|
|
0.12540367245674133,
|
|
0.1253792643547058,
|
|
0.1248319000005722,
|
|
0.12483960390090942,
|
|
0.125506192445755,
|
|
0.12402061372995377
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079428195953369,
|
|
"mean_weights": [
|
|
0.12488104403018951,
|
|
0.12459895014762878,
|
|
0.12424551695585251,
|
|
0.12444087862968445,
|
|
0.12461263686418533,
|
|
0.12524595856666565,
|
|
0.12623095512390137,
|
|
0.1257440745830536
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.1972174644470215,
|
|
"mean_weights": [
|
|
0.11096074432134628,
|
|
0.11101590096950531,
|
|
0.1106940507888794,
|
|
0.11131151020526886,
|
|
0.11191274225711823,
|
|
0.11117846518754959,
|
|
0.11035685986280441,
|
|
0.11126935482025146,
|
|
0.11130036413669586
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.197216749191284,
|
|
"mean_weights": [
|
|
0.11095131188631058,
|
|
0.11080048978328705,
|
|
0.11054704338312149,
|
|
0.11075608432292938,
|
|
0.1108987033367157,
|
|
0.11123715341091156,
|
|
0.11191022396087646,
|
|
0.11176811158657074,
|
|
0.11113087832927704
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.197216510772705,
|
|
"mean_weights": [
|
|
0.1109621673822403,
|
|
0.1111164540052414,
|
|
0.11043901741504669,
|
|
0.11095462739467621,
|
|
0.11190879344940186,
|
|
0.11175167560577393,
|
|
0.11079787462949753,
|
|
0.1112256720662117,
|
|
0.1108437031507492
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.197213649749756,
|
|
"mean_weights": [
|
|
0.11101574450731277,
|
|
0.11100780963897705,
|
|
0.11082783341407776,
|
|
0.11142586171627045,
|
|
0.11015472561120987,
|
|
0.11108498275279999,
|
|
0.11191210150718689,
|
|
0.11077474802732468,
|
|
0.11179618537425995
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.197218894958496,
|
|
"mean_weights": [
|
|
0.11101824045181274,
|
|
0.11099033057689667,
|
|
0.11046893894672394,
|
|
0.11147398501634598,
|
|
0.11092066764831543,
|
|
0.11164902150630951,
|
|
0.11152558028697968,
|
|
0.11112039536237717,
|
|
0.1108328253030777
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.197216033935547,
|
|
"mean_weights": [
|
|
0.11115090548992157,
|
|
0.11169334501028061,
|
|
0.1117531955242157,
|
|
0.1108616292476654,
|
|
0.11132021248340607,
|
|
0.11068520694971085,
|
|
0.1114465519785881,
|
|
0.11033187806606293,
|
|
0.11075706779956818
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1972179412841797,
|
|
"mean_weights": [
|
|
0.11092537641525269,
|
|
0.11108530312776566,
|
|
0.11051007360219955,
|
|
0.11084415018558502,
|
|
0.11164555698633194,
|
|
0.11119014024734497,
|
|
0.11111340671777725,
|
|
0.11183696240186691,
|
|
0.1108490377664566
|
|
],
|
|
"sources": 9
|
|
}
|
|
],
|
|
"layer_input_rms_by_sublayer": [
|
|
0.02802533470094204,
|
|
0.014066163450479507,
|
|
0.014108670875430107,
|
|
0.014229752123355865,
|
|
0.01430919673293829,
|
|
0.014553130604326725,
|
|
0.01460581086575985,
|
|
0.014812584035098553,
|
|
0.014851128682494164,
|
|
0.010233328677713871,
|
|
0.01034309808164835,
|
|
0.010804922319948673,
|
|
0.010769715532660484,
|
|
0.011443440802395344,
|
|
0.011492080986499786,
|
|
0.01227729581296444,
|
|
0.012374433688819408,
|
|
0.009845216758549213,
|
|
0.00994168221950531,
|
|
0.010567485354840755,
|
|
0.01064346730709076,
|
|
0.01164715364575386,
|
|
0.01175782922655344,
|
|
0.012674410827457905,
|
|
0.012870704755187035,
|
|
0.011088024824857712,
|
|
0.011239404790103436,
|
|
0.011724143289029598,
|
|
0.011804761365056038,
|
|
0.012471788562834263,
|
|
0.012632876634597778,
|
|
0.01310542318969965,
|
|
0.013265094719827175,
|
|
0.011508332565426826,
|
|
0.011626233346760273,
|
|
0.012281009927392006,
|
|
0.012384294532239437,
|
|
0.012981574982404709,
|
|
0.013078294694423676,
|
|
0.013564560562372208,
|
|
0.01372351124882698,
|
|
0.012140991166234016,
|
|
0.012265652418136597,
|
|
0.012530013918876648,
|
|
0.012632926926016808,
|
|
0.013094775378704071,
|
|
0.01315103005617857,
|
|
0.01351068913936615,
|
|
0.013631501235067844,
|
|
0.01232061255723238,
|
|
0.012407775968313217,
|
|
0.012646659277379513,
|
|
0.012729056179523468,
|
|
0.01315284613519907,
|
|
0.013267731294035912,
|
|
0.013451422564685345,
|
|
0.01357149239629507,
|
|
0.012415111064910889,
|
|
0.012491659261286259,
|
|
0.01274929754436016,
|
|
0.012818088755011559,
|
|
0.013031110167503357,
|
|
0.013103967532515526,
|
|
0.013315660879015923
|
|
],
|
|
"loss_nats": 4.655891418457031,
|
|
"loss_scale": 1.0,
|
|
"output_weights": {
|
|
"entropy_mean": 2.197218418121338,
|
|
"mean_weights": [
|
|
0.11057738959789276,
|
|
0.11068764328956604,
|
|
0.11082598567008972,
|
|
0.11127202212810516,
|
|
0.11091771721839905,
|
|
0.11130063980817795,
|
|
0.11115016043186188,
|
|
0.11173908412456512,
|
|
0.11152935773134232
|
|
],
|
|
"sources": 9
|
|
},
|
|
"step": 20,
|
|
"stream_state_rms_by_sublayer": [
|
|
0.002443866338580847,
|
|
0.0037520017940551043,
|
|
0.004732752684503794,
|
|
0.005621022544801235,
|
|
0.007002863567322493,
|
|
0.0076507688499987125,
|
|
0.008998939767479897,
|
|
0.009552493691444397,
|
|
0.0047746808268129826,
|
|
0.005728918593376875,
|
|
0.008738124743103981,
|
|
0.009368892759084702,
|
|
0.012762236408889294,
|
|
0.013378600589931011,
|
|
0.01696598529815674,
|
|
0.01761116087436676,
|
|
0.006997238378971815,
|
|
0.00800272449851036,
|
|
0.012584065087139606,
|
|
0.013154285028576851,
|
|
0.018650829792022705,
|
|
0.01923154480755329,
|
|
0.023473192006349564,
|
|
0.024512210860848427,
|
|
0.010676512494683266,
|
|
0.011547097004950047,
|
|
0.01631898619234562,
|
|
0.016813727095723152,
|
|
0.02041666954755783,
|
|
0.02113443799316883,
|
|
0.02490287274122238,
|
|
0.025982216000556946,
|
|
0.009431489743292332,
|
|
0.010269254446029663,
|
|
0.015727587044239044,
|
|
0.01641162671148777,
|
|
0.02073165774345398,
|
|
0.02165052480995655,
|
|
0.02527376264333725,
|
|
0.026260115206241608,
|
|
0.010077033191919327,
|
|
0.011169232428073883,
|
|
0.015674244612455368,
|
|
0.016121258959174156,
|
|
0.019633492454886436,
|
|
0.019987935200333595,
|
|
0.02385960891842842,
|
|
0.02486182004213333,
|
|
0.011994888074696064,
|
|
0.01274633314460516,
|
|
0.015906881541013718,
|
|
0.016767995432019234,
|
|
0.02143402211368084,
|
|
0.022238746285438538,
|
|
0.02530583366751671,
|
|
0.02633264660835266,
|
|
0.009710066020488739,
|
|
0.0102624436840415,
|
|
0.015399535186588764,
|
|
0.016213588416576385,
|
|
0.01968187838792801,
|
|
0.020620016381144524,
|
|
0.024621067568659782,
|
|
0.02528378553688526
|
|
]
|
|
}
|
|
],
|
|
"environment": {
|
|
"autocast": "cuda-bfloat16-forward-fp32-cross-entropy",
|
|
"compile": false,
|
|
"compute_capability": [
|
|
12,
|
|
0
|
|
],
|
|
"cublas_workspace_config": ":4096:8",
|
|
"cuda": "12.8",
|
|
"deterministic_algorithms": true,
|
|
"gpu": "NVIDIA GeForce RTX 5090",
|
|
"python": "3.10.14",
|
|
"torch": "2.11.0+cu128"
|
|
},
|
|
"evaluations": [
|
|
{
|
|
"bits_per_byte": 7.9557801867581395,
|
|
"cross_entropy_nats": 5.514526605606079,
|
|
"step": 0
|
|
},
|
|
{
|
|
"bits_per_byte": 6.720464749984615,
|
|
"cross_entropy_nats": 4.6582711935043335,
|
|
"step": 20
|
|
}
|
|
],
|
|
"forward_intervention": {
|
|
"depth_visit_counts": [
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39
|
|
],
|
|
"forward_calls": 39,
|
|
"output_mixer_selected": false,
|
|
"output_visit_count": 39,
|
|
"passed": true,
|
|
"selected_depth_indices": [],
|
|
"selected_parameter_reachability_gate": true,
|
|
"selected_parameters": {},
|
|
"selected_source_count_checks": {},
|
|
"selector_gate": true,
|
|
"semantics": "selected depth mixers use parameter-free constant-zero logits with the parent softmax+einsum arithmetic kernel",
|
|
"source_counts_by_depth_index": {
|
|
"0": [
|
|
1
|
|
],
|
|
"1": [
|
|
2
|
|
],
|
|
"10": [
|
|
3
|
|
],
|
|
"11": [
|
|
3
|
|
],
|
|
"12": [
|
|
3
|
|
],
|
|
"13": [
|
|
3
|
|
],
|
|
"14": [
|
|
3
|
|
],
|
|
"15": [
|
|
3
|
|
],
|
|
"16": [
|
|
3
|
|
],
|
|
"17": [
|
|
4
|
|
],
|
|
"18": [
|
|
4
|
|
],
|
|
"19": [
|
|
4
|
|
],
|
|
"2": [
|
|
2
|
|
],
|
|
"20": [
|
|
4
|
|
],
|
|
"21": [
|
|
4
|
|
],
|
|
"22": [
|
|
4
|
|
],
|
|
"23": [
|
|
4
|
|
],
|
|
"24": [
|
|
4
|
|
],
|
|
"25": [
|
|
5
|
|
],
|
|
"26": [
|
|
5
|
|
],
|
|
"27": [
|
|
5
|
|
],
|
|
"28": [
|
|
5
|
|
],
|
|
"29": [
|
|
5
|
|
],
|
|
"3": [
|
|
2
|
|
],
|
|
"30": [
|
|
5
|
|
],
|
|
"31": [
|
|
5
|
|
],
|
|
"32": [
|
|
5
|
|
],
|
|
"33": [
|
|
6
|
|
],
|
|
"34": [
|
|
6
|
|
],
|
|
"35": [
|
|
6
|
|
],
|
|
"36": [
|
|
6
|
|
],
|
|
"37": [
|
|
6
|
|
],
|
|
"38": [
|
|
6
|
|
],
|
|
"39": [
|
|
6
|
|
],
|
|
"4": [
|
|
2
|
|
],
|
|
"40": [
|
|
6
|
|
],
|
|
"41": [
|
|
7
|
|
],
|
|
"42": [
|
|
7
|
|
],
|
|
"43": [
|
|
7
|
|
],
|
|
"44": [
|
|
7
|
|
],
|
|
"45": [
|
|
7
|
|
],
|
|
"46": [
|
|
7
|
|
],
|
|
"47": [
|
|
7
|
|
],
|
|
"48": [
|
|
7
|
|
],
|
|
"49": [
|
|
8
|
|
],
|
|
"5": [
|
|
2
|
|
],
|
|
"50": [
|
|
8
|
|
],
|
|
"51": [
|
|
8
|
|
],
|
|
"52": [
|
|
8
|
|
],
|
|
"53": [
|
|
8
|
|
],
|
|
"54": [
|
|
8
|
|
],
|
|
"55": [
|
|
8
|
|
],
|
|
"56": [
|
|
8
|
|
],
|
|
"57": [
|
|
9
|
|
],
|
|
"58": [
|
|
9
|
|
],
|
|
"59": [
|
|
9
|
|
],
|
|
"6": [
|
|
2
|
|
],
|
|
"60": [
|
|
9
|
|
],
|
|
"61": [
|
|
9
|
|
],
|
|
"62": [
|
|
9
|
|
],
|
|
"63": [
|
|
9
|
|
],
|
|
"7": [
|
|
2
|
|
],
|
|
"8": [
|
|
2
|
|
],
|
|
"9": [
|
|
3
|
|
]
|
|
},
|
|
"uniform_weight_gate": true,
|
|
"uniform_weight_max_abs_error": 0.0,
|
|
"uniform_weight_threshold": 1e-12,
|
|
"unselected_parameter_reachability_gate": true,
|
|
"unselected_parameters": {
|
|
"mixers.0.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.0.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.1.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.1.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.10.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.10.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.11.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.11.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.12.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.12.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.13.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.13.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.14.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.14.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.15.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.15.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.16.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.16.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.17.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.17.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.18.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.18.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.19.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.19.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.2.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.2.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.20.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.20.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.21.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.21.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.22.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.22.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.23.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.23.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.24.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.24.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.25.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.25.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.26.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.26.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.27.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.27.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.28.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.28.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.29.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.29.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.3.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.3.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.30.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.30.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.31.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.31.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.32.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.32.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.33.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.33.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.34.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.34.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.35.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.35.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.36.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.36.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.37.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.37.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.38.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.38.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.39.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.39.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.4.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.4.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.40.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.40.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.41.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.41.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.42.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.42.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.43.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.43.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.44.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.44.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.45.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.45.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.46.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.46.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.47.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.47.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.48.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.48.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.49.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.49.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.5.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.5.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.50.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.50.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.51.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.51.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.52.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.52.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.53.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.53.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.54.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.54.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.55.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.55.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.56.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.56.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.57.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.57.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.58.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.58.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.59.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.59.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.6.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.6.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.60.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.60.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.61.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.61.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.62.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.62.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.63.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.63.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.7.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.7.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.8.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.8.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.9.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.9.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"output_mixer.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"output_mixer.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
}
|
|
},
|
|
"variant": "learned_reference",
|
|
"visit_gate": true
|
|
},
|
|
"gradient_gate": {
|
|
"first_to_last_ratio_abs_delta": 0.0,
|
|
"max_abs_scale_ratio_error": 0.0,
|
|
"normalized_spectrum_max_abs_delta": 0.0,
|
|
"passed": true,
|
|
"per_block_scale_ratios": [
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0
|
|
],
|
|
"population_cv_abs_delta": 0.0,
|
|
"thresholds": {
|
|
"scale_ratio_abs": 1e-05,
|
|
"shape_abs": 1e-06
|
|
}
|
|
},
|
|
"hashes": {
|
|
"final_mixer_parameters": "0f212e3b0862c8a0fa1714765ff7cc8edeb23e1470b2107592e11f4eeae72674",
|
|
"final_model_state": "ac8be80504f01ca19f023c383b7139873a7635445148dec83aff2b39b674dd20",
|
|
"final_optimizer_state": "de50114df8c8cb4b59e160333f861411e6676277c7a3a849bd21ada1ea6abc54",
|
|
"final_public_parameters": "063f8f52a2b325c61dff3a9a7bdd1302f383a71c59990ca1765419e2e5dcbb96",
|
|
"initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d",
|
|
"initial_public_parameter_elements": 18985152,
|
|
"initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670",
|
|
"initial_public_parameter_tensors": 227,
|
|
"initial_public_parameters": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58"
|
|
},
|
|
"manifest": {
|
|
"diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716",
|
|
"file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371",
|
|
"formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
|
|
"input_gate_tensor_hashes": {
|
|
"0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434",
|
|
"1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94",
|
|
"7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc"
|
|
},
|
|
"path": "experiments/k3/attnres_gradient/manifest.json",
|
|
"validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338"
|
|
},
|
|
"model": {
|
|
"attnres_aggregation_groups": 8,
|
|
"context": 256,
|
|
"d_ff": 768,
|
|
"d_head": 32,
|
|
"d_model": 192,
|
|
"heads": 6,
|
|
"layers": 32,
|
|
"parameters": {
|
|
"core": 18985152,
|
|
"embedding": 98304,
|
|
"mixer": 24960,
|
|
"total": 19010112
|
|
},
|
|
"sublayers": 64,
|
|
"sublayers_per_attnres_group": 8,
|
|
"transformer_blocks_per_attnres_group": 4,
|
|
"vocabulary": 256
|
|
},
|
|
"optimizer": {
|
|
"betas": [
|
|
0.9,
|
|
0.95
|
|
],
|
|
"epsilon": 1e-08,
|
|
"grad_clip": 1.0,
|
|
"min_lr": 3e-05,
|
|
"name": "AdamW",
|
|
"peak_lr": 0.0003,
|
|
"warmup_steps": 400,
|
|
"weight_decay_ndim_ge_2": 0.1
|
|
},
|
|
"parent_protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
|
|
"parent_runner_canonical_sha256": "fc6cf8d746ae7004851f9d52a5fc29a99061d0d6a5ec28f2f4c70a6313eb712a",
|
|
"protocol_id": "llm-atlas-k3-attnres-forward-training-v1",
|
|
"run_kind": "smoke",
|
|
"schema_version": 2,
|
|
"seed": 2026073001,
|
|
"steps": 20,
|
|
"study_manifest": {
|
|
"file_sha256": "49546ed5baf36bcb30885062b7c671fafe4ccff2e606624cd7dbe23717f9a712",
|
|
"path": "/home/wuyang/Code/K3/experiments/k3/attnres_forward/manifest.json",
|
|
"status": "frozen-before-model-output"
|
|
},
|
|
"target_bytes_seen": 163840,
|
|
"timing": {
|
|
"mean_ms": null,
|
|
"measured_steps": 0,
|
|
"median_ms": null,
|
|
"p95_ms": null,
|
|
"peak_allocated_bytes": 6751356416,
|
|
"peak_reserved_bytes": 13337886720,
|
|
"warmup_steps_excluded": 20
|
|
},
|
|
"training_history": [
|
|
{
|
|
"bits_per_byte": 7.958777844362678,
|
|
"learning_rate": 7.499999999999999e-07,
|
|
"loss_nats": 5.516604423522949,
|
|
"step": 1,
|
|
"unclipped_grad_norm": 21.323171615600586
|
|
},
|
|
{
|
|
"bits_per_byte": 7.247870358756172,
|
|
"learning_rate": 7.499999999999999e-06,
|
|
"loss_nats": 5.02384090423584,
|
|
"step": 10,
|
|
"unclipped_grad_norm": 11.153014183044434
|
|
},
|
|
{
|
|
"bits_per_byte": 6.7336346654397365,
|
|
"learning_rate": 1.4999999999999999e-05,
|
|
"loss_nats": 4.667399883270264,
|
|
"step": 20,
|
|
"unclipped_grad_norm": 3.4888553619384766
|
|
}
|
|
],
|
|
"variant": "learned_reference"
|
|
}
|