wuyang
3624 lines
95 KiB
JSON
3624 lines
95 KiB
JSON
{
|
|
"architecture": "block",
|
|
"batch_size": 32,
|
|
"canonical_sha256_without_self": "f0b86fbc1e9319eaa88f686bf658788c2aaba0dda022580df2fd5e54d3202097",
|
|
"depth": 32,
|
|
"diagnostics": [
|
|
{
|
|
"activation_grad_rms_by_block": [
|
|
0.0002677410375326872,
|
|
0.0002473295899108052,
|
|
0.00023034414334688336,
|
|
0.00021438254043459892,
|
|
0.00020044577831868082,
|
|
0.00018819731485564262,
|
|
0.00017973134526982903,
|
|
0.000171217747265473,
|
|
0.0001640921109355986,
|
|
0.00016061548376455903,
|
|
0.00015463090676348656,
|
|
0.00015024171443656087,
|
|
0.0001478576596127823,
|
|
0.00014403100067283958,
|
|
0.00013951722939964384,
|
|
0.00013661097909789532,
|
|
0.00013379570737015456,
|
|
0.0001311724045081064,
|
|
0.0001286817860091105,
|
|
0.00012687484559137374,
|
|
0.0001249351626029238,
|
|
0.00012369138130452484,
|
|
0.00012227990373503417,
|
|
0.00012101328320568427,
|
|
0.00011953213106608018,
|
|
0.0001181897969217971,
|
|
0.0001171672047348693,
|
|
0.00011609455395955592,
|
|
0.00011492414341773838,
|
|
0.0001140913664130494,
|
|
0.00011346775136189535,
|
|
0.00011285000800853595
|
|
],
|
|
"activation_grad_statistics": {
|
|
"first_quartile_mean": 0.00021242368711682502,
|
|
"first_to_last_ratio": 1.8345658968465297,
|
|
"imbalance_abs_log_ratio": 0.6068078850437921,
|
|
"last_quartile_mean": 0.0001157896194854402,
|
|
"mean": 0.0001511171253696375,
|
|
"normalized": [
|
|
1.7717451736709768,
|
|
1.636674793183293,
|
|
1.5242755762025908,
|
|
1.4186515254985965,
|
|
1.3264266232459345,
|
|
1.2453738409548603,
|
|
1.1893512719369177,
|
|
1.1330135273991528,
|
|
1.085860457802012,
|
|
1.0628542818802846,
|
|
1.023252039669589,
|
|
0.9942070699734702,
|
|
0.9784308644775869,
|
|
0.9531083940389613,
|
|
0.9232390376562555,
|
|
0.9040072643238835,
|
|
0.8853775311228679,
|
|
0.8680181295617843,
|
|
0.8515367513400655,
|
|
0.8395795332993111,
|
|
0.8267439067367663,
|
|
0.8185133286645813,
|
|
0.8091730400239778,
|
|
0.800791325997512,
|
|
0.7909899739933552,
|
|
0.7821072339266707,
|
|
0.7753403490721149,
|
|
0.7682422073314642,
|
|
0.7604971517068638,
|
|
0.7549863467424895,
|
|
0.7508596466770359,
|
|
0.7467718018887739
|
|
],
|
|
"population_cv": 0.2714186609524326
|
|
},
|
|
"activation_output_rms_by_block": [
|
|
0.0034885562490671873,
|
|
0.004831995815038681,
|
|
0.005952076520770788,
|
|
0.006925336085259914,
|
|
0.00385080324485898,
|
|
0.005542940925806761,
|
|
0.006729288958013058,
|
|
0.007927131839096546,
|
|
0.004081308841705322,
|
|
0.005811207927763462,
|
|
0.007534808944910765,
|
|
0.008581922389566898,
|
|
0.00499920267611742,
|
|
0.007102519739419222,
|
|
0.008542952127754688,
|
|
0.010447698645293713,
|
|
0.005041283555328846,
|
|
0.007636935915797949,
|
|
0.009387043304741383,
|
|
0.011020516976714134,
|
|
0.005807018838822842,
|
|
0.008367395959794521,
|
|
0.01024632342159748,
|
|
0.012378672137856483,
|
|
0.006453148555010557,
|
|
0.009127611294388771,
|
|
0.011037957854568958,
|
|
0.012611051090061665,
|
|
0.007010402157902718,
|
|
0.009794782847166061,
|
|
0.012445803731679916,
|
|
0.014087481424212456
|
|
],
|
|
"activation_output_statistics": {
|
|
"first_quartile_mean": 0.0056560162047389895,
|
|
"first_to_last_ratio": 0.5480088979804595,
|
|
"imbalance_abs_log_ratio": 0.6014637549753238,
|
|
"last_quartile_mean": 0.010321029869373888,
|
|
"mean": 0.007962599374877755,
|
|
"normalized": [
|
|
0.4381177659237371,
|
|
0.606836484864952,
|
|
0.747504205668035,
|
|
0.8697330807712821,
|
|
0.4836113263475722,
|
|
0.6961220406611074,
|
|
0.845112084785299,
|
|
0.9955457339856744,
|
|
0.512559862622497,
|
|
0.7298129234151854,
|
|
0.9462750278110585,
|
|
1.077778999737589,
|
|
0.6278355146046978,
|
|
0.8919850673170736,
|
|
1.072884836415099,
|
|
1.3120964842531853,
|
|
0.6331203314377778,
|
|
0.9591008609440675,
|
|
1.1788918244911069,
|
|
1.384035094304029,
|
|
0.7292868277593074,
|
|
1.0508372411895934,
|
|
1.2868063479276561,
|
|
1.554601902603762,
|
|
0.8104324041933391,
|
|
1.1463105029730196,
|
|
1.3862254440922965,
|
|
1.5837857082010076,
|
|
0.8804162846646224,
|
|
1.2300986632667847,
|
|
1.5630327667805075,
|
|
1.7692063559870779
|
|
],
|
|
"population_cv": 0.3451071044649734
|
|
},
|
|
"bits_per_byte": 7.959341947457508,
|
|
"branch_output_rms_by_sublayer": [
|
|
0.0022265110164880753,
|
|
0.0027073738165199757,
|
|
0.0021613112185150385,
|
|
0.002722575329244137,
|
|
0.0025601957459002733,
|
|
0.0026850115973502398,
|
|
0.0025323182344436646,
|
|
0.0026861224323511124,
|
|
0.0027253651060163975,
|
|
0.00271838391199708,
|
|
0.002904455177485943,
|
|
0.0026645169127732515,
|
|
0.0029208383057266474,
|
|
0.0026859452482312918,
|
|
0.00299624796025455,
|
|
0.0027359756641089916,
|
|
0.0030937574338167906,
|
|
0.0026767742820084095,
|
|
0.0033804215490818024,
|
|
0.0027451682835817337,
|
|
0.0034370317589491606,
|
|
0.0026999011170119047,
|
|
0.0038969130255281925,
|
|
0.002663193503394723,
|
|
0.004255065228790045,
|
|
0.002626648638397455,
|
|
0.004119740333408117,
|
|
0.0026531771291047335,
|
|
0.003970506135374308,
|
|
0.0026517712976783514,
|
|
0.004385598469525576,
|
|
0.002676903735846281,
|
|
0.0041999719105660915,
|
|
0.002670086920261383,
|
|
0.005232699681073427,
|
|
0.0026355870068073273,
|
|
0.004664790350943804,
|
|
0.0026610493659973145,
|
|
0.004898969549685717,
|
|
0.0026388936676084995,
|
|
0.005131533369421959,
|
|
0.0026469926815479994,
|
|
0.005069608334451914,
|
|
0.00264862016774714,
|
|
0.0051318020559847355,
|
|
0.002641494618728757,
|
|
0.0058372304774820805,
|
|
0.002674198243767023,
|
|
0.0059690922498703,
|
|
0.002557571977376938,
|
|
0.00554921617731452,
|
|
0.00252733426168561,
|
|
0.0058550527319312096,
|
|
0.0025468147359788418,
|
|
0.006255472078919411,
|
|
0.0026856842450797558,
|
|
0.006502739619463682,
|
|
0.002551089273765683,
|
|
0.006122888531535864,
|
|
0.0025778315030038357,
|
|
0.007071305997669697,
|
|
0.0026001501828432083,
|
|
0.006241494789719582,
|
|
0.0025662274565547705
|
|
],
|
|
"capture": {
|
|
"all_gradients_finite": true,
|
|
"all_gradients_present": true,
|
|
"count": 32,
|
|
"dtypes": [
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32"
|
|
],
|
|
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
|
"shape": [
|
|
16,
|
|
256,
|
|
192
|
|
],
|
|
"storage_unique": true
|
|
},
|
|
"core_parameter_grad_rms_by_block": [
|
|
0.0056993408498569815,
|
|
0.005586029321860134,
|
|
0.005830344684528519,
|
|
0.00540288712216694,
|
|
0.00538025811909819,
|
|
0.0052226633069546126,
|
|
0.005297605878918106,
|
|
0.0052023753769565,
|
|
0.0048107019260181325,
|
|
0.005248672753719287,
|
|
0.004962353375385338,
|
|
0.005501319898161136,
|
|
0.005588445553926757,
|
|
0.005229606380765812,
|
|
0.004884302670286647,
|
|
0.005031817046498285,
|
|
0.005000470961685071,
|
|
0.0053937604587735305,
|
|
0.004980398173225761,
|
|
0.004658922297714218,
|
|
0.00496598185096497,
|
|
0.004578385934052906,
|
|
0.004596161734371138,
|
|
0.004990442293511754,
|
|
0.005201771295872185,
|
|
0.00459813521112035,
|
|
0.004706502430387731,
|
|
0.004781878207244352,
|
|
0.0048378722082161795,
|
|
0.004473110720597102,
|
|
0.005084432926394394,
|
|
0.004685705425265166
|
|
],
|
|
"core_parameter_grad_statistics": {
|
|
"first_quartile_mean": 0.005452688082542498,
|
|
"first_to_last_ratio": 1.1368823875795573,
|
|
"imbalance_abs_log_ratio": 0.12828976841498158,
|
|
"last_quartile_mean": 0.004796176053137182,
|
|
"mean": 0.005075395512328069,
|
|
"normalized": [
|
|
1.122935313319594,
|
|
1.1006096585560166,
|
|
1.1487468652180286,
|
|
1.064525337787646,
|
|
1.060066768398564,
|
|
1.0290160233362764,
|
|
1.0437818818356701,
|
|
1.025018713185997,
|
|
0.947847692723257,
|
|
1.0341406380981206,
|
|
0.9777274230809889,
|
|
1.0839194472230789,
|
|
1.1010857262951226,
|
|
1.0303840100861434,
|
|
0.9623491722808084,
|
|
0.9914137793352396,
|
|
0.9852376922229987,
|
|
1.0627271206100406,
|
|
0.9812827711906272,
|
|
0.9179427074003902,
|
|
0.9784423379227619,
|
|
0.9020747098294245,
|
|
0.9055770576316904,
|
|
0.9832617539638114,
|
|
1.024899691706223,
|
|
0.905965889742295,
|
|
0.9273173724009683,
|
|
0.9421685848185097,
|
|
0.9532010257062827,
|
|
0.8813324419214177,
|
|
1.00178063247374,
|
|
0.9232197596982638
|
|
],
|
|
"population_cv": 0.06921897980134169
|
|
},
|
|
"depth_weights": [
|
|
{
|
|
"entropy_mean": 0.0,
|
|
"mean_weights": [
|
|
1.0
|
|
],
|
|
"sources": 1
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
}
|
|
],
|
|
"layer_input_rms_by_sublayer": [
|
|
0.02802751213312149,
|
|
0.014061377383768559,
|
|
0.01413465291261673,
|
|
0.014165290631353855,
|
|
0.014225807972252369,
|
|
0.014281988143920898,
|
|
0.01433455292135477,
|
|
0.014350161887705326,
|
|
0.01440478302538395,
|
|
0.00966146495193243,
|
|
0.009703479707241058,
|
|
0.009741888381540775,
|
|
0.00977881159633398,
|
|
0.009820535778999329,
|
|
0.009858808480203152,
|
|
0.00991273857653141,
|
|
0.009950309991836548,
|
|
0.0074927168898284435,
|
|
0.00752287870272994,
|
|
0.007546746172010899,
|
|
0.007571155205368996,
|
|
0.00766373984515667,
|
|
0.007693085819482803,
|
|
0.007762390188872814,
|
|
0.007788818795233965,
|
|
0.006325984839349985,
|
|
0.006353048607707024,
|
|
0.006375366821885109,
|
|
0.006389216054230928,
|
|
0.006445344537496567,
|
|
0.0064707002602517605,
|
|
0.006555322092026472,
|
|
0.00656962301582098,
|
|
0.005531130358576775,
|
|
0.005553483963012695,
|
|
0.005611324217170477,
|
|
0.0056249531917274,
|
|
0.0056777093559503555,
|
|
0.005688921548426151,
|
|
0.00574630219489336,
|
|
0.005768724251538515,
|
|
0.004946626722812653,
|
|
0.004966245498508215,
|
|
0.0050392793491482735,
|
|
0.005057469941675663,
|
|
0.005084683652967215,
|
|
0.005098348017781973,
|
|
0.005203457549214363,
|
|
0.005216081161051989,
|
|
0.004667261149734259,
|
|
0.0046758754178881645,
|
|
0.004717131145298481,
|
|
0.004727107938379049,
|
|
0.004768412560224533,
|
|
0.004777040798217058,
|
|
0.004826710093766451,
|
|
0.004837613552808762,
|
|
0.0044052014127373695,
|
|
0.0044186655431985855,
|
|
0.004504534415900707,
|
|
0.004510571248829365,
|
|
0.004574235528707504,
|
|
0.0045849340967834,
|
|
0.004644096828997135
|
|
],
|
|
"loss_nats": 5.516995429992676,
|
|
"loss_scale": 1.0,
|
|
"output_weights": {
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
},
|
|
"step": 0,
|
|
"stream_state_rms_by_sublayer": [
|
|
0.0022265110164880753,
|
|
0.0034885562490671873,
|
|
0.0040087271481752396,
|
|
0.004831995815038681,
|
|
0.005379301495850086,
|
|
0.005952076520770788,
|
|
0.006410556845366955,
|
|
0.006925336085259914,
|
|
0.0027253651060163975,
|
|
0.00385080324485898,
|
|
0.004838948138058186,
|
|
0.005542940925806761,
|
|
0.006175771821290255,
|
|
0.006729288958013058,
|
|
0.00742313964292407,
|
|
0.007927131839096546,
|
|
0.0030937574338167906,
|
|
0.004081308841705322,
|
|
0.005150026176124811,
|
|
0.005811207927763462,
|
|
0.007019025273621082,
|
|
0.007534808944910765,
|
|
0.008143599145114422,
|
|
0.008581922389566898,
|
|
0.004255065228790045,
|
|
0.00499920267611742,
|
|
0.0066224560141563416,
|
|
0.007102519739419222,
|
|
0.008129785768687725,
|
|
0.008542952127754688,
|
|
0.010099578648805618,
|
|
0.010447698645293713,
|
|
0.0041999719105660915,
|
|
0.005041283555328846,
|
|
0.00719039561226964,
|
|
0.007636935915797949,
|
|
0.009030314162373543,
|
|
0.009387043304741383,
|
|
0.010736005380749702,
|
|
0.011020516976714134,
|
|
0.005131533369421959,
|
|
0.005807018838822842,
|
|
0.00787449348717928,
|
|
0.008367395959794521,
|
|
0.009912566281855106,
|
|
0.01024632342159748,
|
|
0.012113703414797783,
|
|
0.012378672137856483,
|
|
0.0059690922498703,
|
|
0.006453148555010557,
|
|
0.00876442901790142,
|
|
0.009127611294388771,
|
|
0.010806876234710217,
|
|
0.011037957854568958,
|
|
0.012353328987956047,
|
|
0.012611051090061665,
|
|
0.006502739619463682,
|
|
0.007010402157902718,
|
|
0.009470963850617409,
|
|
0.009794782847166061,
|
|
0.012123330496251583,
|
|
0.012445803731679916,
|
|
0.01384514570236206,
|
|
0.014087481424212456
|
|
]
|
|
},
|
|
{
|
|
"activation_grad_rms_by_block": [
|
|
3.934350024792366e-05,
|
|
3.887611092068255e-05,
|
|
3.842553996946663e-05,
|
|
3.800440754275769e-05,
|
|
3.7755999073851854e-05,
|
|
3.7545447412412614e-05,
|
|
3.739710518857464e-05,
|
|
3.724786438397132e-05,
|
|
3.721602843143046e-05,
|
|
3.7137662729946896e-05,
|
|
3.714848207891919e-05,
|
|
3.708315853145905e-05,
|
|
3.6884568544337526e-05,
|
|
3.692537211463787e-05,
|
|
3.689494769787416e-05,
|
|
3.690824451041408e-05,
|
|
3.717643267009407e-05,
|
|
3.726343493326567e-05,
|
|
3.729622403625399e-05,
|
|
3.737024235306308e-05,
|
|
3.7416841223603114e-05,
|
|
3.743126580957323e-05,
|
|
3.7427271308843046e-05,
|
|
3.7452777178259566e-05,
|
|
3.766144436667673e-05,
|
|
3.764339999179356e-05,
|
|
3.7673275073757395e-05,
|
|
3.7651498132618144e-05,
|
|
3.7516889278776944e-05,
|
|
3.752906195586547e-05,
|
|
3.751005351659842e-05,
|
|
3.7569901905953884e-05
|
|
],
|
|
"activation_grad_statistics": {
|
|
"first_quartile_mean": 3.807449684245512e-05,
|
|
"first_to_last_ratio": 1.0127693432316311,
|
|
"imbalance_abs_log_ratio": 0.012688502628590252,
|
|
"last_quartile_mean": 3.759444052775507e-05,
|
|
"mean": 3.7512014159801765e-05,
|
|
"normalized": [
|
|
1.0488239869050948,
|
|
1.036364263328269,
|
|
1.0243528861386442,
|
|
1.0131262848445919,
|
|
1.006504180580939,
|
|
1.0008912678607025,
|
|
0.9969367421664534,
|
|
0.99295826199294,
|
|
0.9921095751587637,
|
|
0.9900204924145068,
|
|
0.9903089159826522,
|
|
0.9885675126236682,
|
|
0.9832734757245689,
|
|
0.9843612224429007,
|
|
0.9835501645073257,
|
|
0.9839046326114184,
|
|
0.9910540263639774,
|
|
0.9933733436579241,
|
|
0.994247439696826,
|
|
0.9962206293126586,
|
|
0.9974628678749903,
|
|
0.9978474003052851,
|
|
0.9977409144015111,
|
|
0.9984208530821659,
|
|
1.0039835292831356,
|
|
1.0035025000639017,
|
|
1.0042989137631655,
|
|
1.0037183813223725,
|
|
1.0001299615359072,
|
|
1.0004544622954952,
|
|
0.9999477329264433,
|
|
1.0015431788308011
|
|
],
|
|
"population_cv": 0.013994511621676028
|
|
},
|
|
"activation_output_rms_by_block": [
|
|
0.003752076532691717,
|
|
0.005621279589831829,
|
|
0.007651050575077534,
|
|
0.009552945382893085,
|
|
0.005728233605623245,
|
|
0.00936705432832241,
|
|
0.013376436196267605,
|
|
0.017608575522899628,
|
|
0.008001524023711681,
|
|
0.01315119955688715,
|
|
0.019228288903832436,
|
|
0.02450871281325817,
|
|
0.011544129811227322,
|
|
0.016811303794384003,
|
|
0.021131416782736778,
|
|
0.02597878687083721,
|
|
0.010270251892507076,
|
|
0.016414040699601173,
|
|
0.021650852635502815,
|
|
0.026260782033205032,
|
|
0.011164487339556217,
|
|
0.016113370656967163,
|
|
0.019974395632743835,
|
|
0.024847183376550674,
|
|
0.012743011116981506,
|
|
0.016761260107159615,
|
|
0.022224903106689453,
|
|
0.026317164301872253,
|
|
0.010263347066938877,
|
|
0.016214758157730103,
|
|
0.020621437579393387,
|
|
0.025286227464675903
|
|
],
|
|
"activation_output_statistics": {
|
|
"first_quartile_mean": 0.009082206466700882,
|
|
"first_to_last_ratio": 0.48299297446670986,
|
|
"imbalance_abs_log_ratio": 0.7277531710529308,
|
|
"last_quartile_mean": 0.018804013612680137,
|
|
"mean": 0.015941890233079903,
|
|
"normalized": [
|
|
0.23535957642626626,
|
|
0.3526106068756831,
|
|
0.4799337132055629,
|
|
0.5992354258637684,
|
|
0.3593195989856327,
|
|
0.5875748854979249,
|
|
0.8390746643400603,
|
|
1.1045475326609202,
|
|
0.5019181481445831,
|
|
0.8249460612643045,
|
|
1.2061486199380018,
|
|
1.5373780934962062,
|
|
0.7241380816481162,
|
|
1.0545364162337563,
|
|
1.325527680455888,
|
|
1.6295926324301522,
|
|
0.6442304985387488,
|
|
1.0296169688548942,
|
|
1.358110758445485,
|
|
1.647281573844557,
|
|
0.7003239375208825,
|
|
1.010756592937231,
|
|
1.2529502675470918,
|
|
1.5586096136198466,
|
|
0.7993412908175327,
|
|
1.0513972848953317,
|
|
1.3941196923167938,
|
|
1.650818310570433,
|
|
0.6437973738924712,
|
|
1.01711640970178,
|
|
1.2935377974566205,
|
|
1.5861498915734735
|
|
],
|
|
"population_cv": 0.416946448324543
|
|
},
|
|
"bits_per_byte": 6.717052098243545,
|
|
"branch_output_rms_by_sublayer": [
|
|
0.002443928038701415,
|
|
0.0027545003686100245,
|
|
0.002620922401547432,
|
|
0.002763453871011734,
|
|
0.003418463747948408,
|
|
0.0027414956130087376,
|
|
0.0035784339997917414,
|
|
0.0027198803145438433,
|
|
0.004773888736963272,
|
|
0.0028043061029165983,
|
|
0.005374820902943611,
|
|
0.002722355304285884,
|
|
0.00676629226654768,
|
|
0.0027789603918790817,
|
|
0.007075758185237646,
|
|
0.002833461854606867,
|
|
0.006995974574238062,
|
|
0.0029154156800359488,
|
|
0.008517388254404068,
|
|
0.002774328924715519,
|
|
0.009080596268177032,
|
|
0.0028916397131979465,
|
|
0.009109988808631897,
|
|
0.0032102405093610287,
|
|
0.010673855431377888,
|
|
0.002877794438973069,
|
|
0.009562615305185318,
|
|
0.0026486723218113184,
|
|
0.008646698668599129,
|
|
0.0031627838034182787,
|
|
0.009536299854516983,
|
|
0.0030164923518896103,
|
|
0.009432275779545307,
|
|
0.0027947286143898964,
|
|
0.01090320572257042,
|
|
0.0031009595841169357,
|
|
0.010227161459624767,
|
|
0.002871215809136629,
|
|
0.008522190153598785,
|
|
0.003154836595058441,
|
|
0.010072597302496433,
|
|
0.0032395559828728437,
|
|
0.009548105299472809,
|
|
0.0029102955013513565,
|
|
0.010121389292180538,
|
|
0.0026868272107094526,
|
|
0.009201278910040855,
|
|
0.0029280087910592556,
|
|
0.01199205219745636,
|
|
0.0029097299557179213,
|
|
0.008808780461549759,
|
|
0.0027156553696841,
|
|
0.010769994929432869,
|
|
0.0031467543449252844,
|
|
0.010386301204562187,
|
|
0.003253829199820757,
|
|
0.009710980579257011,
|
|
0.0025951166171580553,
|
|
0.009926634840667248,
|
|
0.003180553438141942,
|
|
0.010113959200680256,
|
|
0.0028141052462160587,
|
|
0.009518159553408623,
|
|
0.0030968233477324247
|
|
],
|
|
"capture": {
|
|
"all_gradients_finite": true,
|
|
"all_gradients_present": true,
|
|
"count": 32,
|
|
"dtypes": [
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32"
|
|
],
|
|
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
|
"shape": [
|
|
16,
|
|
256,
|
|
192
|
|
],
|
|
"storage_unique": true
|
|
},
|
|
"core_parameter_grad_rms_by_block": [
|
|
0.00016737460466537552,
|
|
0.00016955889217530398,
|
|
0.0001941533654368659,
|
|
0.00020677099590829626,
|
|
0.000241377514962491,
|
|
0.00026501239176498695,
|
|
0.00032262838018369365,
|
|
0.00031000316290161927,
|
|
0.0003190796766484002,
|
|
0.00034607256602154805,
|
|
0.0003482012935130775,
|
|
0.00036880192147718024,
|
|
0.00040270042689994603,
|
|
0.0003782755449508064,
|
|
0.00035486502238655745,
|
|
0.0003672093084735214,
|
|
0.00036789155274553697,
|
|
0.0003751571652154002,
|
|
0.0003805970858161285,
|
|
0.00035647119760667416,
|
|
0.00036940450439703476,
|
|
0.00035111776542448423,
|
|
0.00035831874545173094,
|
|
0.000360763035713257,
|
|
0.0004080638981985186,
|
|
0.0003616992623202981,
|
|
0.00038377302017183443,
|
|
0.00035823205404444226,
|
|
0.0003532695727353617,
|
|
0.0003701571311252852,
|
|
0.0003662503537512869,
|
|
0.0003823335970855767
|
|
],
|
|
"core_parameter_grad_statistics": {
|
|
"first_quartile_mean": 0.00023460991349982907,
|
|
"first_to_last_ratio": 0.6290276114781215,
|
|
"imbalance_abs_log_ratio": 0.46358012582366337,
|
|
"last_quartile_mean": 0.0003729723611790755,
|
|
"mean": 0.0003332995315678913,
|
|
"normalized": [
|
|
0.5021747371741573,
|
|
0.5087282642662994,
|
|
0.5825191668393267,
|
|
0.6203758971265705,
|
|
0.7242059832098016,
|
|
0.7951178044515355,
|
|
0.9679832991843735,
|
|
0.9301038061569352,
|
|
0.9573361088970069,
|
|
1.0383229895150783,
|
|
1.0447098196480689,
|
|
1.1065179712143018,
|
|
1.208223800992406,
|
|
1.1349417239542496,
|
|
1.0647030336863026,
|
|
1.101739647656009,
|
|
1.1037865880426523,
|
|
1.125585635991159,
|
|
1.1419070528714588,
|
|
1.0695220480202294,
|
|
1.1083259033077553,
|
|
1.0534601227093638,
|
|
1.0750652536658107,
|
|
1.0823988681177357,
|
|
1.224315846706788,
|
|
1.085207832782753,
|
|
1.1514358222062546,
|
|
1.0748051530683667,
|
|
1.0599161993223583,
|
|
1.1105840124767359,
|
|
1.0988624917304564,
|
|
1.1471171150076982
|
|
],
|
|
"population_cv": 0.19654387017806715
|
|
},
|
|
"depth_weights": [
|
|
{
|
|
"entropy_mean": 0.0,
|
|
"mean_weights": [
|
|
1.0
|
|
],
|
|
"sources": 1
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931430101394653,
|
|
"mean_weights": [
|
|
0.4986528158187866,
|
|
0.5013471841812134
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931370496749878,
|
|
"mean_weights": [
|
|
0.4977952837944031,
|
|
0.5022047162055969
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931354999542236,
|
|
"mean_weights": [
|
|
0.497627854347229,
|
|
0.502372145652771
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931448578834534,
|
|
"mean_weights": [
|
|
0.49909159541130066,
|
|
0.500908374786377
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931440234184265,
|
|
"mean_weights": [
|
|
0.49918830394744873,
|
|
0.5008116364479065
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931461691856384,
|
|
"mean_weights": [
|
|
0.49945223331451416,
|
|
0.5005477666854858
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931438446044922,
|
|
"mean_weights": [
|
|
0.4989117980003357,
|
|
0.5010882019996643
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931412220001221,
|
|
"mean_weights": [
|
|
0.498369038105011,
|
|
0.501630961894989
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986061096191406,
|
|
"mean_weights": [
|
|
0.33209967613220215,
|
|
0.33376386761665344,
|
|
0.334136426448822
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0985990762710571,
|
|
"mean_weights": [
|
|
0.33337461948394775,
|
|
0.3353745937347412,
|
|
0.33125078678131104
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986008644104004,
|
|
"mean_weights": [
|
|
0.33504414558410645,
|
|
0.33362600207328796,
|
|
0.3313298523426056
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986016988754272,
|
|
"mean_weights": [
|
|
0.33131715655326843,
|
|
0.33385908603668213,
|
|
0.33482375741004944
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986084938049316,
|
|
"mean_weights": [
|
|
0.3345443904399872,
|
|
0.3326132297515869,
|
|
0.3328423798084259
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0985959768295288,
|
|
"mean_weights": [
|
|
0.331234872341156,
|
|
0.33299222588539124,
|
|
0.33577287197113037
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986058712005615,
|
|
"mean_weights": [
|
|
0.333141565322876,
|
|
0.3320317268371582,
|
|
0.3348267078399658
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986061096191406,
|
|
"mean_weights": [
|
|
0.3325701653957367,
|
|
0.33248791098594666,
|
|
0.33494192361831665
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862926959991455,
|
|
"mean_weights": [
|
|
0.24962776899337769,
|
|
0.2496623694896698,
|
|
0.25036606192588806,
|
|
0.25034379959106445
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862905502319336,
|
|
"mean_weights": [
|
|
0.2491619884967804,
|
|
0.24970892071723938,
|
|
0.25086259841918945,
|
|
0.25026649236679077
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862870931625366,
|
|
"mean_weights": [
|
|
0.25067847967147827,
|
|
0.24852469563484192,
|
|
0.24991153180599213,
|
|
0.2508852779865265
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862855434417725,
|
|
"mean_weights": [
|
|
0.24860671162605286,
|
|
0.24958062171936035,
|
|
0.2513701319694519,
|
|
0.2504425346851349
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.386291265487671,
|
|
"mean_weights": [
|
|
0.25055184960365295,
|
|
0.24973732233047485,
|
|
0.2504110336303711,
|
|
0.2492997944355011
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862907886505127,
|
|
"mean_weights": [
|
|
0.24893227219581604,
|
|
0.25016486644744873,
|
|
0.25030890107154846,
|
|
0.25059396028518677
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862874507904053,
|
|
"mean_weights": [
|
|
0.24930796027183533,
|
|
0.24942311644554138,
|
|
0.24996015429496765,
|
|
0.25130876898765564
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862826824188232,
|
|
"mean_weights": [
|
|
0.24865901470184326,
|
|
0.24900351464748383,
|
|
0.25141385197639465,
|
|
0.25092363357543945
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094211339950562,
|
|
"mean_weights": [
|
|
0.19921651482582092,
|
|
0.19924485683441162,
|
|
0.1988956332206726,
|
|
0.2007521539926529,
|
|
0.20189084112644196
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.609428882598877,
|
|
"mean_weights": [
|
|
0.19964434206485748,
|
|
0.19919463992118835,
|
|
0.19919714331626892,
|
|
0.20121338963508606,
|
|
0.200750470161438
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094247102737427,
|
|
"mean_weights": [
|
|
0.20065948367118835,
|
|
0.19955207407474518,
|
|
0.20149102807044983,
|
|
0.19979192316532135,
|
|
0.19850550591945648
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094355583190918,
|
|
"mean_weights": [
|
|
0.19937850534915924,
|
|
0.20006614923477173,
|
|
0.20067325234413147,
|
|
0.19996050000190735,
|
|
0.19992157816886902
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094313859939575,
|
|
"mean_weights": [
|
|
0.1997273862361908,
|
|
0.1999678611755371,
|
|
0.1992817223072052,
|
|
0.19971339404582977,
|
|
0.20130965113639832
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094284057617188,
|
|
"mean_weights": [
|
|
0.19933168590068817,
|
|
0.19993484020233154,
|
|
0.19907662272453308,
|
|
0.20011654496192932,
|
|
0.2015402913093567
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094276905059814,
|
|
"mean_weights": [
|
|
0.19931891560554504,
|
|
0.19971159100532532,
|
|
0.20119035243988037,
|
|
0.20086422562599182,
|
|
0.19891491532325745
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094303131103516,
|
|
"mean_weights": [
|
|
0.19955140352249146,
|
|
0.19991253316402435,
|
|
0.20052893459796906,
|
|
0.20107696950435638,
|
|
0.19893015921115875
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.791748046875,
|
|
"mean_weights": [
|
|
0.1663031131029129,
|
|
0.1665959358215332,
|
|
0.1656446009874344,
|
|
0.16612613201141357,
|
|
0.16804239153862,
|
|
0.16728781163692474
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7917547225952148,
|
|
"mean_weights": [
|
|
0.16606608033180237,
|
|
0.16676852107048035,
|
|
0.1667691171169281,
|
|
0.165959894657135,
|
|
0.16733849048614502,
|
|
0.16709789633750916
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791755199432373,
|
|
"mean_weights": [
|
|
0.1671326905488968,
|
|
0.166437566280365,
|
|
0.16707676649093628,
|
|
0.16582348942756653,
|
|
0.1670762449502945,
|
|
0.1664532721042633
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791752815246582,
|
|
"mean_weights": [
|
|
0.16595980525016785,
|
|
0.1665806621313095,
|
|
0.16717451810836792,
|
|
0.16712555289268494,
|
|
0.16733399033546448,
|
|
0.1658255010843277
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7917518615722656,
|
|
"mean_weights": [
|
|
0.16663867235183716,
|
|
0.16675058007240295,
|
|
0.16638454794883728,
|
|
0.16550561785697937,
|
|
0.1674773097038269,
|
|
0.16724327206611633
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7917499542236328,
|
|
"mean_weights": [
|
|
0.16634856164455414,
|
|
0.16704332828521729,
|
|
0.16620689630508423,
|
|
0.1655295491218567,
|
|
0.16771358251571655,
|
|
0.1671580672264099
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7917507886886597,
|
|
"mean_weights": [
|
|
0.16621428728103638,
|
|
0.16654521226882935,
|
|
0.16585756838321686,
|
|
0.1672719269990921,
|
|
0.1677929311990738,
|
|
0.16631805896759033
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.07942795753479,
|
|
"mean_weights": [
|
|
0.12488093227148056,
|
|
0.12459862232208252,
|
|
0.12424520403146744,
|
|
0.12444067001342773,
|
|
0.12461253255605698,
|
|
0.12524709105491638,
|
|
0.12623067200183868,
|
|
0.1257442682981491
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.1972174644470215,
|
|
"mean_weights": [
|
|
0.11096031218767166,
|
|
0.11101566255092621,
|
|
0.11069408804178238,
|
|
0.11131208389997482,
|
|
0.11191314458847046,
|
|
0.11117817461490631,
|
|
0.11035752296447754,
|
|
0.11126880347728729,
|
|
0.11130021512508392
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.197216749191284,
|
|
"mean_weights": [
|
|
0.11095122992992401,
|
|
0.11080076545476913,
|
|
0.11054693907499313,
|
|
0.11075648665428162,
|
|
0.11089976876974106,
|
|
0.11123757809400558,
|
|
0.11190937459468842,
|
|
0.1117669939994812,
|
|
0.11113084852695465
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.197216510772705,
|
|
"mean_weights": [
|
|
0.11096195876598358,
|
|
0.11111634969711304,
|
|
0.11043959856033325,
|
|
0.11095423996448517,
|
|
0.11190803349018097,
|
|
0.11175134778022766,
|
|
0.11079943180084229,
|
|
0.1112256795167923,
|
|
0.11084336042404175
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.197213649749756,
|
|
"mean_weights": [
|
|
0.11101563274860382,
|
|
0.1110072135925293,
|
|
0.11082680523395538,
|
|
0.11142624914646149,
|
|
0.1101548969745636,
|
|
0.11108594387769699,
|
|
0.11191216111183167,
|
|
0.11077424138784409,
|
|
0.11179685592651367
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.197218894958496,
|
|
"mean_weights": [
|
|
0.11101803183555603,
|
|
0.11099022626876831,
|
|
0.11046922951936722,
|
|
0.11147402226924896,
|
|
0.11091943085193634,
|
|
0.11164889484643936,
|
|
0.11152636259794235,
|
|
0.11112086474895477,
|
|
0.11083294451236725
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.197216033935547,
|
|
"mean_weights": [
|
|
0.1111505776643753,
|
|
0.11169322580099106,
|
|
0.11175311356782913,
|
|
0.11086191982030869,
|
|
0.11131928861141205,
|
|
0.11068505048751831,
|
|
0.11144685000181198,
|
|
0.11033272743225098,
|
|
0.1107572466135025
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1972179412841797,
|
|
"mean_weights": [
|
|
0.11092513799667358,
|
|
0.11108554899692535,
|
|
0.11051031202077866,
|
|
0.11084482073783875,
|
|
0.11164578050374985,
|
|
0.11118975281715393,
|
|
0.11111322790384293,
|
|
0.11183710396289825,
|
|
0.1108483225107193
|
|
],
|
|
"sources": 9
|
|
}
|
|
],
|
|
"layer_input_rms_by_sublayer": [
|
|
0.028025340288877487,
|
|
0.014066023752093315,
|
|
0.014108708128333092,
|
|
0.014229794964194298,
|
|
0.014309127815067768,
|
|
0.01455320231616497,
|
|
0.014605770818889141,
|
|
0.01481270045042038,
|
|
0.014851206913590431,
|
|
0.010233239270746708,
|
|
0.010343058034777641,
|
|
0.010804706253111362,
|
|
0.010769513435661793,
|
|
0.01144309900701046,
|
|
0.011491727083921432,
|
|
0.012276813387870789,
|
|
0.012374046258628368,
|
|
0.009844657965004444,
|
|
0.009941100142896175,
|
|
0.010566700249910355,
|
|
0.010642717592418194,
|
|
0.011646243743598461,
|
|
0.011756961233913898,
|
|
0.012673276476562023,
|
|
0.012869661673903465,
|
|
0.011087263002991676,
|
|
0.01123880036175251,
|
|
0.011723686009645462,
|
|
0.0118045499548316,
|
|
0.012471769005060196,
|
|
0.012632780708372593,
|
|
0.013105461373925209,
|
|
0.013265169225633144,
|
|
0.011508514173328876,
|
|
0.0116264121606946,
|
|
0.012281306087970734,
|
|
0.012384527362883091,
|
|
0.012981628999114037,
|
|
0.013078375719487667,
|
|
0.013564801774919033,
|
|
0.013736819848418236,
|
|
0.012090411968529224,
|
|
0.01221960224211216,
|
|
0.012518657371401787,
|
|
0.012597586959600449,
|
|
0.013060149736702442,
|
|
0.013144387863576412,
|
|
0.013466374948620796,
|
|
0.013602014631032944,
|
|
0.012321850284934044,
|
|
0.012410826981067657,
|
|
0.012648497708141804,
|
|
0.012738197110593319,
|
|
0.013148728758096695,
|
|
0.013268771581351757,
|
|
0.013453229330480099,
|
|
0.013569011352956295,
|
|
0.01241307333111763,
|
|
0.012489519082009792,
|
|
0.012747267261147499,
|
|
0.012816172093153,
|
|
0.01302917767316103,
|
|
0.013102042488753796,
|
|
0.013313685543835163
|
|
],
|
|
"loss_nats": 4.655905723571777,
|
|
"loss_scale": 1.0,
|
|
"output_weights": {
|
|
"entropy_mean": 2.197218418121338,
|
|
"mean_weights": [
|
|
0.11057741940021515,
|
|
0.1106872707605362,
|
|
0.11082567274570465,
|
|
0.11127195507287979,
|
|
0.11091846227645874,
|
|
0.11130088567733765,
|
|
0.11114984005689621,
|
|
0.11173895001411438,
|
|
0.11152952909469604
|
|
],
|
|
"sources": 9
|
|
},
|
|
"step": 20,
|
|
"stream_state_rms_by_sublayer": [
|
|
0.002443928038701415,
|
|
0.003752076532691717,
|
|
0.004733026959002018,
|
|
0.005621279589831829,
|
|
0.007003156002610922,
|
|
0.007651050575077534,
|
|
0.008999467827379704,
|
|
0.009552945382893085,
|
|
0.004773888736963272,
|
|
0.005728233605623245,
|
|
0.008736123330891132,
|
|
0.00936705432832241,
|
|
0.012760105542838573,
|
|
0.013376436196267605,
|
|
0.016963720321655273,
|
|
0.017608575522899628,
|
|
0.006995974574238062,
|
|
0.008001524023711681,
|
|
0.012580735608935356,
|
|
0.01315119955688715,
|
|
0.0186475720256567,
|
|
0.019228288903832436,
|
|
0.02346930466592312,
|
|
0.02450871281325817,
|
|
0.010673855431377888,
|
|
0.011544129811227322,
|
|
0.016316605731844902,
|
|
0.016811303794384003,
|
|
0.020413896068930626,
|
|
0.021131416782736778,
|
|
0.024899384006857872,
|
|
0.02597878687083721,
|
|
0.009432275779545307,
|
|
0.010270251892507076,
|
|
0.015730082988739014,
|
|
0.016414040699601173,
|
|
0.020732076838612556,
|
|
0.021650852635502815,
|
|
0.025274164974689484,
|
|
0.026260782033205032,
|
|
0.010072597302496433,
|
|
0.011164487339556217,
|
|
0.015665393322706223,
|
|
0.016113370656967163,
|
|
0.019619770348072052,
|
|
0.019974395632743835,
|
|
0.02384515292942524,
|
|
0.024847183376550674,
|
|
0.01199205219745636,
|
|
0.012743011116981506,
|
|
0.0159013532102108,
|
|
0.016761260107159615,
|
|
0.021420039236545563,
|
|
0.022224903106689453,
|
|
0.025291141122579575,
|
|
0.026317164301872253,
|
|
0.009710980579257011,
|
|
0.010263347066938877,
|
|
0.01540068257600069,
|
|
0.016214758157730103,
|
|
0.019683225080370903,
|
|
0.020621437579393387,
|
|
0.024622956290841103,
|
|
0.025286227464675903
|
|
]
|
|
}
|
|
],
|
|
"environment": {
|
|
"autocast": "cuda-bfloat16-forward-fp32-cross-entropy",
|
|
"compile": false,
|
|
"compute_capability": [
|
|
12,
|
|
0
|
|
],
|
|
"cublas_workspace_config": ":4096:8",
|
|
"cuda": "12.8",
|
|
"deterministic_algorithms": true,
|
|
"gpu": "NVIDIA GeForce RTX 5090",
|
|
"python": "3.10.14",
|
|
"torch": "2.11.0+cu128"
|
|
},
|
|
"evaluations": [
|
|
{
|
|
"bits_per_byte": 7.9557801867581395,
|
|
"cross_entropy_nats": 5.514526605606079,
|
|
"step": 0
|
|
},
|
|
{
|
|
"bits_per_byte": 6.720469307524863,
|
|
"cross_entropy_nats": 4.658274352550507,
|
|
"step": 20
|
|
}
|
|
],
|
|
"forward_intervention": {
|
|
"depth_visit_counts": [
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39
|
|
],
|
|
"forward_calls": 39,
|
|
"output_mixer_selected": false,
|
|
"output_visit_count": 39,
|
|
"passed": true,
|
|
"selected_depth_indices": [
|
|
40,
|
|
41,
|
|
42,
|
|
43,
|
|
44,
|
|
45,
|
|
46,
|
|
47,
|
|
48,
|
|
49,
|
|
50,
|
|
51,
|
|
52,
|
|
53,
|
|
54,
|
|
55
|
|
],
|
|
"selected_parameter_reachability_gate": true,
|
|
"selected_parameters": {
|
|
"mixers.40.key_norm.weight": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.40.query": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.41.key_norm.weight": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.41.query": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.42.key_norm.weight": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.42.query": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.43.key_norm.weight": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.43.query": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.44.key_norm.weight": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.44.query": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.45.key_norm.weight": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.45.query": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.46.key_norm.weight": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.46.query": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.47.key_norm.weight": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.47.query": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.48.key_norm.weight": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.48.query": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.49.key_norm.weight": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.49.query": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.50.key_norm.weight": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.50.query": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.51.key_norm.weight": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.51.query": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.52.key_norm.weight": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.52.query": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.53.key_norm.weight": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.53.query": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.54.key_norm.weight": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.54.query": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.55.key_norm.weight": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
},
|
|
"mixers.55.query": {
|
|
"final_equals_initial": true,
|
|
"gradient_hook_calls": 0,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": false
|
|
}
|
|
},
|
|
"selected_source_count_checks": {
|
|
"40": true,
|
|
"41": true,
|
|
"42": true,
|
|
"43": true,
|
|
"44": true,
|
|
"45": true,
|
|
"46": true,
|
|
"47": true,
|
|
"48": true,
|
|
"49": true,
|
|
"50": true,
|
|
"51": true,
|
|
"52": true,
|
|
"53": true,
|
|
"54": true,
|
|
"55": true
|
|
},
|
|
"selector_gate": true,
|
|
"semantics": "selected depth mixers use parameter-free constant-zero logits with the parent softmax+einsum arithmetic kernel",
|
|
"source_counts_by_depth_index": {
|
|
"0": [
|
|
1
|
|
],
|
|
"1": [
|
|
2
|
|
],
|
|
"10": [
|
|
3
|
|
],
|
|
"11": [
|
|
3
|
|
],
|
|
"12": [
|
|
3
|
|
],
|
|
"13": [
|
|
3
|
|
],
|
|
"14": [
|
|
3
|
|
],
|
|
"15": [
|
|
3
|
|
],
|
|
"16": [
|
|
3
|
|
],
|
|
"17": [
|
|
4
|
|
],
|
|
"18": [
|
|
4
|
|
],
|
|
"19": [
|
|
4
|
|
],
|
|
"2": [
|
|
2
|
|
],
|
|
"20": [
|
|
4
|
|
],
|
|
"21": [
|
|
4
|
|
],
|
|
"22": [
|
|
4
|
|
],
|
|
"23": [
|
|
4
|
|
],
|
|
"24": [
|
|
4
|
|
],
|
|
"25": [
|
|
5
|
|
],
|
|
"26": [
|
|
5
|
|
],
|
|
"27": [
|
|
5
|
|
],
|
|
"28": [
|
|
5
|
|
],
|
|
"29": [
|
|
5
|
|
],
|
|
"3": [
|
|
2
|
|
],
|
|
"30": [
|
|
5
|
|
],
|
|
"31": [
|
|
5
|
|
],
|
|
"32": [
|
|
5
|
|
],
|
|
"33": [
|
|
6
|
|
],
|
|
"34": [
|
|
6
|
|
],
|
|
"35": [
|
|
6
|
|
],
|
|
"36": [
|
|
6
|
|
],
|
|
"37": [
|
|
6
|
|
],
|
|
"38": [
|
|
6
|
|
],
|
|
"39": [
|
|
6
|
|
],
|
|
"4": [
|
|
2
|
|
],
|
|
"40": [
|
|
6
|
|
],
|
|
"41": [
|
|
7
|
|
],
|
|
"42": [
|
|
7
|
|
],
|
|
"43": [
|
|
7
|
|
],
|
|
"44": [
|
|
7
|
|
],
|
|
"45": [
|
|
7
|
|
],
|
|
"46": [
|
|
7
|
|
],
|
|
"47": [
|
|
7
|
|
],
|
|
"48": [
|
|
7
|
|
],
|
|
"49": [
|
|
8
|
|
],
|
|
"5": [
|
|
2
|
|
],
|
|
"50": [
|
|
8
|
|
],
|
|
"51": [
|
|
8
|
|
],
|
|
"52": [
|
|
8
|
|
],
|
|
"53": [
|
|
8
|
|
],
|
|
"54": [
|
|
8
|
|
],
|
|
"55": [
|
|
8
|
|
],
|
|
"56": [
|
|
8
|
|
],
|
|
"57": [
|
|
9
|
|
],
|
|
"58": [
|
|
9
|
|
],
|
|
"59": [
|
|
9
|
|
],
|
|
"6": [
|
|
2
|
|
],
|
|
"60": [
|
|
9
|
|
],
|
|
"61": [
|
|
9
|
|
],
|
|
"62": [
|
|
9
|
|
],
|
|
"63": [
|
|
9
|
|
],
|
|
"7": [
|
|
2
|
|
],
|
|
"8": [
|
|
2
|
|
],
|
|
"9": [
|
|
3
|
|
]
|
|
},
|
|
"uniform_weight_gate": true,
|
|
"uniform_weight_max_abs_error": 0.0,
|
|
"uniform_weight_threshold": 1e-12,
|
|
"unselected_parameter_reachability_gate": true,
|
|
"unselected_parameters": {
|
|
"mixers.0.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.0.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.1.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.1.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.10.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.10.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.11.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.11.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.12.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.12.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.13.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.13.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.14.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.14.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.15.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.15.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.16.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.16.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.17.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.17.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.18.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.18.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.19.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.19.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.2.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.2.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.20.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.20.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.21.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.21.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.22.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.22.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.23.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.23.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.24.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.24.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.25.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.25.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.26.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.26.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.27.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.27.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.28.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.28.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.29.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.29.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.3.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.3.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.30.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.30.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.31.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.31.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.32.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.32.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.33.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.33.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.34.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.34.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.35.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.35.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.36.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.36.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.37.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.37.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.38.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.38.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.39.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.39.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.4.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.4.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.5.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.5.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.56.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.56.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.57.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.57.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.58.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.58.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.59.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.59.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.6.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.6.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.60.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.60.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.61.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.61.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.62.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.62.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.63.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.63.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.7.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.7.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.8.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.8.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.9.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"mixers.9.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"output_mixer.key_norm.weight": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
},
|
|
"output_mixer.query": {
|
|
"gradient_hook_calls": 23,
|
|
"in_optimizer_param_group": true,
|
|
"optimizer_state_present": true
|
|
}
|
|
},
|
|
"variant": "uniform_groups_6_7_forward",
|
|
"visit_gate": true
|
|
},
|
|
"gradient_gate": {
|
|
"first_to_last_ratio_abs_delta": 0.0,
|
|
"max_abs_scale_ratio_error": 0.0,
|
|
"normalized_spectrum_max_abs_delta": 0.0,
|
|
"passed": true,
|
|
"per_block_scale_ratios": [
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0
|
|
],
|
|
"population_cv_abs_delta": 0.0,
|
|
"thresholds": {
|
|
"scale_ratio_abs": 1e-05,
|
|
"shape_abs": 1e-06
|
|
}
|
|
},
|
|
"hashes": {
|
|
"final_mixer_parameters": "30089f6da7b0a0bb5e6cdbe490b747650c45dffe323fff195a5826f29e2bda97",
|
|
"final_model_state": "71293f52d25e18a11cac9db25eecf5c920366a19345d3d559938ba4cc52d1247",
|
|
"final_optimizer_state": "7e585c88cf40b3eb2c7b31477ccbe63c4ed526936da5da54efb9d7514f9d2e7a",
|
|
"final_public_parameters": "7688156eb65ae21c33116d50a5e129a6cb020cb4391543dccf282e44f8ab6905",
|
|
"initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d",
|
|
"initial_public_parameter_elements": 18985152,
|
|
"initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670",
|
|
"initial_public_parameter_tensors": 227,
|
|
"initial_public_parameters": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58"
|
|
},
|
|
"manifest": {
|
|
"diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716",
|
|
"file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371",
|
|
"formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
|
|
"input_gate_tensor_hashes": {
|
|
"0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434",
|
|
"1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94",
|
|
"7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc"
|
|
},
|
|
"path": "experiments/k3/attnres_gradient/manifest.json",
|
|
"validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338"
|
|
},
|
|
"model": {
|
|
"attnres_aggregation_groups": 8,
|
|
"context": 256,
|
|
"d_ff": 768,
|
|
"d_head": 32,
|
|
"d_model": 192,
|
|
"heads": 6,
|
|
"layers": 32,
|
|
"parameters": {
|
|
"core": 18985152,
|
|
"embedding": 98304,
|
|
"mixer": 24960,
|
|
"total": 19010112
|
|
},
|
|
"sublayers": 64,
|
|
"sublayers_per_attnres_group": 8,
|
|
"transformer_blocks_per_attnres_group": 4,
|
|
"vocabulary": 256
|
|
},
|
|
"optimizer": {
|
|
"betas": [
|
|
0.9,
|
|
0.95
|
|
],
|
|
"epsilon": 1e-08,
|
|
"grad_clip": 1.0,
|
|
"min_lr": 3e-05,
|
|
"name": "AdamW",
|
|
"peak_lr": 0.0003,
|
|
"warmup_steps": 400,
|
|
"weight_decay_ndim_ge_2": 0.1
|
|
},
|
|
"parent_protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
|
|
"parent_runner_canonical_sha256": "15cea1697eaa21a7245460ad114a4b39e8f1a153d60fda0b6828a1ecaa98232a",
|
|
"protocol_id": "llm-atlas-k3-attnres-forward-training-v1",
|
|
"run_kind": "smoke",
|
|
"schema_version": 2,
|
|
"seed": 2026073001,
|
|
"steps": 20,
|
|
"study_manifest": {
|
|
"file_sha256": "49546ed5baf36bcb30885062b7c671fafe4ccff2e606624cd7dbe23717f9a712",
|
|
"path": "/home/wuyang/Code/K3/experiments/k3/attnres_forward/manifest.json",
|
|
"status": "frozen-before-model-output"
|
|
},
|
|
"target_bytes_seen": 163840,
|
|
"timing": {
|
|
"mean_ms": null,
|
|
"measured_steps": 0,
|
|
"median_ms": null,
|
|
"p95_ms": null,
|
|
"peak_allocated_bytes": 5610759680,
|
|
"peak_reserved_bytes": 11068768256,
|
|
"warmup_steps_excluded": 20
|
|
},
|
|
"training_history": [
|
|
{
|
|
"bits_per_byte": 7.958777844362678,
|
|
"learning_rate": 7.499999999999999e-07,
|
|
"loss_nats": 5.516604423522949,
|
|
"step": 1,
|
|
"unclipped_grad_norm": 21.321983337402344
|
|
},
|
|
{
|
|
"bits_per_byte": 7.24805334829669,
|
|
"learning_rate": 7.499999999999999e-06,
|
|
"loss_nats": 5.023967742919922,
|
|
"step": 10,
|
|
"unclipped_grad_norm": 11.155019760131836
|
|
},
|
|
{
|
|
"bits_per_byte": 6.733684884373789,
|
|
"learning_rate": 1.4999999999999999e-05,
|
|
"loss_nats": 4.6674346923828125,
|
|
"step": 20,
|
|
"unclipped_grad_norm": 3.4893624782562256
|
|
}
|
|
],
|
|
"variant": "uniform_groups_6_7_forward"
|
|
}
|