wuyang
701 lines
20 KiB
JSON
701 lines
20 KiB
JSON
{
|
|
"architecture": "baseline",
|
|
"batch_size": 32,
|
|
"canonical_sha256_without_self": "0ae9ad1697ca15ec4c84270ad82b9fa98056f14f20fc7372c4b9886c29845415",
|
|
"depth": 16,
|
|
"diagnostics": [
|
|
{
|
|
"activation_grad_rms_by_block": [
|
|
0.0002865509013645351,
|
|
0.00024785567075014114,
|
|
0.00021551651298068464,
|
|
0.00019683866412378848,
|
|
0.0001823508064262569,
|
|
0.0001679604029050097,
|
|
0.00015584587526973337,
|
|
0.00014737028686795384,
|
|
0.00014130656199995428,
|
|
0.00013612695329356939,
|
|
0.00013129493163432926,
|
|
0.00012807638267986476,
|
|
0.00012550120300147682,
|
|
0.00012324050476308912,
|
|
0.00012108208466088399,
|
|
0.00011888353037647903
|
|
],
|
|
"activation_grad_statistics": {
|
|
"first_quartile_mean": 0.00023669043730478734,
|
|
"first_to_last_ratio": 1.9372775995887173,
|
|
"imbalance_abs_log_ratio": 0.6612836883477485,
|
|
"last_quartile_mean": 0.00012217683070048224,
|
|
"mean": 0.00016411257956860936,
|
|
"normalized": [
|
|
1.7460629899168627,
|
|
1.5102783187106137,
|
|
1.313223602646409,
|
|
1.1994124072706904,
|
|
1.1111324123086055,
|
|
1.0234462424910682,
|
|
0.9496278449793059,
|
|
0.8979828801383491,
|
|
0.8610343117596252,
|
|
0.8294729974472175,
|
|
0.8000296624393728,
|
|
0.7804178266926869,
|
|
0.7647262832098098,
|
|
0.7509509940495869,
|
|
0.7377989242455608,
|
|
0.7244023016942358
|
|
],
|
|
"population_cv": 0.29361872380036635
|
|
},
|
|
"activation_output_rms_by_block": [
|
|
0.02866268903017044,
|
|
0.029099803417921066,
|
|
0.02953805774450302,
|
|
0.030088091269135475,
|
|
0.030644793063402176,
|
|
0.03131929785013199,
|
|
0.03213750571012497,
|
|
0.03286394104361534,
|
|
0.033869802951812744,
|
|
0.03499744459986687,
|
|
0.03588006645441055,
|
|
0.037365153431892395,
|
|
0.03789033368229866,
|
|
0.039445169270038605,
|
|
0.04050002992153168,
|
|
0.04121527820825577
|
|
],
|
|
"activation_output_statistics": {
|
|
"first_quartile_mean": 0.0293471603654325,
|
|
"first_to_last_ratio": 0.7380574840395957,
|
|
"imbalance_abs_log_ratio": 0.3037335657624931,
|
|
"last_quartile_mean": 0.03976270277053118,
|
|
"mean": 0.034094841103069484,
|
|
"normalized": [
|
|
0.8406752488894867,
|
|
0.8534957922212247,
|
|
0.8663497699023965,
|
|
0.8824822259232266,
|
|
0.8988102619619861,
|
|
0.9185934539320196,
|
|
0.942591449919727,
|
|
0.9638977622528551,
|
|
0.9933996421752943,
|
|
1.0264733158329964,
|
|
1.0523605710888722,
|
|
1.0959180985456622,
|
|
1.1113216092650298,
|
|
1.1569248600043318,
|
|
1.1878638706395246,
|
|
1.2088420674453664
|
|
],
|
|
"population_cv": 0.11952987076282337
|
|
},
|
|
"bits_per_byte": 8.096463027059821,
|
|
"branch_output_rms_by_sublayer": [
|
|
0.0033055038657039404,
|
|
0.0037518907338380814,
|
|
0.0033751516602933407,
|
|
0.003911525942385197,
|
|
0.004229962360113859,
|
|
0.0038461871445178986,
|
|
0.004243654198944569,
|
|
0.0038366704247891903,
|
|
0.004372371360659599,
|
|
0.003851204412057996,
|
|
0.005078981164842844,
|
|
0.003818925702944398,
|
|
0.0054668826051056385,
|
|
0.0038749484810978174,
|
|
0.005823117680847645,
|
|
0.0037566579412668943,
|
|
0.0065947300754487514,
|
|
0.003876061411574483,
|
|
0.006763988174498081,
|
|
0.003795720636844635,
|
|
0.006863070651888847,
|
|
0.0038706199266016483,
|
|
0.007856340147554874,
|
|
0.003839300014078617,
|
|
0.00830968376249075,
|
|
0.003927029203623533,
|
|
0.009655521251261234,
|
|
0.0038513424806296825,
|
|
0.009004893712699413,
|
|
0.0039484030567109585,
|
|
0.008608299307525158,
|
|
0.003791053779423237
|
|
],
|
|
"capture": {
|
|
"all_gradients_finite": true,
|
|
"all_gradients_present": true,
|
|
"count": 16,
|
|
"dtypes": [
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32"
|
|
],
|
|
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
|
"shape": [
|
|
16,
|
|
256,
|
|
192
|
|
],
|
|
"storage_unique": true
|
|
},
|
|
"core_parameter_grad_rms_by_block": [
|
|
0.00809059897248305,
|
|
0.007313812062277777,
|
|
0.007090710224412361,
|
|
0.0067033738367094624,
|
|
0.006535407867454384,
|
|
0.006785021795736036,
|
|
0.006295993569919972,
|
|
0.005509300326230647,
|
|
0.006231736048910802,
|
|
0.006047390980174007,
|
|
0.005629405359532731,
|
|
0.005784042737956458,
|
|
0.0059591736113607996,
|
|
0.0061983173180850636,
|
|
0.006571690039252182,
|
|
0.005060205437758869
|
|
],
|
|
"core_parameter_grad_statistics": {
|
|
"first_quartile_mean": 0.007299623773970663,
|
|
"first_to_last_ratio": 1.227374872012571,
|
|
"imbalance_abs_log_ratio": 0.2048776382299483,
|
|
"last_quartile_mean": 0.005947346601614228,
|
|
"mean": 0.006362886261765913,
|
|
"normalized": [
|
|
1.271529717747562,
|
|
1.1494488132258316,
|
|
1.1143858200043408,
|
|
1.05351149791715,
|
|
1.0271137340180259,
|
|
1.066343404015675,
|
|
0.9894870520870547,
|
|
0.8658492545019393,
|
|
0.9793882512652816,
|
|
0.9504163254515969,
|
|
0.8847251275509024,
|
|
0.909028151691524,
|
|
0.9365519618304368,
|
|
0.9741361173356609,
|
|
1.0328158902888074,
|
|
0.7952688810682109
|
|
],
|
|
"population_cv": 0.11384977604868386
|
|
},
|
|
"depth_weights": [],
|
|
"layer_input_rms_by_sublayer": [
|
|
0.02823694422841072,
|
|
0.028404271230101585,
|
|
0.02866268903017044,
|
|
0.02886480651795864,
|
|
0.029099803417921066,
|
|
0.029365191236138344,
|
|
0.02953805774450302,
|
|
0.029881296679377556,
|
|
0.030088091269135475,
|
|
0.030441828072071075,
|
|
0.030644793063402176,
|
|
0.03106885403394699,
|
|
0.03131929785013199,
|
|
0.031892918050289154,
|
|
0.03213750571012497,
|
|
0.03266792371869087,
|
|
0.03286394104361534,
|
|
0.03364725783467293,
|
|
0.033869802951812744,
|
|
0.03485054895281792,
|
|
0.03499744459986687,
|
|
0.03562505170702934,
|
|
0.03588006645441055,
|
|
0.03719272464513779,
|
|
0.037365153431892395,
|
|
0.03768136352300644,
|
|
0.03789033368229866,
|
|
0.03927159309387207,
|
|
0.039445169270038605,
|
|
0.04041972756385803,
|
|
0.04050002992153168,
|
|
0.04110744968056679
|
|
],
|
|
"loss_nats": 5.6120405197143555,
|
|
"loss_scale": 1.0,
|
|
"output_weights": null,
|
|
"step": 0,
|
|
"stream_state_rms_by_sublayer": [
|
|
0.028404271230101585,
|
|
0.02866268903017044,
|
|
0.02886480651795864,
|
|
0.029099803417921066,
|
|
0.029365191236138344,
|
|
0.02953805774450302,
|
|
0.029881296679377556,
|
|
0.030088091269135475,
|
|
0.030441828072071075,
|
|
0.030644793063402176,
|
|
0.03106885403394699,
|
|
0.03131929785013199,
|
|
0.031892918050289154,
|
|
0.03213750571012497,
|
|
0.03266792371869087,
|
|
0.03286394104361534,
|
|
0.03364725783467293,
|
|
0.033869802951812744,
|
|
0.03485054895281792,
|
|
0.03499744459986687,
|
|
0.03562505170702934,
|
|
0.03588006645441055,
|
|
0.03719272464513779,
|
|
0.037365153431892395,
|
|
0.03768136352300644,
|
|
0.03789033368229866,
|
|
0.03927159309387207,
|
|
0.039445169270038605,
|
|
0.04041972756385803,
|
|
0.04050002992153168,
|
|
0.04110744968056679,
|
|
0.04121527820825577
|
|
]
|
|
},
|
|
{
|
|
"activation_grad_rms_by_block": [
|
|
5.69471885683015e-05,
|
|
5.527245593839325e-05,
|
|
5.393734318204224e-05,
|
|
5.292302375892177e-05,
|
|
5.23102717124857e-05,
|
|
5.178990977583453e-05,
|
|
5.1564093155320734e-05,
|
|
5.129844430484809e-05,
|
|
5.121947833686136e-05,
|
|
5.116340616950765e-05,
|
|
5.109804988023825e-05,
|
|
5.104695082991384e-05,
|
|
5.09646451973822e-05,
|
|
5.100828275317326e-05,
|
|
5.106439857627265e-05,
|
|
5.11144389747642e-05
|
|
],
|
|
"activation_grad_statistics": {
|
|
"first_quartile_mean": 5.477000286191469e-05,
|
|
"first_to_last_ratio": 1.0731232762518041,
|
|
"imbalance_abs_log_ratio": 0.07057334637995329,
|
|
"last_quartile_mean": 5.103794137539808e-05,
|
|
"mean": 5.2170148819641327e-05,
|
|
"normalized": [
|
|
1.0915665348238701,
|
|
1.0594651767139285,
|
|
1.033873669184083,
|
|
1.0144311441756324,
|
|
1.002685882559561,
|
|
0.9927115591500164,
|
|
0.988383094968431,
|
|
0.9832911246274795,
|
|
0.9817775010367221,
|
|
0.9807027069519371,
|
|
0.9794499543578176,
|
|
0.9784704852268963,
|
|
0.9768928467805085,
|
|
0.9777292936141551,
|
|
0.9788049244944386,
|
|
0.9797641013345227
|
|
],
|
|
"population_cv": 0.032815404485150704
|
|
},
|
|
"activation_output_rms_by_block": [
|
|
0.028743742033839226,
|
|
0.02942933700978756,
|
|
0.030596865341067314,
|
|
0.03249936178326607,
|
|
0.03542664647102356,
|
|
0.03894684836268425,
|
|
0.04318666458129883,
|
|
0.04936420917510986,
|
|
0.05370106175541878,
|
|
0.06048284471035004,
|
|
0.0664696991443634,
|
|
0.07231792062520981,
|
|
0.07627613097429276,
|
|
0.07997097074985504,
|
|
0.08494995534420013,
|
|
0.08972473442554474
|
|
],
|
|
"activation_output_statistics": {
|
|
"first_quartile_mean": 0.03031732654199004,
|
|
"first_to_last_ratio": 0.3664591129538783,
|
|
"imbalance_abs_log_ratio": 1.0038683247140607,
|
|
"last_quartile_mean": 0.08273044787347317,
|
|
"mean": 0.05450543703045696,
|
|
"normalized": [
|
|
0.5273555006590916,
|
|
0.5399339701348108,
|
|
0.5613543713807885,
|
|
0.5962590808162097,
|
|
0.6499653686150171,
|
|
0.7145497859400932,
|
|
0.7923368187501488,
|
|
0.9056749539963465,
|
|
0.9852422929002714,
|
|
1.1096662646068314,
|
|
1.21950584686113,
|
|
1.326801958945847,
|
|
1.399422427007981,
|
|
1.4672108895332452,
|
|
1.5585592919240543,
|
|
1.6461611779281335
|
|
],
|
|
"population_cv": 0.3808757530834043
|
|
},
|
|
"bits_per_byte": 6.7509657011324835,
|
|
"branch_output_rms_by_sublayer": [
|
|
0.003436450148001313,
|
|
0.003791899885982275,
|
|
0.003964710980653763,
|
|
0.003938332665711641,
|
|
0.0056790695525705814,
|
|
0.003919641952961683,
|
|
0.006960175931453705,
|
|
0.0038887569680809975,
|
|
0.008646421134471893,
|
|
0.003928068559616804,
|
|
0.009785857982933521,
|
|
0.0038193853106349707,
|
|
0.010625048540532589,
|
|
0.003996006678789854,
|
|
0.012892307713627815,
|
|
0.0039014811627566814,
|
|
0.012757784686982632,
|
|
0.003861474571749568,
|
|
0.014726920053362846,
|
|
0.003962590359151363,
|
|
0.012820222415030003,
|
|
0.004551138263195753,
|
|
0.01401793584227562,
|
|
0.004161422606557608,
|
|
0.01284075528383255,
|
|
0.004107494372874498,
|
|
0.012963366694748402,
|
|
0.0034911984112113714,
|
|
0.014392351731657982,
|
|
0.004168955609202385,
|
|
0.013408888131380081,
|
|
0.003909120801836252
|
|
],
|
|
"capture": {
|
|
"all_gradients_finite": true,
|
|
"all_gradients_present": true,
|
|
"count": 16,
|
|
"dtypes": [
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32"
|
|
],
|
|
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
|
"shape": [
|
|
16,
|
|
256,
|
|
192
|
|
],
|
|
"storage_unique": true
|
|
},
|
|
"core_parameter_grad_rms_by_block": [
|
|
0.0004507690637370199,
|
|
0.0004530669153600251,
|
|
0.0005174622333717066,
|
|
0.0005579942111201043,
|
|
0.0006494724560479704,
|
|
0.0007477190266084469,
|
|
0.0008433132451421535,
|
|
0.0008983201732199723,
|
|
0.001008715304343127,
|
|
0.0011143118429534733,
|
|
0.0010428854825857738,
|
|
0.0010916116075341876,
|
|
0.0011085001103672855,
|
|
0.0012404769719650786,
|
|
0.0013545740271648265,
|
|
0.0014243271893498689
|
|
],
|
|
"core_parameter_grad_statistics": {
|
|
"first_quartile_mean": 0.000494823105897214,
|
|
"first_to_last_ratio": 0.385986622193018,
|
|
"imbalance_abs_log_ratio": 0.9519525676489338,
|
|
"last_quartile_mean": 0.001281969574711765,
|
|
"mean": 0.0009064699913044387,
|
|
"normalized": [
|
|
0.49727963204644987,
|
|
0.4998145771025995,
|
|
0.5708542349284638,
|
|
0.6155683215912455,
|
|
0.7164853357289404,
|
|
0.824869034585972,
|
|
0.930326710461313,
|
|
0.99100927977468,
|
|
1.112795033503044,
|
|
1.2292870736404011,
|
|
1.1504909071341995,
|
|
1.2042446170372658,
|
|
1.2228756836970622,
|
|
1.3684699812069823,
|
|
1.494339625314625,
|
|
1.571289952246756
|
|
],
|
|
"population_cv": 0.338812252534038
|
|
},
|
|
"depth_weights": [],
|
|
"layer_input_rms_by_sublayer": [
|
|
0.028241552412509918,
|
|
0.028533460572361946,
|
|
0.028743742033839226,
|
|
0.029236938804388046,
|
|
0.02942933700978756,
|
|
0.03035305254161358,
|
|
0.030596865341067314,
|
|
0.032195623964071274,
|
|
0.03249936178326607,
|
|
0.03506496921181679,
|
|
0.03542664647102356,
|
|
0.03852483630180359,
|
|
0.03894684836268425,
|
|
0.04251723736524582,
|
|
0.04318666458129883,
|
|
0.04861506074666977,
|
|
0.04936420917510986,
|
|
0.052915990352630615,
|
|
0.05370106175541878,
|
|
0.0598941408097744,
|
|
0.06048284471035004,
|
|
0.06538087129592896,
|
|
0.0664696991443634,
|
|
0.07142146676778793,
|
|
0.07231792062520981,
|
|
0.07521561533212662,
|
|
0.07627613097429276,
|
|
0.07935076206922531,
|
|
0.07997097074985504,
|
|
0.08368266373872757,
|
|
0.08494995534420013,
|
|
0.08860929310321808
|
|
],
|
|
"loss_nats": 4.679412841796875,
|
|
"loss_scale": 1.0,
|
|
"output_weights": null,
|
|
"step": 20,
|
|
"stream_state_rms_by_sublayer": [
|
|
0.028533460572361946,
|
|
0.028743742033839226,
|
|
0.029236938804388046,
|
|
0.02942933700978756,
|
|
0.03035305254161358,
|
|
0.030596865341067314,
|
|
0.032195623964071274,
|
|
0.03249936178326607,
|
|
0.03506496921181679,
|
|
0.03542664647102356,
|
|
0.03852483630180359,
|
|
0.03894684836268425,
|
|
0.04251723736524582,
|
|
0.04318666458129883,
|
|
0.04861506074666977,
|
|
0.04936420917510986,
|
|
0.052915990352630615,
|
|
0.05370106175541878,
|
|
0.0598941408097744,
|
|
0.06048284471035004,
|
|
0.06538087129592896,
|
|
0.0664696991443634,
|
|
0.07142146676778793,
|
|
0.07231792062520981,
|
|
0.07521561533212662,
|
|
0.07627613097429276,
|
|
0.07935076206922531,
|
|
0.07997097074985504,
|
|
0.08368266373872757,
|
|
0.08494995534420013,
|
|
0.08860929310321808,
|
|
0.08972473442554474
|
|
]
|
|
}
|
|
],
|
|
"environment": {
|
|
"autocast": "cuda-bfloat16-forward-fp32-cross-entropy",
|
|
"compile": false,
|
|
"compute_capability": [
|
|
12,
|
|
0
|
|
],
|
|
"cublas_workspace_config": ":4096:8",
|
|
"cuda": "12.8",
|
|
"deterministic_algorithms": true,
|
|
"gpu": "NVIDIA GeForce RTX 5090",
|
|
"python": "3.10.14",
|
|
"torch": "2.11.0+cu128"
|
|
},
|
|
"evaluations": [
|
|
{
|
|
"bits_per_byte": 8.076786148009306,
|
|
"cross_entropy_nats": 5.5984015464782715,
|
|
"step": 0
|
|
},
|
|
{
|
|
"bits_per_byte": 6.752401670275862,
|
|
"cross_entropy_nats": 4.680408179759979,
|
|
"step": 20
|
|
}
|
|
],
|
|
"gradient_gate": {
|
|
"first_to_last_ratio_abs_delta": 0.0,
|
|
"max_abs_scale_ratio_error": 0.0,
|
|
"normalized_spectrum_max_abs_delta": 0.0,
|
|
"passed": true,
|
|
"per_block_scale_ratios": [
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
2.0
|
|
],
|
|
"population_cv_abs_delta": 0.0,
|
|
"thresholds": {
|
|
"scale_ratio_abs": 1e-05,
|
|
"shape_abs": 1e-06
|
|
}
|
|
},
|
|
"hashes": {
|
|
"final_mixer_parameters": null,
|
|
"final_model_state": "d53dab0fa7c76215ab91de676b2aef3f9ef14cb0cc1819b7c4a887915bed97c0",
|
|
"final_optimizer_state": "a7bce44db1478ce53933758aa5033bbb1e0aa21296e9615c33146920f30a2057",
|
|
"final_public_parameters": "d53dab0fa7c76215ab91de676b2aef3f9ef14cb0cc1819b7c4a887915bed97c0",
|
|
"initial_mixer_parameters": null,
|
|
"initial_public_parameter_elements": 9541824,
|
|
"initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe",
|
|
"initial_public_parameter_tensors": 115,
|
|
"initial_public_parameters": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1"
|
|
},
|
|
"manifest": {
|
|
"diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716",
|
|
"file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371",
|
|
"formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
|
|
"input_gate_tensor_hashes": {
|
|
"0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434",
|
|
"1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94",
|
|
"7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc"
|
|
},
|
|
"path": "experiments/k3/attnres_gradient/manifest.json",
|
|
"validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338"
|
|
},
|
|
"model": {
|
|
"attnres_aggregation_groups": 8,
|
|
"context": 256,
|
|
"d_ff": 768,
|
|
"d_head": 32,
|
|
"d_model": 192,
|
|
"heads": 6,
|
|
"layers": 16,
|
|
"parameters": {
|
|
"core": 9541824,
|
|
"embedding": 98304,
|
|
"mixer": 0,
|
|
"total": 9541824
|
|
},
|
|
"sublayers": 32,
|
|
"sublayers_per_attnres_group": 4,
|
|
"transformer_blocks_per_attnres_group": 2,
|
|
"vocabulary": 256
|
|
},
|
|
"optimizer": {
|
|
"betas": [
|
|
0.9,
|
|
0.95
|
|
],
|
|
"epsilon": 1e-08,
|
|
"grad_clip": 1.0,
|
|
"min_lr": 3e-05,
|
|
"name": "AdamW",
|
|
"peak_lr": 0.0003,
|
|
"warmup_steps": 400,
|
|
"weight_decay_ndim_ge_2": 0.1
|
|
},
|
|
"protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
|
|
"run_kind": "smoke",
|
|
"schema_version": 1,
|
|
"seed": 2026073001,
|
|
"steps": 20,
|
|
"target_bytes_seen": 163840,
|
|
"timing": {
|
|
"mean_ms": null,
|
|
"measured_steps": 0,
|
|
"median_ms": null,
|
|
"p95_ms": null,
|
|
"peak_allocated_bytes": 1648265728,
|
|
"peak_reserved_bytes": 3282042880,
|
|
"warmup_steps_excluded": 20
|
|
},
|
|
"training_history": [
|
|
{
|
|
"bits_per_byte": 8.088097790921855,
|
|
"learning_rate": 7.499999999999999e-07,
|
|
"loss_nats": 5.6062421798706055,
|
|
"step": 1,
|
|
"unclipped_grad_norm": 19.475919723510742
|
|
},
|
|
{
|
|
"bits_per_byte": 7.474302716882146,
|
|
"learning_rate": 7.499999999999999e-06,
|
|
"loss_nats": 5.180791854858398,
|
|
"step": 10,
|
|
"unclipped_grad_norm": 12.938376426696777
|
|
},
|
|
{
|
|
"bits_per_byte": 6.789615018295581,
|
|
"learning_rate": 1.4999999999999999e-05,
|
|
"loss_nats": 4.706202507019043,
|
|
"step": 20,
|
|
"unclipped_grad_norm": 4.482712745666504
|
|
}
|
|
]
|
|
}
|