wuyang
9617 lines
272 KiB
JSON
9617 lines
272 KiB
JSON
{
|
|
"architecture": "block",
|
|
"batch_size": 32,
|
|
"canonical_sha256_without_self": "6b2a0f3320fece089c65a158888f4aafe0879ff220d9835ed3614be007b68068",
|
|
"depth": 16,
|
|
"diagnostics": [
|
|
{
|
|
"activation_grad_rms_by_block": [
|
|
0.00027713447343558073,
|
|
0.00024001447309274226,
|
|
0.00020889485313091427,
|
|
0.00019106797117274255,
|
|
0.00017709653184283525,
|
|
0.00016345064796041697,
|
|
0.0001520135992905125,
|
|
0.00014396758342627436,
|
|
0.00013824697816744447,
|
|
0.0001333381951553747,
|
|
0.00012879272981081158,
|
|
0.00012578140012919903,
|
|
0.00012338913802523166,
|
|
0.0001212825154652819,
|
|
0.00011928759340662509,
|
|
0.00011723516217898577
|
|
],
|
|
"activation_grad_statistics": {
|
|
"first_quartile_mean": 0.00022927794270799495,
|
|
"first_to_last_ratio": 1.9059069547229377,
|
|
"imbalance_abs_log_ratio": 0.6449579870010378,
|
|
"last_quartile_mean": 0.00012029860226903111,
|
|
"mean": 0.00016006211535568582,
|
|
"normalized": [
|
|
1.7314182860806244,
|
|
1.4995083162520277,
|
|
1.3050861702453052,
|
|
1.1937113960299495,
|
|
1.1064237870985023,
|
|
1.021170110098828,
|
|
0.9497162957812465,
|
|
0.8994482117542517,
|
|
0.8637083038683806,
|
|
0.8330403159990363,
|
|
0.8046421823465957,
|
|
0.7858286756343993,
|
|
0.7708828397715447,
|
|
0.7577215582573746,
|
|
0.7452581339534802,
|
|
0.7324354168284537
|
|
],
|
|
"population_cv": 0.28697332073733334
|
|
},
|
|
"activation_output_rms_by_block": [
|
|
0.005004490725696087,
|
|
0.007150472607463598,
|
|
0.0056586493737995625,
|
|
0.007886149920523167,
|
|
0.005775847006589174,
|
|
0.008285986259579659,
|
|
0.006597128231078386,
|
|
0.009296777658164501,
|
|
0.007501539774239063,
|
|
0.01081777922809124,
|
|
0.007698386441916227,
|
|
0.011321330443024635,
|
|
0.009009703993797302,
|
|
0.013792803511023521,
|
|
0.00953624676913023,
|
|
0.012954792007803917
|
|
],
|
|
"activation_output_statistics": {
|
|
"first_quartile_mean": 0.0064249406568706036,
|
|
"first_to_last_ratio": 0.567404514268178,
|
|
"imbalance_abs_log_ratio": 0.5666828005795043,
|
|
"last_quartile_mean": 0.011323386570438743,
|
|
"mean": 0.008643005246995017,
|
|
"normalized": [
|
|
0.5790220626599802,
|
|
0.8273132322752738,
|
|
0.6547085431617609,
|
|
0.9124314628022479,
|
|
0.6682683674867963,
|
|
0.9586927258271091,
|
|
0.7632910130850682,
|
|
1.0756417926966766,
|
|
0.8679318778438997,
|
|
1.251622429808468,
|
|
0.8907071350665657,
|
|
1.3098835554867694,
|
|
1.04242722714183,
|
|
1.5958342170183197,
|
|
1.103348487778111,
|
|
1.4988758698611242
|
|
],
|
|
"population_cv": 0.28641618387108014
|
|
},
|
|
"bits_per_byte": 8.092078157393424,
|
|
"branch_output_rms_by_sublayer": [
|
|
0.0033052791841328144,
|
|
0.003737553022801876,
|
|
0.0033692142460495234,
|
|
0.0038969451561570168,
|
|
0.004222586750984192,
|
|
0.003809630637988448,
|
|
0.004224879201501608,
|
|
0.003801452461630106,
|
|
0.004351162351667881,
|
|
0.0037875697016716003,
|
|
0.005037518683820963,
|
|
0.0037579576019197702,
|
|
0.005422801710665226,
|
|
0.003782435320317745,
|
|
0.005750134587287903,
|
|
0.003670723643153906,
|
|
0.006515308283269405,
|
|
0.003754077944904566,
|
|
0.006654165219515562,
|
|
0.0036852110642939806,
|
|
0.006751408334821463,
|
|
0.003720880253240466,
|
|
0.007695567328482866,
|
|
0.003703791182488203,
|
|
0.008140962570905685,
|
|
0.003749529365450144,
|
|
0.009426429867744446,
|
|
0.0036915375385433435,
|
|
0.008798080496490002,
|
|
0.003752112854272127,
|
|
0.008380400948226452,
|
|
0.0036098985001444817
|
|
],
|
|
"capture": {
|
|
"all_gradients_finite": true,
|
|
"all_gradients_present": true,
|
|
"count": 16,
|
|
"dtypes": [
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32"
|
|
],
|
|
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
|
"shape": [
|
|
16,
|
|
256,
|
|
192
|
|
],
|
|
"storage_unique": true
|
|
},
|
|
"core_parameter_grad_rms_by_block": [
|
|
0.007836087849617155,
|
|
0.00707663314616961,
|
|
0.006860545956961424,
|
|
0.006477463491545069,
|
|
0.00630808320659222,
|
|
0.0065403273634858875,
|
|
0.006076989561376891,
|
|
0.0053044433268254945,
|
|
0.006002429844215364,
|
|
0.005811298045810947,
|
|
0.00541486468124986,
|
|
0.005552686577658202,
|
|
0.005724443119797102,
|
|
0.005942467552368739,
|
|
0.006312250195815329,
|
|
0.004848514440357715
|
|
],
|
|
"core_parameter_grad_statistics": {
|
|
"first_quartile_mean": 0.007062682611073314,
|
|
"first_to_last_ratio": 1.237564932158175,
|
|
"imbalance_abs_log_ratio": 0.21314568451503188,
|
|
"last_quartile_mean": 0.005706918827084721,
|
|
"mean": 0.006130595522490438,
|
|
"normalized": [
|
|
1.2781935818257821,
|
|
1.154314147819503,
|
|
1.119066807097797,
|
|
1.0565798163950184,
|
|
1.0289511326347756,
|
|
1.0668339379905794,
|
|
0.9912559944760846,
|
|
0.865241118479574,
|
|
0.9790940899942118,
|
|
0.947917380047643,
|
|
0.883252640201938,
|
|
0.9057336366895933,
|
|
0.9337499267072272,
|
|
0.9693132633801169,
|
|
1.0296308364592772,
|
|
0.7908716898008789
|
|
],
|
|
"population_cv": 0.1166547247084935
|
|
},
|
|
"depth_weights": [
|
|
{
|
|
"entropy_mean": 0.0,
|
|
"mean_weights": [
|
|
1.0
|
|
],
|
|
"sources": 1
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
}
|
|
],
|
|
"layer_input_rms_by_sublayer": [
|
|
0.02823692187666893,
|
|
0.014202049933373928,
|
|
0.014330258592963219,
|
|
0.014431006275117397,
|
|
0.0145475585013628,
|
|
0.009805543348193169,
|
|
0.009861846454441547,
|
|
0.009975572116672993,
|
|
0.010043247602880001,
|
|
0.007605585735291243,
|
|
0.007654537446796894,
|
|
0.0077590313740074635,
|
|
0.007819762453436852,
|
|
0.006375181954354048,
|
|
0.006421996746212244,
|
|
0.0065260399132966995,
|
|
0.006563509814441204,
|
|
0.005602568853646517,
|
|
0.0056375316344201565,
|
|
0.0057973419316112995,
|
|
0.005820456892251968,
|
|
0.005056646186858416,
|
|
0.00509054446592927,
|
|
0.005272198468446732,
|
|
0.005295110866427422,
|
|
0.0046798414550721645,
|
|
0.004703808110207319,
|
|
0.004869809839874506,
|
|
0.004889858886599541,
|
|
0.004459173884242773,
|
|
0.00446688337251544,
|
|
0.00453129643574357
|
|
],
|
|
"loss_nats": 5.609001159667969,
|
|
"loss_scale": 1.0,
|
|
"output_weights": {
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
},
|
|
"step": 0,
|
|
"stream_state_rms_by_sublayer": [
|
|
0.0033052791841328144,
|
|
0.005004490725696087,
|
|
0.005999269895255566,
|
|
0.007150472607463598,
|
|
0.004222586750984192,
|
|
0.0056586493737995625,
|
|
0.006890070624649525,
|
|
0.007886149920523167,
|
|
0.004351162351667881,
|
|
0.005775847006589174,
|
|
0.007411441300064325,
|
|
0.008285986259579659,
|
|
0.005422801710665226,
|
|
0.006597128231078386,
|
|
0.008578632958233356,
|
|
0.009296777658164501,
|
|
0.006515308283269405,
|
|
0.007501539774239063,
|
|
0.010256913490593433,
|
|
0.01081777922809124,
|
|
0.006751408334821463,
|
|
0.007698386441916227,
|
|
0.01081965770572424,
|
|
0.011321330443024635,
|
|
0.008140962570905685,
|
|
0.009009703993797302,
|
|
0.013233783654868603,
|
|
0.013792803511023521,
|
|
0.008798080496490002,
|
|
0.00953624676913023,
|
|
0.01231932733207941,
|
|
0.012954792007803917
|
|
]
|
|
},
|
|
{
|
|
"activation_grad_rms_by_block": [
|
|
8.295638690469787e-05,
|
|
7.495813042623922e-05,
|
|
6.424964522011578e-05,
|
|
5.6337856221944094e-05,
|
|
4.736588743980974e-05,
|
|
4.000150511274114e-05,
|
|
3.3406991860829294e-05,
|
|
2.9257240385049954e-05,
|
|
2.6081905161845498e-05,
|
|
2.4221646526711993e-05,
|
|
2.3017932107904926e-05,
|
|
2.205361488449853e-05,
|
|
2.1097952412674204e-05,
|
|
2.0617559130187146e-05,
|
|
2.0049723389092833e-05,
|
|
1.9771381630562246e-05
|
|
],
|
|
"activation_grad_statistics": {
|
|
"first_quartile_mean": 6.962550469324924e-05,
|
|
"first_to_last_ratio": 3.415668082811133,
|
|
"imbalance_abs_log_ratio": 1.2283731057520397,
|
|
"last_quartile_mean": 2.0384154140629107e-05,
|
|
"mean": 3.784033492593153e-05,
|
|
"normalized": [
|
|
2.1922741187961536,
|
|
1.980905575306399,
|
|
1.6979142850050797,
|
|
1.4888308026929338,
|
|
1.2517301322127163,
|
|
1.0571128715176512,
|
|
0.8828408079954885,
|
|
0.7731760419752606,
|
|
0.6892620060815551,
|
|
0.640101272203941,
|
|
0.6082909190143296,
|
|
0.5828070741885916,
|
|
0.557551946989138,
|
|
0.5448566766267753,
|
|
0.5298505795030106,
|
|
0.5224948898909759
|
|
],
|
|
"population_cv": 0.5389270020989329
|
|
},
|
|
"activation_output_rms_by_block": [
|
|
0.00972363818436861,
|
|
0.018413539975881577,
|
|
0.011301152408123016,
|
|
0.020593363791704178,
|
|
0.012286243960261345,
|
|
0.022828614339232445,
|
|
0.014672646299004555,
|
|
0.02904798462986946,
|
|
0.01802690140902996,
|
|
0.03463379293680191,
|
|
0.0215163491666317,
|
|
0.04299422353506088,
|
|
0.024622667580842972,
|
|
0.048123423010110855,
|
|
0.032037004828453064,
|
|
0.06016550958156586
|
|
],
|
|
"activation_output_statistics": {
|
|
"first_quartile_mean": 0.015007923590019345,
|
|
"first_to_last_ratio": 0.36394181302547757,
|
|
"imbalance_abs_log_ratio": 1.010761278449072,
|
|
"last_quartile_mean": 0.04123715125024319,
|
|
"mean": 0.0263116909773089,
|
|
"normalized": [
|
|
0.3695558066851058,
|
|
0.6998235116002747,
|
|
0.4295106847320868,
|
|
0.7826697193070493,
|
|
0.4669499946186262,
|
|
0.8676224708978132,
|
|
0.5576474089657784,
|
|
1.1039953553316026,
|
|
0.6851289574879962,
|
|
1.3162891342358027,
|
|
0.8177486268437599,
|
|
1.6340349836176982,
|
|
0.9358071133503911,
|
|
1.8289749241739084,
|
|
1.2175958153385753,
|
|
2.2866454928135314
|
|
],
|
|
"population_cv": 0.5242218345998277
|
|
},
|
|
"bits_per_byte": 5.094476963164152,
|
|
"branch_output_rms_by_sublayer": [
|
|
0.004319524392485619,
|
|
0.008673875592648983,
|
|
0.004246409051120281,
|
|
0.009991174563765526,
|
|
0.005452096462249756,
|
|
0.010073051787912846,
|
|
0.006387368775904179,
|
|
0.010035603307187557,
|
|
0.0060663470067083836,
|
|
0.01051078736782074,
|
|
0.007258581463247538,
|
|
0.010655198246240616,
|
|
0.007839187048375607,
|
|
0.011261194944381714,
|
|
0.010483811609447002,
|
|
0.011669226922094822,
|
|
0.01079322025179863,
|
|
0.012236819602549076,
|
|
0.012645936571061611,
|
|
0.012139025144279003,
|
|
0.013178177177906036,
|
|
0.012943001464009285,
|
|
0.017010189592838287,
|
|
0.012494352646172047,
|
|
0.016437800601124763,
|
|
0.012577332556247711,
|
|
0.018056757748126984,
|
|
0.01252446137368679,
|
|
0.022778941318392754,
|
|
0.013186860829591751,
|
|
0.022304536774754524,
|
|
0.012990611605346203
|
|
],
|
|
"capture": {
|
|
"all_gradients_finite": true,
|
|
"all_gradients_present": true,
|
|
"count": 16,
|
|
"dtypes": [
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32"
|
|
],
|
|
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
|
"shape": [
|
|
16,
|
|
256,
|
|
192
|
|
],
|
|
"storage_unique": true
|
|
},
|
|
"core_parameter_grad_rms_by_block": [
|
|
0.00047768131303286524,
|
|
0.00037801938737084786,
|
|
0.0003757481810571057,
|
|
0.0003849183374966076,
|
|
0.0003205908784645981,
|
|
0.0003135875371373823,
|
|
0.0002702256708410831,
|
|
0.00025052984822069896,
|
|
0.00022846424191684566,
|
|
0.00020301436013832287,
|
|
0.00018877625003321395,
|
|
0.00018901092530252878,
|
|
0.00017529332745812984,
|
|
0.0001779656532508539,
|
|
0.00018745944157007413,
|
|
0.00017444725355961893
|
|
],
|
|
"core_parameter_grad_statistics": {
|
|
"first_quartile_mean": 0.0004040918047393566,
|
|
"first_to_last_ratio": 2.260129748343848,
|
|
"imbalance_abs_log_ratio": 0.8154222224078733,
|
|
"last_quartile_mean": 0.0001787914189596692,
|
|
"mean": 0.00026848328792817357,
|
|
"normalized": [
|
|
1.779184532188304,
|
|
1.407981071328276,
|
|
1.39952167584311,
|
|
1.4336770845848084,
|
|
1.1940813185749002,
|
|
1.167996487071014,
|
|
1.0064897257715932,
|
|
0.9331301406280542,
|
|
0.8509439961043904,
|
|
0.7561526890740202,
|
|
0.703121045223927,
|
|
0.7039951229779867,
|
|
0.6529021929477618,
|
|
0.6628556087202881,
|
|
0.6982164253747687,
|
|
0.649750883586796
|
|
],
|
|
"population_cv": 0.3443291255737099
|
|
},
|
|
"depth_weights": [
|
|
{
|
|
"entropy_mean": 0.0,
|
|
"mean_weights": [
|
|
1.0
|
|
],
|
|
"sources": 1
|
|
},
|
|
{
|
|
"entropy_mean": 0.6925109028816223,
|
|
"mean_weights": [
|
|
0.516845703125,
|
|
0.4831542670726776
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6916897296905518,
|
|
"mean_weights": [
|
|
0.5256155133247375,
|
|
0.47438448667526245
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6921632289886475,
|
|
"mean_weights": [
|
|
0.5142087936401367,
|
|
0.4857912063598633
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6927341222763062,
|
|
"mean_weights": [
|
|
0.5126981735229492,
|
|
0.4873018264770508
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 1.0980398654937744,
|
|
"mean_weights": [
|
|
0.3447725176811218,
|
|
0.324658066034317,
|
|
0.33056944608688354
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.098333477973938,
|
|
"mean_weights": [
|
|
0.3432988226413727,
|
|
0.3276415765285492,
|
|
0.3290596008300781
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0971107482910156,
|
|
"mean_weights": [
|
|
0.354239284992218,
|
|
0.3338475227355957,
|
|
0.3119131922721863
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.098445177078247,
|
|
"mean_weights": [
|
|
0.33804112672805786,
|
|
0.3361448645591736,
|
|
0.3258139491081238
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.3858705759048462,
|
|
"mean_weights": [
|
|
0.25839337706565857,
|
|
0.24941343069076538,
|
|
0.24149224162101746,
|
|
0.2507009506225586
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.38621187210083,
|
|
"mean_weights": [
|
|
0.2517760992050171,
|
|
0.2535240054130554,
|
|
0.24683426320552826,
|
|
0.24786563217639923
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.385884165763855,
|
|
"mean_weights": [
|
|
0.2573123574256897,
|
|
0.2553749680519104,
|
|
0.24531587958335876,
|
|
0.24199676513671875
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3861134052276611,
|
|
"mean_weights": [
|
|
0.2514210641384125,
|
|
0.2551700472831726,
|
|
0.24895039200782776,
|
|
0.24445852637290955
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.6088017225265503,
|
|
"mean_weights": [
|
|
0.20787756145000458,
|
|
0.20764237642288208,
|
|
0.1969079077243805,
|
|
0.1933688223361969,
|
|
0.19420333206653595
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6093868017196655,
|
|
"mean_weights": [
|
|
0.1996297687292099,
|
|
0.200723797082901,
|
|
0.20193718373775482,
|
|
0.19867070019245148,
|
|
0.199038565158844
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.607616901397705,
|
|
"mean_weights": [
|
|
0.20617185533046722,
|
|
0.2148669958114624,
|
|
0.20431196689605713,
|
|
0.19360044598579407,
|
|
0.18104872107505798
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094070672988892,
|
|
"mean_weights": [
|
|
0.1992160826921463,
|
|
0.20051369071006775,
|
|
0.198721244931221,
|
|
0.19902712106704712,
|
|
0.20252184569835663
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.7898081541061401,
|
|
"mean_weights": [
|
|
0.17206060886383057,
|
|
0.18212765455245972,
|
|
0.17207278311252594,
|
|
0.1640574187040329,
|
|
0.15447911620140076,
|
|
0.15520241856575012
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7917404174804688,
|
|
"mean_weights": [
|
|
0.16550754010677338,
|
|
0.16615179181098938,
|
|
0.16667258739471436,
|
|
0.16704785823822021,
|
|
0.16769331693649292,
|
|
0.16692689061164856
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7875205278396606,
|
|
"mean_weights": [
|
|
0.1741587072610855,
|
|
0.18743160367012024,
|
|
0.17662332952022552,
|
|
0.16453337669372559,
|
|
0.15226861834526062,
|
|
0.14498436450958252
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7916977405548096,
|
|
"mean_weights": [
|
|
0.1655852198600769,
|
|
0.16538481414318085,
|
|
0.16465555131435394,
|
|
0.16688036918640137,
|
|
0.16788282990455627,
|
|
0.16961121559143066
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.941636562347412,
|
|
"mean_weights": [
|
|
0.15022630989551544,
|
|
0.1620834320783615,
|
|
0.15417584776878357,
|
|
0.14502081274986267,
|
|
0.1326843798160553,
|
|
0.12827157974243164,
|
|
0.12753766775131226
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9458396434783936,
|
|
"mean_weights": [
|
|
0.14176656305789948,
|
|
0.14105038344860077,
|
|
0.14134296774864197,
|
|
0.14280946552753448,
|
|
0.145027756690979,
|
|
0.14322641491889954,
|
|
0.14477643370628357
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9395363330841064,
|
|
"mean_weights": [
|
|
0.1491325944662094,
|
|
0.16522708535194397,
|
|
0.15644297003746033,
|
|
0.1476437896490097,
|
|
0.13585588335990906,
|
|
0.12666445970535278,
|
|
0.11903323233127594
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9456722736358643,
|
|
"mean_weights": [
|
|
0.14162756502628326,
|
|
0.1389600932598114,
|
|
0.139722540974617,
|
|
0.1427886188030243,
|
|
0.14271582663059235,
|
|
0.14634007215499878,
|
|
0.14784526824951172
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 2.0722482204437256,
|
|
"mean_weights": [
|
|
0.13311916589736938,
|
|
0.1474711298942566,
|
|
0.1391790360212326,
|
|
0.13093729317188263,
|
|
0.12214824557304382,
|
|
0.11368396133184433,
|
|
0.1066313236951828,
|
|
0.10682985186576843
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079228401184082,
|
|
"mean_weights": [
|
|
0.12360984086990356,
|
|
0.12237860262393951,
|
|
0.12229418754577637,
|
|
0.12343960255384445,
|
|
0.1245395839214325,
|
|
0.12723875045776367,
|
|
0.12765389680862427,
|
|
0.12884552776813507
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.0708298683166504,
|
|
"mean_weights": [
|
|
0.13347145915031433,
|
|
0.14857617020606995,
|
|
0.14165782928466797,
|
|
0.1326693594455719,
|
|
0.12101326882839203,
|
|
0.11364143341779709,
|
|
0.10599445551633835,
|
|
0.10297602415084839
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079227924346924,
|
|
"mean_weights": [
|
|
0.12400860339403152,
|
|
0.12165012210607529,
|
|
0.12233424186706543,
|
|
0.12391965091228485,
|
|
0.12502582371234894,
|
|
0.12579283118247986,
|
|
0.12804390490055084,
|
|
0.12922482192516327
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.1904914379119873,
|
|
"mean_weights": [
|
|
0.1202651709318161,
|
|
0.13243305683135986,
|
|
0.1248447597026825,
|
|
0.11750722676515579,
|
|
0.10865370184183121,
|
|
0.1034422218799591,
|
|
0.09818274527788162,
|
|
0.09678354859352112,
|
|
0.09788758307695389
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1969027519226074,
|
|
"mean_weights": [
|
|
0.10986955463886261,
|
|
0.108255535364151,
|
|
0.10776711255311966,
|
|
0.10903829336166382,
|
|
0.11005579680204391,
|
|
0.11137478053569794,
|
|
0.11346087604761124,
|
|
0.11412093043327332,
|
|
0.1160571351647377
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1876320838928223,
|
|
"mean_weights": [
|
|
0.12014202773571014,
|
|
0.1339646577835083,
|
|
0.12840712070465088,
|
|
0.12060266733169556,
|
|
0.11135218292474747,
|
|
0.10444460809230804,
|
|
0.09650979936122894,
|
|
0.09281164407730103,
|
|
0.09176531434059143
|
|
],
|
|
"sources": 9
|
|
}
|
|
],
|
|
"layer_input_rms_by_sublayer": [
|
|
0.028673440217971802,
|
|
0.01493624784052372,
|
|
0.015959296375513077,
|
|
0.015908053144812584,
|
|
0.017468810081481934,
|
|
0.011937789618968964,
|
|
0.013394257053732872,
|
|
0.013926870189607143,
|
|
0.015381631441414356,
|
|
0.011804917827248573,
|
|
0.013283023610711098,
|
|
0.013736946508288383,
|
|
0.015293780714273453,
|
|
0.01275910809636116,
|
|
0.014138885773718357,
|
|
0.014833617955446243,
|
|
0.01643991842865944,
|
|
0.014291302300989628,
|
|
0.01566910557448864,
|
|
0.016281748190522194,
|
|
0.01796886697411537,
|
|
0.016028419137001038,
|
|
0.017591506242752075,
|
|
0.01816963590681553,
|
|
0.020232705399394035,
|
|
0.018053309991955757,
|
|
0.019960105419158936,
|
|
0.01998855359852314,
|
|
0.02254442311823368,
|
|
0.020374728366732597,
|
|
0.022746482864022255,
|
|
0.022396961227059364
|
|
],
|
|
"loss_nats": 3.531222343444824,
|
|
"loss_scale": 1.0,
|
|
"output_weights": {
|
|
"entropy_mean": 2.1935343742370605,
|
|
"mean_weights": [
|
|
0.0999753400683403,
|
|
0.09610259532928467,
|
|
0.10262835025787354,
|
|
0.1083034798502922,
|
|
0.11600279062986374,
|
|
0.11817552894353867,
|
|
0.12095586210489273,
|
|
0.11944535374641418,
|
|
0.11841070652008057
|
|
],
|
|
"sources": 9
|
|
},
|
|
"step": 100,
|
|
"stream_state_rms_by_sublayer": [
|
|
0.004319524392485619,
|
|
0.00972363818436861,
|
|
0.010906110517680645,
|
|
0.018413539975881577,
|
|
0.005452096462249756,
|
|
0.011301152408123016,
|
|
0.013437297195196152,
|
|
0.020593363791704178,
|
|
0.0060663470067083836,
|
|
0.012286243960261345,
|
|
0.014960306696593761,
|
|
0.022828614339232445,
|
|
0.007839187048375607,
|
|
0.014672646299004555,
|
|
0.020714404061436653,
|
|
0.02904798462986946,
|
|
0.01079322025179863,
|
|
0.01802690140902996,
|
|
0.026133937761187553,
|
|
0.03463379293680191,
|
|
0.013178177177906036,
|
|
0.0215163491666317,
|
|
0.03426522761583328,
|
|
0.04299422353506088,
|
|
0.016437800601124763,
|
|
0.024622667580842972,
|
|
0.03885738551616669,
|
|
0.048123423010110855,
|
|
0.022778941318392754,
|
|
0.032037004828453064,
|
|
0.050110094249248505,
|
|
0.06016550958156586
|
|
]
|
|
},
|
|
{
|
|
"activation_grad_rms_by_block": [
|
|
0.00015777233056724072,
|
|
0.00015884912863839418,
|
|
0.00014271712279878557,
|
|
0.0001244439627043903,
|
|
0.00010494918387848884,
|
|
9.392325591761619e-05,
|
|
8.538185647921637e-05,
|
|
7.606096187373623e-05,
|
|
7.664377335458994e-05,
|
|
6.531321560032666e-05,
|
|
6.149107503006235e-05,
|
|
5.037344453739934e-05,
|
|
4.3381573050282896e-05,
|
|
3.9420847315341234e-05,
|
|
3.5541354009183124e-05,
|
|
3.5066059353994206e-05
|
|
],
|
|
"activation_grad_statistics": {
|
|
"first_quartile_mean": 0.0001459456361772027,
|
|
"first_to_last_ratio": 3.805378902507801,
|
|
"imbalance_abs_log_ratio": 1.3364155665156117,
|
|
"last_quartile_mean": 3.8352458432200365e-05,
|
|
"mean": 8.445807156931551e-05,
|
|
"normalized": [
|
|
1.8680550909542795,
|
|
1.8808045896243943,
|
|
1.6897985017531016,
|
|
1.4734407309105797,
|
|
1.2426187566022755,
|
|
1.1120696242813515,
|
|
1.0109377930698158,
|
|
0.9005765874172524,
|
|
0.907477188745515,
|
|
0.7733211804004252,
|
|
0.7280662923921495,
|
|
0.5964313842526867,
|
|
0.513646265468887,
|
|
0.46675050214695213,
|
|
0.42081654658683526,
|
|
0.4151889653934991
|
|
],
|
|
"population_cv": 0.4877143313799778
|
|
},
|
|
"activation_output_rms_by_block": [
|
|
0.025262881070375443,
|
|
0.04275903478264809,
|
|
0.027615655213594437,
|
|
0.03926205262541771,
|
|
0.023017436265945435,
|
|
0.03517371043562889,
|
|
0.029550904408097267,
|
|
0.04735429957509041,
|
|
0.028369002044200897,
|
|
0.04305245727300644,
|
|
0.030713967978954315,
|
|
0.05133935809135437,
|
|
0.032636724412441254,
|
|
0.06983363628387451,
|
|
0.0380333848297596,
|
|
0.08374716341495514
|
|
],
|
|
"activation_output_statistics": {
|
|
"first_quartile_mean": 0.03372490592300892,
|
|
"first_to_last_ratio": 0.601556641750376,
|
|
"imbalance_abs_log_ratio": 0.508234580502337,
|
|
"last_quartile_mean": 0.056062727235257626,
|
|
"mean": 0.04048260429408401,
|
|
"normalized": [
|
|
0.6240428823910242,
|
|
1.056232313317272,
|
|
0.6821610342304507,
|
|
0.9698499716125064,
|
|
0.5685759764548824,
|
|
0.868859873246076,
|
|
0.7299654981044719,
|
|
1.1697443976451536,
|
|
0.7007701836106094,
|
|
1.0634804263148154,
|
|
0.7586954573335775,
|
|
1.2681831860026092,
|
|
0.8061913254234652,
|
|
1.7250282560027828,
|
|
0.939499458914941,
|
|
2.0687197593953623
|
|
],
|
|
"population_cv": 0.3940384688106746
|
|
},
|
|
"bits_per_byte": 3.0824777281177833,
|
|
"branch_output_rms_by_sublayer": [
|
|
0.014225655235350132,
|
|
0.01885763369500637,
|
|
0.016379276290535927,
|
|
0.015279849991202354,
|
|
0.023593612015247345,
|
|
0.011461553163826466,
|
|
0.02286384254693985,
|
|
0.009474686346948147,
|
|
0.01935836300253868,
|
|
0.010116237215697765,
|
|
0.017970046028494835,
|
|
0.009506942704319954,
|
|
0.022477619349956512,
|
|
0.013764957897365093,
|
|
0.025794917717576027,
|
|
0.012601078487932682,
|
|
0.02379908226430416,
|
|
0.014191905967891216,
|
|
0.0178237184882164,
|
|
0.0165114663541317,
|
|
0.023045850917696953,
|
|
0.019447455182671547,
|
|
0.020932989194989204,
|
|
0.02472713589668274,
|
|
0.017093855887651443,
|
|
0.028764601796865463,
|
|
0.021727578714489937,
|
|
0.03578802943229675,
|
|
0.008686712011694908,
|
|
0.036414168775081635,
|
|
0.008637740276753902,
|
|
0.04455644637346268
|
|
],
|
|
"capture": {
|
|
"all_gradients_finite": true,
|
|
"all_gradients_present": true,
|
|
"count": 16,
|
|
"dtypes": [
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32"
|
|
],
|
|
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
|
"shape": [
|
|
16,
|
|
256,
|
|
192
|
|
],
|
|
"storage_unique": true
|
|
},
|
|
"core_parameter_grad_rms_by_block": [
|
|
0.0006504433508330581,
|
|
0.00048523602227392907,
|
|
0.0003980638932136608,
|
|
0.0004387300445530515,
|
|
0.000365155189968218,
|
|
0.0002788779263831531,
|
|
0.00029885208568229324,
|
|
0.00026779671325771547,
|
|
0.0002902285411340871,
|
|
0.00023612702925164195,
|
|
0.00026277969610552434,
|
|
0.0002548116213942568,
|
|
0.0001837802586554031,
|
|
0.00016271499692659642,
|
|
0.00013374164438063855,
|
|
0.00014095740133328614
|
|
],
|
|
"core_parameter_grad_statistics": {
|
|
"first_quartile_mean": 0.0004931183277184248,
|
|
"first_to_last_ratio": 3.1752920249892207,
|
|
"imbalance_abs_log_ratio": 1.1553996044896453,
|
|
"last_quartile_mean": 0.00015529857532398106,
|
|
"mean": 0.0003030185259591571,
|
|
"normalized": [
|
|
2.1465464818501867,
|
|
1.6013411085609086,
|
|
1.3136619022010376,
|
|
1.4478654173513685,
|
|
1.20505896071,
|
|
0.9203329251913204,
|
|
0.9862502127100129,
|
|
0.8837635006310174,
|
|
0.9577914096684838,
|
|
0.7792494815431475,
|
|
0.8672067005597657,
|
|
0.8409110320489186,
|
|
0.6064984247206531,
|
|
0.536980359242221,
|
|
0.4413645797968972,
|
|
0.46517750321406204
|
|
],
|
|
"population_cv": 0.440031122144889
|
|
},
|
|
"depth_weights": [
|
|
{
|
|
"entropy_mean": 0.0,
|
|
"mean_weights": [
|
|
1.0
|
|
],
|
|
"sources": 1
|
|
},
|
|
{
|
|
"entropy_mean": 0.6679806113243103,
|
|
"mean_weights": [
|
|
0.4191037714481354,
|
|
0.580896258354187
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6774038076400757,
|
|
"mean_weights": [
|
|
0.4775225520133972,
|
|
0.522477388381958
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.686615526676178,
|
|
"mean_weights": [
|
|
0.49171629548072815,
|
|
0.5082836747169495
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6786561012268066,
|
|
"mean_weights": [
|
|
0.48390668630599976,
|
|
0.516093373298645
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 1.0939385890960693,
|
|
"mean_weights": [
|
|
0.32296979427337646,
|
|
0.3461979627609253,
|
|
0.33083218336105347
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0876491069793701,
|
|
"mean_weights": [
|
|
0.3022151589393616,
|
|
0.3410899341106415,
|
|
0.35669487714767456
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.094268560409546,
|
|
"mean_weights": [
|
|
0.3278208076953888,
|
|
0.3347434997558594,
|
|
0.33743569254875183
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0785255432128906,
|
|
"mean_weights": [
|
|
0.30184268951416016,
|
|
0.3319981098175049,
|
|
0.3661591410636902
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.3751009702682495,
|
|
"mean_weights": [
|
|
0.2335343211889267,
|
|
0.2411448359489441,
|
|
0.23957431316375732,
|
|
0.2857465147972107
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3803684711456299,
|
|
"mean_weights": [
|
|
0.23392163217067719,
|
|
0.24959179759025574,
|
|
0.25211477279663086,
|
|
0.264371782541275
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.377173662185669,
|
|
"mean_weights": [
|
|
0.2529916763305664,
|
|
0.22492024302482605,
|
|
0.24044713377952576,
|
|
0.2816409468650818
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3786447048187256,
|
|
"mean_weights": [
|
|
0.23216357827186584,
|
|
0.24167919158935547,
|
|
0.2545858919620514,
|
|
0.2715713381767273
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.6062697172164917,
|
|
"mean_weights": [
|
|
0.20799711346626282,
|
|
0.1897224485874176,
|
|
0.18646025657653809,
|
|
0.19469907879829407,
|
|
0.22112110257148743
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6025722026824951,
|
|
"mean_weights": [
|
|
0.20026245713233948,
|
|
0.18544277548789978,
|
|
0.1946466565132141,
|
|
0.19751906394958496,
|
|
0.22212904691696167
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6060729026794434,
|
|
"mean_weights": [
|
|
0.20172038674354553,
|
|
0.20138651132583618,
|
|
0.18545961380004883,
|
|
0.19016142189502716,
|
|
0.2212720513343811
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.601726770401001,
|
|
"mean_weights": [
|
|
0.20073023438453674,
|
|
0.1886606216430664,
|
|
0.19257165491580963,
|
|
0.19004762172698975,
|
|
0.22798986732959747
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.788693904876709,
|
|
"mean_weights": [
|
|
0.16298983991146088,
|
|
0.17133885622024536,
|
|
0.15878048539161682,
|
|
0.15357322990894318,
|
|
0.17383432388305664,
|
|
0.17948326468467712
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7880157232284546,
|
|
"mean_weights": [
|
|
0.16720382869243622,
|
|
0.15951547026634216,
|
|
0.1605137139558792,
|
|
0.15542668104171753,
|
|
0.17394030094146729,
|
|
0.1834000051021576
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7797719240188599,
|
|
"mean_weights": [
|
|
0.15782463550567627,
|
|
0.17797376215457916,
|
|
0.15211433172225952,
|
|
0.14025422930717468,
|
|
0.16258476674556732,
|
|
0.20924828946590424
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7884608507156372,
|
|
"mean_weights": [
|
|
0.16717270016670227,
|
|
0.17295360565185547,
|
|
0.1665436029434204,
|
|
0.1547294557094574,
|
|
0.1560850888490677,
|
|
0.18251556158065796
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.927689790725708,
|
|
"mean_weights": [
|
|
0.1226624995470047,
|
|
0.15493766963481903,
|
|
0.12924720346927643,
|
|
0.12375974655151367,
|
|
0.1207568496465683,
|
|
0.16289259493350983,
|
|
0.18574345111846924
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9359079599380493,
|
|
"mean_weights": [
|
|
0.1374971866607666,
|
|
0.15181128680706024,
|
|
0.14001096785068512,
|
|
0.13405972719192505,
|
|
0.11918757855892181,
|
|
0.14624495804309845,
|
|
0.17118829488754272
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9246108531951904,
|
|
"mean_weights": [
|
|
0.12061166763305664,
|
|
0.16360490024089813,
|
|
0.1358117014169693,
|
|
0.1262153834104538,
|
|
0.1155187338590622,
|
|
0.15234649181365967,
|
|
0.18589113652706146
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.937231183052063,
|
|
"mean_weights": [
|
|
0.1359395533800125,
|
|
0.16017206013202667,
|
|
0.1499013900756836,
|
|
0.1417488306760788,
|
|
0.12114370614290237,
|
|
0.13951142132282257,
|
|
0.15158303081989288
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 2.052294969558716,
|
|
"mean_weights": [
|
|
0.0988478884100914,
|
|
0.15235479176044464,
|
|
0.12090037763118744,
|
|
0.11009317636489868,
|
|
0.09369668364524841,
|
|
0.12056069076061249,
|
|
0.13413047790527344,
|
|
0.1694159209728241
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.061406135559082,
|
|
"mean_weights": [
|
|
0.1149308905005455,
|
|
0.14702443778514862,
|
|
0.13390076160430908,
|
|
0.12344712018966675,
|
|
0.0966324657201767,
|
|
0.11953848600387573,
|
|
0.1299745738506317,
|
|
0.1345512568950653
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.0511577129364014,
|
|
"mean_weights": [
|
|
0.09969542920589447,
|
|
0.16962948441505432,
|
|
0.1319108009338379,
|
|
0.11863738298416138,
|
|
0.09143088757991791,
|
|
0.12074249982833862,
|
|
0.14481820166110992,
|
|
0.12313534319400787
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.069282054901123,
|
|
"mean_weights": [
|
|
0.12384641170501709,
|
|
0.13645249605178833,
|
|
0.13362589478492737,
|
|
0.12747827172279358,
|
|
0.10400287806987762,
|
|
0.11596200615167618,
|
|
0.12676866352558136,
|
|
0.13186340034008026
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.165989875793457,
|
|
"mean_weights": [
|
|
0.08691617846488953,
|
|
0.1522500216960907,
|
|
0.1198691874742508,
|
|
0.10617777705192566,
|
|
0.08211278915405273,
|
|
0.11005747318267822,
|
|
0.12842655181884766,
|
|
0.08939606696367264,
|
|
0.12479399144649506
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1907525062561035,
|
|
"mean_weights": [
|
|
0.1032973974943161,
|
|
0.12799270451068878,
|
|
0.11672255396842957,
|
|
0.11367214471101761,
|
|
0.10446079820394516,
|
|
0.11008220911026001,
|
|
0.12035597860813141,
|
|
0.1041758805513382,
|
|
0.09924033284187317
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1538312435150146,
|
|
"mean_weights": [
|
|
0.09009353071451187,
|
|
0.16798722743988037,
|
|
0.13133394718170166,
|
|
0.11392836272716522,
|
|
0.08597822487354279,
|
|
0.10755141079425812,
|
|
0.13270318508148193,
|
|
0.09302861988544464,
|
|
0.07739552855491638
|
|
],
|
|
"sources": 9
|
|
}
|
|
],
|
|
"layer_input_rms_by_sublayer": [
|
|
0.04356693848967552,
|
|
0.020052054896950722,
|
|
0.025759555399417877,
|
|
0.02808709815144539,
|
|
0.03107464499771595,
|
|
0.023514779284596443,
|
|
0.025064069777727127,
|
|
0.026588447391986847,
|
|
0.02779453620314598,
|
|
0.02102271094918251,
|
|
0.022771678864955902,
|
|
0.023088837042450905,
|
|
0.024504151195287704,
|
|
0.02003801427781582,
|
|
0.021661879494786263,
|
|
0.021999599412083626,
|
|
0.02357223629951477,
|
|
0.01831224001944065,
|
|
0.01920042186975479,
|
|
0.01775558479130268,
|
|
0.01933184079825878,
|
|
0.014507194049656391,
|
|
0.0161737073212862,
|
|
0.015147161670029163,
|
|
0.017492808401584625,
|
|
0.01415062416344881,
|
|
0.01712540164589882,
|
|
0.017588311806321144,
|
|
0.021003147587180138,
|
|
0.01741618849337101,
|
|
0.020855708047747612,
|
|
0.02042423188686371
|
|
],
|
|
"loss_nats": 2.136610746383667,
|
|
"loss_scale": 1.0,
|
|
"output_weights": {
|
|
"entropy_mean": 2.1821343898773193,
|
|
"mean_weights": [
|
|
0.12100096046924591,
|
|
0.09246665984392166,
|
|
0.10230658948421478,
|
|
0.10203823447227478,
|
|
0.09980043023824692,
|
|
0.10532397031784058,
|
|
0.11232466995716095,
|
|
0.1358221024274826,
|
|
0.12891638278961182
|
|
],
|
|
"sources": 9
|
|
},
|
|
"step": 500,
|
|
"stream_state_rms_by_sublayer": [
|
|
0.014225655235350132,
|
|
0.025262881070375443,
|
|
0.03417503461241722,
|
|
0.04275903478264809,
|
|
0.023593612015247345,
|
|
0.027615655213594437,
|
|
0.035288337618112564,
|
|
0.03926205262541771,
|
|
0.01935836300253868,
|
|
0.023017436265945435,
|
|
0.030542491003870964,
|
|
0.03517371043562889,
|
|
0.022477619349956512,
|
|
0.029550904408097267,
|
|
0.04085070639848709,
|
|
0.04735429957509041,
|
|
0.02379908226430416,
|
|
0.028369002044200897,
|
|
0.03469245135784149,
|
|
0.04305245727300644,
|
|
0.023045850917696953,
|
|
0.030713967978954315,
|
|
0.037714507430791855,
|
|
0.05133935809135437,
|
|
0.017093855887651443,
|
|
0.032636724412441254,
|
|
0.041826508939266205,
|
|
0.06983363628387451,
|
|
0.008686712011694908,
|
|
0.0380333848297596,
|
|
0.04225953668355942,
|
|
0.08374716341495514
|
|
]
|
|
},
|
|
{
|
|
"activation_grad_rms_by_block": [
|
|
7.445879600709304e-05,
|
|
7.514825847465545e-05,
|
|
8.149984205374494e-05,
|
|
7.616886432515457e-05,
|
|
7.93620856711641e-05,
|
|
7.107412966433913e-05,
|
|
6.609321280848235e-05,
|
|
6.663892418146133e-05,
|
|
0.00013160602247808129,
|
|
0.00010466711682965979,
|
|
0.00017213824321515858,
|
|
0.0001231937640113756,
|
|
0.00011169029312441126,
|
|
7.821891631465405e-05,
|
|
5.0732542149489745e-05,
|
|
4.2012750782305375e-05
|
|
],
|
|
"activation_grad_statistics": {
|
|
"first_quartile_mean": 7.6818940215162e-05,
|
|
"first_to_last_ratio": 1.087107257387618,
|
|
"imbalance_abs_log_ratio": 0.08352027612092149,
|
|
"last_quartile_mean": 7.066362559271511e-05,
|
|
"mean": 8.779398513070191e-05,
|
|
"normalized": [
|
|
0.8481081693266763,
|
|
0.8559613550151489,
|
|
0.9283078098392884,
|
|
0.8675863638238926,
|
|
0.9039581191469443,
|
|
0.8095557976839601,
|
|
0.752821650709751,
|
|
0.7590374680253287,
|
|
1.499032334415107,
|
|
1.1921900648869996,
|
|
1.9607065672994624,
|
|
1.403214170401,
|
|
1.2721861635297007,
|
|
0.8909370749967309,
|
|
0.5778589737550069,
|
|
0.47853791714500205
|
|
],
|
|
"population_cv": 0.365612445298668
|
|
},
|
|
"activation_output_rms_by_block": [
|
|
0.10262583196163177,
|
|
0.17122715711593628,
|
|
0.1563396155834198,
|
|
0.23706908524036407,
|
|
0.04811444878578186,
|
|
0.07098451256752014,
|
|
0.05486894026398659,
|
|
0.08282698690891266,
|
|
0.05263432487845421,
|
|
0.09547916799783707,
|
|
0.05760422348976135,
|
|
0.09091533720493317,
|
|
0.05383981764316559,
|
|
0.1016460508108139,
|
|
0.07459317892789841,
|
|
0.18281812965869904
|
|
],
|
|
"activation_output_statistics": {
|
|
"first_quartile_mean": 0.16681542247533798,
|
|
"first_to_last_ratio": 1.6160480792916094,
|
|
"imbalance_abs_log_ratio": 0.4799837116932076,
|
|
"last_quartile_mean": 0.10322429426014423,
|
|
"mean": 0.10209917556494474,
|
|
"normalized": [
|
|
1.005158282559804,
|
|
1.6770669906831872,
|
|
1.531252477978855,
|
|
2.3219490650006835,
|
|
0.4712520793586283,
|
|
0.6952505950683928,
|
|
0.5374082597668455,
|
|
0.8112405066016116,
|
|
0.5155215464494499,
|
|
0.9351610085930936,
|
|
0.5641987133688422,
|
|
0.8904610316574242,
|
|
0.5273286228341615,
|
|
0.9955619156411052,
|
|
0.7305953110311854,
|
|
1.7905935934067303
|
|
],
|
|
"population_cv": 0.5286889875537891
|
|
},
|
|
"bits_per_byte": 2.0839361382492645,
|
|
"branch_output_rms_by_sublayer": [
|
|
0.05184043571352959,
|
|
0.0848206952214241,
|
|
0.0633782222867012,
|
|
0.04741951450705528,
|
|
0.15336942672729492,
|
|
0.017866143956780434,
|
|
0.08626767247915268,
|
|
0.013284381479024887,
|
|
0.043669264763593674,
|
|
0.012495848350226879,
|
|
0.03683627396821976,
|
|
0.02719804272055626,
|
|
0.04707624390721321,
|
|
0.01943410001695156,
|
|
0.031199999153614044,
|
|
0.02732040360569954,
|
|
0.04137895256280899,
|
|
0.029010025784373283,
|
|
0.04094765707850456,
|
|
0.04834925755858421,
|
|
0.03695840761065483,
|
|
0.041233282536268234,
|
|
0.026400581002235413,
|
|
0.05263383314013481,
|
|
0.021495802327990532,
|
|
0.04698725789785385,
|
|
0.018388623371720314,
|
|
0.06700151413679123,
|
|
0.015488670207560062,
|
|
0.0718570277094841,
|
|
0.015049737878143787,
|
|
0.12199355661869049
|
|
],
|
|
"capture": {
|
|
"all_gradients_finite": true,
|
|
"all_gradients_present": true,
|
|
"count": 16,
|
|
"dtypes": [
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32"
|
|
],
|
|
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
|
"shape": [
|
|
16,
|
|
256,
|
|
192
|
|
],
|
|
"storage_unique": true
|
|
},
|
|
"core_parameter_grad_rms_by_block": [
|
|
0.0003583741677462985,
|
|
0.0002652231972340971,
|
|
0.00033417461006948805,
|
|
0.0002275253539728081,
|
|
0.0001893834795064905,
|
|
0.00020636019253143534,
|
|
0.0002096112726133392,
|
|
0.0001923796657115476,
|
|
0.00047635112247231166,
|
|
0.000565001794959564,
|
|
0.0009549288128880833,
|
|
0.0007263576654831343,
|
|
0.0005471283815528999,
|
|
0.0004183078461688176,
|
|
0.0002607600761312772,
|
|
0.0002482750049477455
|
|
],
|
|
"core_parameter_grad_statistics": {
|
|
"first_quartile_mean": 0.00029632433225567294,
|
|
"first_to_last_ratio": 0.8038795478406102,
|
|
"imbalance_abs_log_ratio": 0.21830583714537038,
|
|
"last_quartile_mean": 0.00036861782720018504,
|
|
"mean": 0.0003862589152493336,
|
|
"normalized": [
|
|
0.9278081452565691,
|
|
0.6866461504529756,
|
|
0.8651570148323317,
|
|
0.5890488089470723,
|
|
0.49030189862217616,
|
|
0.5342535392308757,
|
|
0.5426703807678671,
|
|
0.49805883596852335,
|
|
1.2332430493282536,
|
|
1.462754056032209,
|
|
2.472250542804088,
|
|
1.8804942405387945,
|
|
1.4164809146210218,
|
|
1.0829726632945058,
|
|
0.6750914110628468,
|
|
0.6427683482398892
|
|
],
|
|
"population_cv": 0.550745478202196
|
|
},
|
|
"depth_weights": [
|
|
{
|
|
"entropy_mean": 0.0,
|
|
"mean_weights": [
|
|
1.0
|
|
],
|
|
"sources": 1
|
|
},
|
|
{
|
|
"entropy_mean": 0.4576239287853241,
|
|
"mean_weights": [
|
|
0.8160475492477417,
|
|
0.18395249545574188
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.622429370880127,
|
|
"mean_weights": [
|
|
0.626032829284668,
|
|
0.37396717071533203
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.5034195184707642,
|
|
"mean_weights": [
|
|
0.7919335961341858,
|
|
0.20806637406349182
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6750276684761047,
|
|
"mean_weights": [
|
|
0.4605996012687683,
|
|
0.5394004583358765
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.9881983399391174,
|
|
"mean_weights": [
|
|
0.5066143870353699,
|
|
0.2128569334745407,
|
|
0.2805286943912506
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0713173151016235,
|
|
"mean_weights": [
|
|
0.2439693659543991,
|
|
0.33285707235336304,
|
|
0.42317354679107666
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0050503015518188,
|
|
"mean_weights": [
|
|
0.46526122093200684,
|
|
0.19465598464012146,
|
|
0.3400827646255493
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.091313123703003,
|
|
"mean_weights": [
|
|
0.29163461923599243,
|
|
0.3559022545814514,
|
|
0.35246312618255615
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.3032760620117188,
|
|
"mean_weights": [
|
|
0.25388818979263306,
|
|
0.13041767477989197,
|
|
0.27188149094581604,
|
|
0.34381261467933655
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3785908222198486,
|
|
"mean_weights": [
|
|
0.2572174668312073,
|
|
0.24623332917690277,
|
|
0.2131483256816864,
|
|
0.28340083360671997
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.2377310991287231,
|
|
"mean_weights": [
|
|
0.36895960569381714,
|
|
0.12289825826883316,
|
|
0.12992563843727112,
|
|
0.3782165050506592
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.370632529258728,
|
|
"mean_weights": [
|
|
0.2267947793006897,
|
|
0.24114219844341278,
|
|
0.21443253755569458,
|
|
0.31763046979904175
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.5590848922729492,
|
|
"mean_weights": [
|
|
0.22265344858169556,
|
|
0.11684703826904297,
|
|
0.1529940664768219,
|
|
0.24088799953460693,
|
|
0.26661744713783264
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.5987765789031982,
|
|
"mean_weights": [
|
|
0.19780412316322327,
|
|
0.19732677936553955,
|
|
0.1696196049451828,
|
|
0.187029629945755,
|
|
0.24821987748146057
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.5600793361663818,
|
|
"mean_weights": [
|
|
0.19789859652519226,
|
|
0.13498559594154358,
|
|
0.15099790692329407,
|
|
0.2153477668762207,
|
|
0.300770103931427
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6049537658691406,
|
|
"mean_weights": [
|
|
0.21525533497333527,
|
|
0.2069544643163681,
|
|
0.18156641721725464,
|
|
0.19566501677036285,
|
|
0.20055876672267914
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.7536540031433105,
|
|
"mean_weights": [
|
|
0.1706119179725647,
|
|
0.12355935573577881,
|
|
0.11797749251127243,
|
|
0.17735670506954193,
|
|
0.15967294573783875,
|
|
0.250821590423584
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7486846446990967,
|
|
"mean_weights": [
|
|
0.1656067669391632,
|
|
0.16547001898288727,
|
|
0.14594469964504242,
|
|
0.13893218338489532,
|
|
0.11710066348314285,
|
|
0.26694566011428833
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.502744197845459,
|
|
"mean_weights": [
|
|
0.14790554344654083,
|
|
0.12157414853572845,
|
|
0.05193060636520386,
|
|
0.11843410134315491,
|
|
0.09838350117206573,
|
|
0.4617721140384674
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7703553438186646,
|
|
"mean_weights": [
|
|
0.17112275958061218,
|
|
0.18443143367767334,
|
|
0.1734921634197235,
|
|
0.14492332935333252,
|
|
0.11953528970479965,
|
|
0.2064950168132782
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.6120144128799438,
|
|
"mean_weights": [
|
|
0.08869323134422302,
|
|
0.07363369315862656,
|
|
0.0430244579911232,
|
|
0.08643164485692978,
|
|
0.0696420818567276,
|
|
0.21115350723266602,
|
|
0.4274213910102844
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.8458226919174194,
|
|
"mean_weights": [
|
|
0.12519317865371704,
|
|
0.13179585337638855,
|
|
0.1208491250872612,
|
|
0.10168728232383728,
|
|
0.08638136088848114,
|
|
0.13217970728874207,
|
|
0.3019134998321533
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.5814783573150635,
|
|
"mean_weights": [
|
|
0.09110043942928314,
|
|
0.07703252136707306,
|
|
0.04712430387735367,
|
|
0.08346284925937653,
|
|
0.06837025284767151,
|
|
0.1662226915359497,
|
|
0.4666869342327118
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.864192008972168,
|
|
"mean_weights": [
|
|
0.13281965255737305,
|
|
0.1363598257303238,
|
|
0.09730516374111176,
|
|
0.10794492065906525,
|
|
0.09621791541576385,
|
|
0.14952389895915985,
|
|
0.27982860803604126
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.7858508825302124,
|
|
"mean_weights": [
|
|
0.0760645866394043,
|
|
0.06870019435882568,
|
|
0.041292984038591385,
|
|
0.06624549627304077,
|
|
0.05510660633444786,
|
|
0.12923449277877808,
|
|
0.3349960446357727,
|
|
0.22835959494113922
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.9164016246795654,
|
|
"mean_weights": [
|
|
0.10010629147291183,
|
|
0.09289439767599106,
|
|
0.06040576845407486,
|
|
0.08482401818037033,
|
|
0.06471984833478928,
|
|
0.12172460556030273,
|
|
0.20661231875419617,
|
|
0.26871275901794434
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.7764818668365479,
|
|
"mean_weights": [
|
|
0.07597766816616058,
|
|
0.07146216928958893,
|
|
0.04444725811481476,
|
|
0.06381020694971085,
|
|
0.052670422941446304,
|
|
0.10254879295825958,
|
|
0.2695360779762268,
|
|
0.3195474147796631
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.9026113748550415,
|
|
"mean_weights": [
|
|
0.09874680638313293,
|
|
0.09077301621437073,
|
|
0.052286967635154724,
|
|
0.08586552739143372,
|
|
0.061087287962436676,
|
|
0.1283089518547058,
|
|
0.24689394235610962,
|
|
0.2360374927520752
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.0007686614990234,
|
|
"mean_weights": [
|
|
0.0738106444478035,
|
|
0.07652983069419861,
|
|
0.04215084761381149,
|
|
0.06585545837879181,
|
|
0.053016483783721924,
|
|
0.10676231980323792,
|
|
0.23121783137321472,
|
|
0.19090119004249573,
|
|
0.1597554087638855
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.05625581741333,
|
|
"mean_weights": [
|
|
0.08243454247713089,
|
|
0.09636277705430984,
|
|
0.04735179618000984,
|
|
0.07971908152103424,
|
|
0.057041361927986145,
|
|
0.1233949214220047,
|
|
0.21250130236148834,
|
|
0.17240752279758453,
|
|
0.12878668308258057
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.023190498352051,
|
|
"mean_weights": [
|
|
0.08014722913503647,
|
|
0.08487560600042343,
|
|
0.04445280879735947,
|
|
0.06905653327703476,
|
|
0.057849444448947906,
|
|
0.1067991554737091,
|
|
0.20276135206222534,
|
|
0.2105080783367157,
|
|
0.14354978501796722
|
|
],
|
|
"sources": 9
|
|
}
|
|
],
|
|
"layer_input_rms_by_sublayer": [
|
|
0.05538630113005638,
|
|
0.04719197377562523,
|
|
0.05779172107577324,
|
|
0.05863569676876068,
|
|
0.10090170800685883,
|
|
0.07870100438594818,
|
|
0.1107865646481514,
|
|
0.10731016099452972,
|
|
0.12961514294147491,
|
|
0.08656422793865204,
|
|
0.08832070231437683,
|
|
0.06243300065398216,
|
|
0.09176014363765717,
|
|
0.06557498127222061,
|
|
0.0774947926402092,
|
|
0.06959337741136551,
|
|
0.08217275887727737,
|
|
0.054640792310237885,
|
|
0.061008501797914505,
|
|
0.04349073767662048,
|
|
0.07030727714300156,
|
|
0.028683170676231384,
|
|
0.04603966698050499,
|
|
0.03691008314490318,
|
|
0.04661448672413826,
|
|
0.035560719668865204,
|
|
0.03472456708550453,
|
|
0.035963866859674454,
|
|
0.04182035103440285,
|
|
0.0347055159509182,
|
|
0.03997879475355148,
|
|
0.04123349115252495
|
|
],
|
|
"loss_nats": 1.444474458694458,
|
|
"loss_scale": 1.0,
|
|
"output_weights": {
|
|
"entropy_mean": 1.9153306484222412,
|
|
"mean_weights": [
|
|
0.0914863795042038,
|
|
0.05659400671720505,
|
|
0.05275696888566017,
|
|
0.06721317768096924,
|
|
0.05635443329811096,
|
|
0.0678911805152893,
|
|
0.08835235983133316,
|
|
0.2647843658924103,
|
|
0.25456711649894714
|
|
],
|
|
"sources": 9
|
|
},
|
|
"step": 2000,
|
|
"stream_state_rms_by_sublayer": [
|
|
0.05184043571352959,
|
|
0.10262583196163177,
|
|
0.1408688724040985,
|
|
0.17122715711593628,
|
|
0.15336942672729492,
|
|
0.1563396155834198,
|
|
0.2344740480184555,
|
|
0.23706908524036407,
|
|
0.043669264763593674,
|
|
0.04811444878578186,
|
|
0.059994228184223175,
|
|
0.07098451256752014,
|
|
0.04707624390721321,
|
|
0.05486894026398659,
|
|
0.069931261241436,
|
|
0.08282698690891266,
|
|
0.04137895256280899,
|
|
0.05263432487845421,
|
|
0.07298346608877182,
|
|
0.09547916799783707,
|
|
0.03695840761065483,
|
|
0.05760422348976135,
|
|
0.0662730410695076,
|
|
0.09091533720493317,
|
|
0.021495802327990532,
|
|
0.05383981764316559,
|
|
0.058252494782209396,
|
|
0.1016460508108139,
|
|
0.015488670207560062,
|
|
0.07459317892789841,
|
|
0.07778036594390869,
|
|
0.18281812965869904
|
|
]
|
|
},
|
|
{
|
|
"activation_grad_rms_by_block": [
|
|
5.599572978098877e-05,
|
|
5.6567834690213203e-05,
|
|
7.252454088302329e-05,
|
|
6.816006498411298e-05,
|
|
6.961339386180043e-05,
|
|
6.354664219543338e-05,
|
|
5.091843559057452e-05,
|
|
5.004410559195094e-05,
|
|
0.00011432960309321061,
|
|
8.808868005871773e-05,
|
|
0.0001649916812311858,
|
|
0.00011419985094107687,
|
|
0.0001274511159863323,
|
|
8.248462108895183e-05,
|
|
5.212613905314356e-05,
|
|
4.046529647894204e-05
|
|
],
|
|
"activation_grad_statistics": {
|
|
"first_quartile_mean": 6.331204258458456e-05,
|
|
"first_to_last_ratio": 0.8371088393670095,
|
|
"imbalance_abs_log_ratio": 0.1778011818604578,
|
|
"last_quartile_mean": 7.563179315184243e-05,
|
|
"mean": 7.946923346935364e-05,
|
|
"normalized": [
|
|
0.7046214910652542,
|
|
0.711820565276094,
|
|
0.9126115569114117,
|
|
0.8576912348147676,
|
|
0.8759791786420843,
|
|
0.7996382929745935,
|
|
0.6407314298584572,
|
|
0.6297293104160852,
|
|
1.438664978910366,
|
|
1.1084626869175487,
|
|
2.076170538310438,
|
|
1.4370322444989567,
|
|
1.6037793548805563,
|
|
1.0379440883969395,
|
|
0.6559285496725645,
|
|
0.5091944984538828
|
|
],
|
|
"population_cv": 0.418216483754139
|
|
},
|
|
"activation_output_rms_by_block": [
|
|
0.16897788643836975,
|
|
0.2546124756336212,
|
|
0.24958282709121704,
|
|
0.3488057255744934,
|
|
0.05895345285534859,
|
|
0.09218393266201019,
|
|
0.05982525274157524,
|
|
0.10435724258422852,
|
|
0.07404384016990662,
|
|
0.13799543678760529,
|
|
0.072679303586483,
|
|
0.12079861760139465,
|
|
0.06610427051782608,
|
|
0.11929895728826523,
|
|
0.08813745528459549,
|
|
0.2320432960987091
|
|
],
|
|
"activation_output_statistics": {
|
|
"first_quartile_mean": 0.25549472868442535,
|
|
"first_to_last_ratio": 2.0213831070680737,
|
|
"imbalance_abs_log_ratio": 0.7037819835779293,
|
|
"last_quartile_mean": 0.12639599479734898,
|
|
"mean": 0.1405249983072281,
|
|
"normalized": [
|
|
1.2024756340429583,
|
|
1.8118660644063134,
|
|
1.776074222364032,
|
|
2.4821613931772033,
|
|
0.419522886073689,
|
|
0.6559966822448885,
|
|
0.425726763652258,
|
|
0.7426240444143152,
|
|
0.5269086714950565,
|
|
0.9819992061904,
|
|
0.5171983950327836,
|
|
0.8596236901372816,
|
|
0.47040933153618064,
|
|
0.8489518500291555,
|
|
0.6272012549105437,
|
|
1.6512599102929408
|
|
],
|
|
"population_cv": 0.5965464726510664
|
|
},
|
|
"bits_per_byte": 1.9646943750306483,
|
|
"branch_output_rms_by_sublayer": [
|
|
0.07875782251358032,
|
|
0.1447884738445282,
|
|
0.07949890196323395,
|
|
0.06182052940130234,
|
|
0.24504326283931732,
|
|
0.02123360149562359,
|
|
0.10607437044382095,
|
|
0.01836485043168068,
|
|
0.053805794566869736,
|
|
0.015964502468705177,
|
|
0.03685670718550682,
|
|
0.03905382379889488,
|
|
0.05105716362595558,
|
|
0.02389056608080864,
|
|
0.045697249472141266,
|
|
0.039935220032930374,
|
|
0.059221696108579636,
|
|
0.040779970586299896,
|
|
0.054663438349962234,
|
|
0.080495685338974,
|
|
0.04156494140625,
|
|
0.05664549022912979,
|
|
0.0355471596121788,
|
|
0.07668386399745941,
|
|
0.026702450588345528,
|
|
0.05748681351542473,
|
|
0.024266516789793968,
|
|
0.08028528094291687,
|
|
0.019828684628009796,
|
|
0.08536289632320404,
|
|
0.0202633123844862,
|
|
0.16591297090053558
|
|
],
|
|
"capture": {
|
|
"all_gradients_finite": true,
|
|
"all_gradients_present": true,
|
|
"count": 16,
|
|
"dtypes": [
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32"
|
|
],
|
|
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
|
"shape": [
|
|
16,
|
|
256,
|
|
192
|
|
],
|
|
"storage_unique": true
|
|
},
|
|
"core_parameter_grad_rms_by_block": [
|
|
0.00033760869044818646,
|
|
0.0001988249533593431,
|
|
0.00022180290878374173,
|
|
0.00015787488110689858,
|
|
0.0001349156990162873,
|
|
0.00018159413720443885,
|
|
0.00013882426062051804,
|
|
0.00016817749051027943,
|
|
0.0004737077931871952,
|
|
0.0005939057773758512,
|
|
0.0010653831085878915,
|
|
0.0008603315741687424,
|
|
0.000762827758322017,
|
|
0.0005453805470031765,
|
|
0.00031779802289983946,
|
|
0.00029451637801964313
|
|
],
|
|
"core_parameter_grad_statistics": {
|
|
"first_quartile_mean": 0.00022902785842454247,
|
|
"first_to_last_ratio": 0.47701150875195986,
|
|
"imbalance_abs_log_ratio": 0.7402146610222566,
|
|
"last_quartile_mean": 0.000480130676561169,
|
|
"mean": 0.00040334212378837815,
|
|
"normalized": [
|
|
0.8370280973313858,
|
|
0.49294368634717844,
|
|
0.5499125821534951,
|
|
0.3914167943185893,
|
|
0.33449444295353004,
|
|
0.45022358562210574,
|
|
0.34418488036066147,
|
|
0.41695989729681016,
|
|
1.1744565351565805,
|
|
1.4724615713271154,
|
|
2.6413881559935133,
|
|
2.1330070018179734,
|
|
1.89126727245144,
|
|
1.3521537048516206,
|
|
0.7879118102392371,
|
|
0.7301899817787623
|
|
],
|
|
"population_cv": 0.6909894412084847
|
|
},
|
|
"depth_weights": [
|
|
{
|
|
"entropy_mean": 0.0,
|
|
"mean_weights": [
|
|
1.0
|
|
],
|
|
"sources": 1
|
|
},
|
|
{
|
|
"entropy_mean": 0.27112114429473877,
|
|
"mean_weights": [
|
|
0.9155579805374146,
|
|
0.08444204181432724
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.5555346012115479,
|
|
"mean_weights": [
|
|
0.7120839953422546,
|
|
0.28791600465774536
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.4612463414669037,
|
|
"mean_weights": [
|
|
0.8216869831085205,
|
|
0.1783129870891571
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6770275831222534,
|
|
"mean_weights": [
|
|
0.48782873153686523,
|
|
0.5121712684631348
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.9962381720542908,
|
|
"mean_weights": [
|
|
0.465767502784729,
|
|
0.19465303421020508,
|
|
0.3395794630050659
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0414454936981201,
|
|
"mean_weights": [
|
|
0.22931429743766785,
|
|
0.28697463870048523,
|
|
0.4837110638618469
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 0.989799976348877,
|
|
"mean_weights": [
|
|
0.40070968866348267,
|
|
0.17750780284404755,
|
|
0.4217824935913086
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.085202932357788,
|
|
"mean_weights": [
|
|
0.2848654091358185,
|
|
0.3230235278606415,
|
|
0.39211106300354004
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.2675609588623047,
|
|
"mean_weights": [
|
|
0.23358023166656494,
|
|
0.10898500680923462,
|
|
0.35628458857536316,
|
|
0.3011501431465149
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3760833740234375,
|
|
"mean_weights": [
|
|
0.2731572389602661,
|
|
0.2245836853981018,
|
|
0.22189702093601227,
|
|
0.280362069606781
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.2111725807189941,
|
|
"mean_weights": [
|
|
0.41297993063926697,
|
|
0.11292652785778046,
|
|
0.1222275048494339,
|
|
0.3518660068511963
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3647139072418213,
|
|
"mean_weights": [
|
|
0.23418807983398438,
|
|
0.22505050897598267,
|
|
0.2112433910369873,
|
|
0.32951802015304565
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.5535553693771362,
|
|
"mean_weights": [
|
|
0.21889275312423706,
|
|
0.10524161159992218,
|
|
0.1673421710729599,
|
|
0.2413085401058197,
|
|
0.26721489429473877
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.5950126647949219,
|
|
"mean_weights": [
|
|
0.2038668543100357,
|
|
0.17965060472488403,
|
|
0.17214898765087128,
|
|
0.19203421473503113,
|
|
0.25229936838150024
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.5395398139953613,
|
|
"mean_weights": [
|
|
0.2043805718421936,
|
|
0.12131387740373611,
|
|
0.1313788890838623,
|
|
0.23064203560352325,
|
|
0.3122846484184265
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6000796556472778,
|
|
"mean_weights": [
|
|
0.22543779015541077,
|
|
0.20209266245365143,
|
|
0.18844717741012573,
|
|
0.20001280307769775,
|
|
0.18400955200195312
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.7217694520950317,
|
|
"mean_weights": [
|
|
0.1735105812549591,
|
|
0.1090376153588295,
|
|
0.10670766234397888,
|
|
0.18005813658237457,
|
|
0.14383366703987122,
|
|
0.28685232996940613
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7343032360076904,
|
|
"mean_weights": [
|
|
0.16843369603157043,
|
|
0.15050029754638672,
|
|
0.15599164366722107,
|
|
0.14163106679916382,
|
|
0.10262320935726166,
|
|
0.2808200716972351
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.4446122646331787,
|
|
"mean_weights": [
|
|
0.1531161069869995,
|
|
0.1061544418334961,
|
|
0.04045919328927994,
|
|
0.12559477984905243,
|
|
0.08803897351026535,
|
|
0.4866364896297455
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7605862617492676,
|
|
"mean_weights": [
|
|
0.1779545247554779,
|
|
0.16784369945526123,
|
|
0.1857004165649414,
|
|
0.14843404293060303,
|
|
0.10648007690906525,
|
|
0.21358723938465118
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.5263525247573853,
|
|
"mean_weights": [
|
|
0.08794503659009933,
|
|
0.0601879358291626,
|
|
0.033082835376262665,
|
|
0.08346094191074371,
|
|
0.05888189375400543,
|
|
0.20315150916576385,
|
|
0.4732898473739624
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.7944886684417725,
|
|
"mean_weights": [
|
|
0.12276870757341385,
|
|
0.11536462604999542,
|
|
0.1304849088191986,
|
|
0.09876619279384613,
|
|
0.07423829287290573,
|
|
0.11369137465953827,
|
|
0.3446859121322632
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.4918596744537354,
|
|
"mean_weights": [
|
|
0.09284307807683945,
|
|
0.06384406983852386,
|
|
0.03825954347848892,
|
|
0.08267752081155777,
|
|
0.05671031028032303,
|
|
0.1521964967250824,
|
|
0.5134689807891846
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.830867052078247,
|
|
"mean_weights": [
|
|
0.13634362816810608,
|
|
0.12310001999139786,
|
|
0.09669966250658035,
|
|
0.1085672378540039,
|
|
0.08431269228458405,
|
|
0.13993649184703827,
|
|
0.3110402822494507
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.6549372673034668,
|
|
"mean_weights": [
|
|
0.06966839730739594,
|
|
0.049915459007024765,
|
|
0.029968995600938797,
|
|
0.05769501253962517,
|
|
0.040842454880476,
|
|
0.10442843288183212,
|
|
0.30951738357543945,
|
|
0.33796387910842896
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.859694480895996,
|
|
"mean_weights": [
|
|
0.09671731293201447,
|
|
0.07844845205545425,
|
|
0.05707312002778053,
|
|
0.0829043835401535,
|
|
0.06170058995485306,
|
|
0.10576222091913223,
|
|
0.19003936648368835,
|
|
0.3273545503616333
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.677255392074585,
|
|
"mean_weights": [
|
|
0.07509785890579224,
|
|
0.05865459889173508,
|
|
0.03770594298839569,
|
|
0.059113647788763046,
|
|
0.042673639953136444,
|
|
0.08797852694988251,
|
|
0.24025270342826843,
|
|
0.39852309226989746
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.8695217370986938,
|
|
"mean_weights": [
|
|
0.0984041839838028,
|
|
0.08118931949138641,
|
|
0.05154065042734146,
|
|
0.08640529215335846,
|
|
0.05755901336669922,
|
|
0.11663993448019028,
|
|
0.23645231127738953,
|
|
0.27180927991867065
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.8854753971099854,
|
|
"mean_weights": [
|
|
0.06882007420063019,
|
|
0.0593036487698555,
|
|
0.03391250967979431,
|
|
0.060251303017139435,
|
|
0.038275185972452164,
|
|
0.09127645194530487,
|
|
0.20795494318008423,
|
|
0.23203802108764648,
|
|
0.20816785097122192
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.032858371734619,
|
|
"mean_weights": [
|
|
0.08241330087184906,
|
|
0.08549442887306213,
|
|
0.041513457894325256,
|
|
0.08080939948558807,
|
|
0.054553937166929245,
|
|
0.12166906148195267,
|
|
0.2086121290922165,
|
|
0.17801183462142944,
|
|
0.14692245423793793
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 1.9540549516677856,
|
|
"mean_weights": [
|
|
0.07821957767009735,
|
|
0.07357673346996307,
|
|
0.037884071469306946,
|
|
0.06657841801643372,
|
|
0.044617801904678345,
|
|
0.09479035437107086,
|
|
0.1742672622203827,
|
|
0.2624226212501526,
|
|
0.16764315962791443
|
|
],
|
|
"sources": 9
|
|
}
|
|
],
|
|
"layer_input_rms_by_sublayer": [
|
|
0.058635130524635315,
|
|
0.05468355491757393,
|
|
0.07153555005788803,
|
|
0.0688173845410347,
|
|
0.13985061645507812,
|
|
0.1219802051782608,
|
|
0.16997404396533966,
|
|
0.17499428987503052,
|
|
0.1912512183189392,
|
|
0.14618010818958282,
|
|
0.12251991033554077,
|
|
0.0794699639081955,
|
|
0.12536334991455078,
|
|
0.09043660014867783,
|
|
0.10440115630626678,
|
|
0.08707059919834137,
|
|
0.11446695029735565,
|
|
0.06907524168491364,
|
|
0.0854191705584526,
|
|
0.06048329547047615,
|
|
0.10043052583932877,
|
|
0.03875526040792465,
|
|
0.06678051501512527,
|
|
0.05217510461807251,
|
|
0.06635956466197968,
|
|
0.04468905180692673,
|
|
0.04571816325187683,
|
|
0.04713807255029678,
|
|
0.05554596707224846,
|
|
0.0446290485560894,
|
|
0.052154671400785446,
|
|
0.054206155240535736
|
|
],
|
|
"loss_nats": 1.3618223667144775,
|
|
"loss_scale": 1.0,
|
|
"output_weights": {
|
|
"entropy_mean": 1.836374282836914,
|
|
"mean_weights": [
|
|
0.08612492680549622,
|
|
0.0513404943048954,
|
|
0.04786331206560135,
|
|
0.0643724650144577,
|
|
0.0475543811917305,
|
|
0.059286996722221375,
|
|
0.07609974592924118,
|
|
0.2768789529800415,
|
|
0.2904787063598633
|
|
],
|
|
"sources": 9
|
|
},
|
|
"step": 4000,
|
|
"stream_state_rms_by_sublayer": [
|
|
0.07875782251358032,
|
|
0.16897788643836975,
|
|
0.21303832530975342,
|
|
0.2546124756336212,
|
|
0.24504326283931732,
|
|
0.24958282709121704,
|
|
0.345289409160614,
|
|
0.3488057255744934,
|
|
0.053805794566869736,
|
|
0.05895345285534859,
|
|
0.07887569069862366,
|
|
0.09218393266201019,
|
|
0.05105716362595558,
|
|
0.05982525274157524,
|
|
0.08438026905059814,
|
|
0.10435724258422852,
|
|
0.059221696108579636,
|
|
0.07404384016990662,
|
|
0.09824100881814957,
|
|
0.13799543678760529,
|
|
0.04156494140625,
|
|
0.072679303586483,
|
|
0.08400582522153854,
|
|
0.12079861760139465,
|
|
0.026702450588345528,
|
|
0.06610427051782608,
|
|
0.07228080183267593,
|
|
0.11929895728826523,
|
|
0.019828684628009796,
|
|
0.08813745528459549,
|
|
0.09191317111253738,
|
|
0.2320432960987091
|
|
]
|
|
},
|
|
{
|
|
"activation_grad_rms_by_block": [
|
|
7.052410364849493e-05,
|
|
7.120969530660659e-05,
|
|
9.769770258571953e-05,
|
|
9.110000246437266e-05,
|
|
9.818621765589342e-05,
|
|
8.837205677991733e-05,
|
|
7.198825915111229e-05,
|
|
6.996823503868654e-05,
|
|
0.00017213664250448346,
|
|
0.00013030946138314903,
|
|
0.0002552570658735931,
|
|
0.00017121800919994712,
|
|
0.00020642777963075787,
|
|
0.00012888041965197772,
|
|
9.31880422285758e-05,
|
|
5.8820565755013376e-05
|
|
],
|
|
"activation_grad_statistics": {
|
|
"first_quartile_mean": 8.263287600129843e-05,
|
|
"first_to_last_ratio": 0.6782682211585493,
|
|
"imbalance_abs_log_ratio": 0.3882124628447938,
|
|
"last_quartile_mean": 0.00012182920181658119,
|
|
"mean": 0.0001172052661786438,
|
|
"normalized": [
|
|
0.6017144617120052,
|
|
0.6075639570500958,
|
|
0.8335606903260565,
|
|
0.7772688500661601,
|
|
0.8377287203651615,
|
|
0.7539939088165287,
|
|
0.6142066947861206,
|
|
0.5969717685896574,
|
|
1.4686766910465736,
|
|
1.1118055155007445,
|
|
2.1778634543991506,
|
|
1.4608388750977905,
|
|
1.7612500390223207,
|
|
1.0996128745233913,
|
|
0.7950840885130448,
|
|
0.5018594101851985
|
|
],
|
|
"population_cv": 0.46767509062432555
|
|
},
|
|
"activation_output_rms_by_block": [
|
|
0.17702145874500275,
|
|
0.2651609778404236,
|
|
0.2740711271762848,
|
|
0.3786091208457947,
|
|
0.05278070271015167,
|
|
0.09138669073581696,
|
|
0.05598878860473633,
|
|
0.10357969254255295,
|
|
0.06921031326055527,
|
|
0.1325109452009201,
|
|
0.06760682165622711,
|
|
0.11279971897602081,
|
|
0.060408592224121094,
|
|
0.10416899621486664,
|
|
0.073752261698246,
|
|
0.19092032313346863
|
|
],
|
|
"activation_output_statistics": {
|
|
"first_quartile_mean": 0.27371567115187645,
|
|
"first_to_last_ratio": 2.5506400527811586,
|
|
"imbalance_abs_log_ratio": 0.9363443287654816,
|
|
"last_quartile_mean": 0.10731254331767559,
|
|
"mean": 0.13812353322282434,
|
|
"normalized": [
|
|
1.2816169309789325,
|
|
1.9197378727103604,
|
|
1.984246426234597,
|
|
2.7410906165788056,
|
|
0.38212679243445447,
|
|
0.661630126333285,
|
|
0.40535300030599286,
|
|
0.7499061899571856,
|
|
0.5010754622740751,
|
|
0.9593654470679528,
|
|
0.4894663500039642,
|
|
0.8166582214056853,
|
|
0.43735191834883375,
|
|
0.7541726871902314,
|
|
0.5339586960845188,
|
|
1.3822432620911254
|
|
],
|
|
"population_cv": 0.6672200194546598
|
|
},
|
|
"bits_per_byte": 1.8951255009474826,
|
|
"branch_output_rms_by_sublayer": [
|
|
0.08229005336761475,
|
|
0.1513136327266693,
|
|
0.08296028524637222,
|
|
0.06130215525627136,
|
|
0.2681557536125183,
|
|
0.023455867543816566,
|
|
0.11001982539892197,
|
|
0.01849420741200447,
|
|
0.047267504036426544,
|
|
0.01671545021235943,
|
|
0.035008542239665985,
|
|
0.04324181750416756,
|
|
0.04728955402970314,
|
|
0.02490479126572609,
|
|
0.04628673940896988,
|
|
0.041959211230278015,
|
|
0.055535849183797836,
|
|
0.03813145309686661,
|
|
0.04947454109787941,
|
|
0.083168163895607,
|
|
0.037920478731393814,
|
|
0.05371610447764397,
|
|
0.03159874305129051,
|
|
0.07563052326440811,
|
|
0.023936081677675247,
|
|
0.05285703018307686,
|
|
0.021295147016644478,
|
|
0.07199715077877045,
|
|
0.018087781965732574,
|
|
0.0706687644124031,
|
|
0.01923547312617302,
|
|
0.13923443853855133
|
|
],
|
|
"capture": {
|
|
"all_gradients_finite": true,
|
|
"all_gradients_present": true,
|
|
"count": 16,
|
|
"dtypes": [
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32"
|
|
],
|
|
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
|
"shape": [
|
|
16,
|
|
256,
|
|
192
|
|
],
|
|
"storage_unique": true
|
|
},
|
|
"core_parameter_grad_rms_by_block": [
|
|
0.0004973569892103084,
|
|
0.0002771506747073045,
|
|
0.0003595853430528865,
|
|
0.00025621221416812444,
|
|
0.0002736875635381804,
|
|
0.00034154014112545435,
|
|
0.0002723491519532143,
|
|
0.0002954443301970981,
|
|
0.0007531989220686374,
|
|
0.0008991377204894639,
|
|
0.0016884293947290697,
|
|
0.0013942943414501116,
|
|
0.0012489939262520248,
|
|
0.0008593069460315814,
|
|
0.000558781441282585,
|
|
0.00045362324645161216
|
|
],
|
|
"core_parameter_grad_statistics": {
|
|
"first_quartile_mean": 0.00034757630528465595,
|
|
"first_to_last_ratio": 0.44550989973264,
|
|
"imbalance_abs_log_ratio": 0.8085358106960822,
|
|
"last_quartile_mean": 0.0007801763900044509,
|
|
"mean": 0.0006518182716692285,
|
|
"normalized": [
|
|
0.7630301432585446,
|
|
0.42519623452339683,
|
|
0.5516650248727019,
|
|
0.39307307773376104,
|
|
0.41988323346214174,
|
|
0.523980618479459,
|
|
0.417829882638547,
|
|
0.45326181090398177,
|
|
1.1555351465367567,
|
|
1.3794300644363342,
|
|
2.5903376264755575,
|
|
2.139084468865058,
|
|
1.916168939317244,
|
|
1.3183228874989945,
|
|
0.8572656913277571,
|
|
0.6959351496697651
|
|
],
|
|
"population_cv": 0.6710654904340576
|
|
},
|
|
"depth_weights": [
|
|
{
|
|
"entropy_mean": 0.0,
|
|
"mean_weights": [
|
|
1.0
|
|
],
|
|
"sources": 1
|
|
},
|
|
{
|
|
"entropy_mean": 0.2534913420677185,
|
|
"mean_weights": [
|
|
0.9232981204986572,
|
|
0.07670190930366516
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.5290617346763611,
|
|
"mean_weights": [
|
|
0.7383309602737427,
|
|
0.26166898012161255
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.4635298252105713,
|
|
"mean_weights": [
|
|
0.8211046457290649,
|
|
0.17889533936977386
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.680541455745697,
|
|
"mean_weights": [
|
|
0.4997203052043915,
|
|
0.5002796649932861
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.9984116554260254,
|
|
"mean_weights": [
|
|
0.45704665780067444,
|
|
0.20008087158203125,
|
|
0.3428725004196167
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0224952697753906,
|
|
"mean_weights": [
|
|
0.22097927331924438,
|
|
0.2672557532787323,
|
|
0.5117650032043457
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 0.9886391162872314,
|
|
"mean_weights": [
|
|
0.4091557264328003,
|
|
0.17683157324790955,
|
|
0.41401270031929016
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.083122730255127,
|
|
"mean_weights": [
|
|
0.2858731150627136,
|
|
0.31187689304351807,
|
|
0.4022499620914459
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.262911081314087,
|
|
"mean_weights": [
|
|
0.24318832159042358,
|
|
0.1005454957485199,
|
|
0.3312486708164215,
|
|
0.325017511844635
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3697116374969482,
|
|
"mean_weights": [
|
|
0.2760387659072876,
|
|
0.21072158217430115,
|
|
0.21145658195018768,
|
|
0.30178308486938477
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.1750009059906006,
|
|
"mean_weights": [
|
|
0.40358343720436096,
|
|
0.10160854458808899,
|
|
0.10028939694166183,
|
|
0.3945186138153076
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.355452299118042,
|
|
"mean_weights": [
|
|
0.23967716097831726,
|
|
0.20835967361927032,
|
|
0.20370042324066162,
|
|
0.348262757062912
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.5273280143737793,
|
|
"mean_weights": [
|
|
0.2132018804550171,
|
|
0.09696971625089645,
|
|
0.14153459668159485,
|
|
0.2435910701751709,
|
|
0.3047027587890625
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.5883060693740845,
|
|
"mean_weights": [
|
|
0.2065391093492508,
|
|
0.16525766253471375,
|
|
0.16744214296340942,
|
|
0.19437497854232788,
|
|
0.26638609170913696
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.4673986434936523,
|
|
"mean_weights": [
|
|
0.19910070300102234,
|
|
0.10532978177070618,
|
|
0.08866854012012482,
|
|
0.23074698448181152,
|
|
0.37615397572517395
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6001981496810913,
|
|
"mean_weights": [
|
|
0.22463412582874298,
|
|
0.18846800923347473,
|
|
0.19619622826576233,
|
|
0.2022438496351242,
|
|
0.18845780193805695
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.658735752105713,
|
|
"mean_weights": [
|
|
0.16469183564186096,
|
|
0.09157001227140427,
|
|
0.085207998752594,
|
|
0.16954544186592102,
|
|
0.1403937190771103,
|
|
0.34859102964401245
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7217588424682617,
|
|
"mean_weights": [
|
|
0.1685413271188736,
|
|
0.1354280561208725,
|
|
0.15638956427574158,
|
|
0.13712063431739807,
|
|
0.10325069725513458,
|
|
0.29926973581314087
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.383854627609253,
|
|
"mean_weights": [
|
|
0.14582018554210663,
|
|
0.09375129640102386,
|
|
0.03237149864435196,
|
|
0.11780930310487747,
|
|
0.08738291263580322,
|
|
0.522864818572998
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.758126974105835,
|
|
"mean_weights": [
|
|
0.18006329238414764,
|
|
0.15370121598243713,
|
|
0.18850785493850708,
|
|
0.14673401415348053,
|
|
0.10883604735136032,
|
|
0.2221575826406479
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.4768933057785034,
|
|
"mean_weights": [
|
|
0.08639779686927795,
|
|
0.05247560143470764,
|
|
0.02685363218188286,
|
|
0.07686029374599457,
|
|
0.05649450421333313,
|
|
0.2057446390390396,
|
|
0.49517351388931274
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.754528522491455,
|
|
"mean_weights": [
|
|
0.1207476407289505,
|
|
0.10044647008180618,
|
|
0.12684977054595947,
|
|
0.0954352393746376,
|
|
0.07363080978393555,
|
|
0.10486453771591187,
|
|
0.37802553176879883
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.434983491897583,
|
|
"mean_weights": [
|
|
0.09186187386512756,
|
|
0.054585013538599014,
|
|
0.030214153230190277,
|
|
0.07775386422872543,
|
|
0.05580390244722366,
|
|
0.14904344081878662,
|
|
0.5407377481460571
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.814563274383545,
|
|
"mean_weights": [
|
|
0.13872049748897552,
|
|
0.10858511924743652,
|
|
0.09594539552927017,
|
|
0.10938869416713715,
|
|
0.08644531667232513,
|
|
0.13206249475479126,
|
|
0.32885247468948364
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.5878477096557617,
|
|
"mean_weights": [
|
|
0.06617997586727142,
|
|
0.04098459705710411,
|
|
0.023611580953001976,
|
|
0.05256733298301697,
|
|
0.03869973123073578,
|
|
0.09328385442495346,
|
|
0.30499327182769775,
|
|
0.37967967987060547
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.8095037937164307,
|
|
"mean_weights": [
|
|
0.0948307067155838,
|
|
0.0694318562746048,
|
|
0.0552619993686676,
|
|
0.07500732690095901,
|
|
0.061008021235466,
|
|
0.09275255352258682,
|
|
0.18348734080791473,
|
|
0.3682202100753784
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.5925970077514648,
|
|
"mean_weights": [
|
|
0.07132496684789658,
|
|
0.04804965853691101,
|
|
0.030455447733402252,
|
|
0.054331667721271515,
|
|
0.038998860865831375,
|
|
0.07944463938474655,
|
|
0.22967059910297394,
|
|
0.4477241635322571
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.8567713499069214,
|
|
"mean_weights": [
|
|
0.09893427789211273,
|
|
0.07625022530555725,
|
|
0.05004327371716499,
|
|
0.08632857352495193,
|
|
0.05865295231342316,
|
|
0.10661771893501282,
|
|
0.2290721982717514,
|
|
0.2941007912158966
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.7706098556518555,
|
|
"mean_weights": [
|
|
0.0600857213139534,
|
|
0.04508119821548462,
|
|
0.026483699679374695,
|
|
0.05021750181913376,
|
|
0.03222868591547012,
|
|
0.07324853539466858,
|
|
0.17921699583530426,
|
|
0.23399141430854797,
|
|
0.2994462847709656
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.0186803340911865,
|
|
"mean_weights": [
|
|
0.08094437420368195,
|
|
0.07421140372753143,
|
|
0.03662773221731186,
|
|
0.07851459085941315,
|
|
0.05622554570436478,
|
|
0.11645162105560303,
|
|
0.2003173828125,
|
|
0.18591775000095367,
|
|
0.17078962922096252
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 1.8739352226257324,
|
|
"mean_weights": [
|
|
0.07170331478118896,
|
|
0.05929408222436905,
|
|
0.03108443133533001,
|
|
0.05872419476509094,
|
|
0.03831540048122406,
|
|
0.07797900587320328,
|
|
0.1519450843334198,
|
|
0.2740318179130554,
|
|
0.23692265152931213
|
|
],
|
|
"sources": 9
|
|
}
|
|
],
|
|
"layer_input_rms_by_sublayer": [
|
|
0.061437346041202545,
|
|
0.05765428766608238,
|
|
0.07246167957782745,
|
|
0.07205748558044434,
|
|
0.1429983228445053,
|
|
0.13230633735656738,
|
|
0.18651185929775238,
|
|
0.18575440347194672,
|
|
0.20683544874191284,
|
|
0.14580051600933075,
|
|
0.1242423728108406,
|
|
0.07512427866458893,
|
|
0.12742595374584198,
|
|
0.08523519337177277,
|
|
0.10536280274391174,
|
|
0.07712213695049286,
|
|
0.12031000852584839,
|
|
0.06152798980474472,
|
|
0.08708149939775467,
|
|
0.05821346119046211,
|
|
0.10479474812746048,
|
|
0.03660533204674721,
|
|
0.06689487397670746,
|
|
0.049551647156476974,
|
|
0.06684304773807526,
|
|
0.04078952595591545,
|
|
0.04357287660241127,
|
|
0.04368526116013527,
|
|
0.05231116712093353,
|
|
0.03767167031764984,
|
|
0.04748373478651047,
|
|
0.0490146204829216
|
|
],
|
|
"loss_nats": 1.3136008977890015,
|
|
"loss_scale": 1.0,
|
|
"output_weights": {
|
|
"entropy_mean": 1.685392141342163,
|
|
"mean_weights": [
|
|
0.08037909120321274,
|
|
0.03995431959629059,
|
|
0.0371873639523983,
|
|
0.05237603187561035,
|
|
0.03903363645076752,
|
|
0.04661020264029503,
|
|
0.0628754124045372,
|
|
0.26517099142074585,
|
|
0.37641292810440063
|
|
],
|
|
"sources": 9
|
|
},
|
|
"step": 8000,
|
|
"stream_state_rms_by_sublayer": [
|
|
0.08229005336761475,
|
|
0.17702145874500275,
|
|
0.22416120767593384,
|
|
0.2651609778404236,
|
|
0.2681557536125183,
|
|
0.2740711271762848,
|
|
0.37491148710250854,
|
|
0.3786091208457947,
|
|
0.047267504036426544,
|
|
0.05278070271015167,
|
|
0.07517243921756744,
|
|
0.09138669073581696,
|
|
0.04728955402970314,
|
|
0.05598878860473633,
|
|
0.08201033622026443,
|
|
0.10357969254255295,
|
|
0.055535849183797836,
|
|
0.06921031326055527,
|
|
0.0903589129447937,
|
|
0.1325109452009201,
|
|
0.037920478731393814,
|
|
0.06760682165622711,
|
|
0.07624635845422745,
|
|
0.11279971897602081,
|
|
0.023936081677675247,
|
|
0.060408592224121094,
|
|
0.0652509555220604,
|
|
0.10416899621486664,
|
|
0.018087781965732574,
|
|
0.073752261698246,
|
|
0.0771600753068924,
|
|
0.19092032313346863
|
|
]
|
|
}
|
|
],
|
|
"environment": {
|
|
"autocast": "cuda-bfloat16-forward-fp32-cross-entropy",
|
|
"compile": false,
|
|
"compute_capability": [
|
|
12,
|
|
0
|
|
],
|
|
"cublas_workspace_config": ":4096:8",
|
|
"cuda": "12.8",
|
|
"deterministic_algorithms": true,
|
|
"gpu": "NVIDIA GeForce RTX 5090",
|
|
"python": "3.10.14",
|
|
"torch": "2.11.0+cu128"
|
|
},
|
|
"evaluations": [
|
|
{
|
|
"bits_per_byte": 8.073347956844582,
|
|
"cross_entropy_nats": 5.596018373966217,
|
|
"step": 0
|
|
},
|
|
{
|
|
"bits_per_byte": 5.112499455104693,
|
|
"cross_entropy_nats": 3.5437145829200745,
|
|
"step": 100
|
|
},
|
|
{
|
|
"bits_per_byte": 3.1188867992707414,
|
|
"cross_entropy_nats": 2.1618475914001465,
|
|
"step": 500
|
|
},
|
|
{
|
|
"bits_per_byte": 1.962477669145846,
|
|
"cross_entropy_nats": 1.3602858632802963,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8161008917328916,
|
|
"cross_entropy_nats": 1.2588252127170563,
|
|
"step": 4000
|
|
},
|
|
{
|
|
"bits_per_byte": 1.737390666804078,
|
|
"cross_entropy_nats": 1.20426744222641,
|
|
"step": 8000
|
|
}
|
|
],
|
|
"gradient_gate": null,
|
|
"hashes": {
|
|
"final_mixer_parameters": "0bff629038a5dd94843a123dbdfc2bdaca027f8a9025a2a7688adc4ec0647b31",
|
|
"final_model_state": "efa098925d29c4e5158cd73c7764d7014b6c4ee53df36a146dbd0cff04321b62",
|
|
"final_optimizer_state": "b86655a1b88951524b7f3bfca00df30a0958104385aa444bf7b109fbe1d9e51b",
|
|
"final_public_parameters": "87107ee0633f1863f2fcaf1d5f91af597c67acce2c9777fec3486aa4fd1463ef",
|
|
"initial_mixer_parameters": "64fe0b90f5ea84ed88b6d93838558443c700c2b89ade2455ed526bd8db5d506f",
|
|
"initial_public_parameter_elements": 9541824,
|
|
"initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe",
|
|
"initial_public_parameter_tensors": 115,
|
|
"initial_public_parameters": "af2724a1c34bcfd61d8a8bef402246430898c815e56b6e5c5257949a4eb0e7b1"
|
|
},
|
|
"manifest": {
|
|
"diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716",
|
|
"file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371",
|
|
"formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
|
|
"input_gate_tensor_hashes": {
|
|
"0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434",
|
|
"1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94",
|
|
"7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc"
|
|
},
|
|
"path": "experiments/k3/attnres_gradient/manifest.json",
|
|
"validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338"
|
|
},
|
|
"model": {
|
|
"attnres_aggregation_groups": 8,
|
|
"context": 256,
|
|
"d_ff": 768,
|
|
"d_head": 32,
|
|
"d_model": 192,
|
|
"heads": 6,
|
|
"layers": 16,
|
|
"parameters": {
|
|
"core": 9541824,
|
|
"embedding": 98304,
|
|
"mixer": 12672,
|
|
"total": 9554496
|
|
},
|
|
"sublayers": 32,
|
|
"sublayers_per_attnres_group": 4,
|
|
"transformer_blocks_per_attnres_group": 2,
|
|
"vocabulary": 256
|
|
},
|
|
"optimizer": {
|
|
"betas": [
|
|
0.9,
|
|
0.95
|
|
],
|
|
"epsilon": 1e-08,
|
|
"grad_clip": 1.0,
|
|
"min_lr": 3e-05,
|
|
"name": "AdamW",
|
|
"peak_lr": 0.0003,
|
|
"warmup_steps": 400,
|
|
"weight_decay_ndim_ge_2": 0.1
|
|
},
|
|
"protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
|
|
"run_kind": "formal",
|
|
"schema_version": 1,
|
|
"seed": 2026073001,
|
|
"steps": 8000,
|
|
"target_bytes_seen": 65536000,
|
|
"timing": {
|
|
"mean_ms": 54.63475258274364,
|
|
"measured_steps": 7980,
|
|
"median_ms": 54.539098986424506,
|
|
"p95_ms": 55.062895112496335,
|
|
"peak_allocated_bytes": 6536921088,
|
|
"peak_reserved_bytes": 6889144320,
|
|
"warmup_steps_excluded": 20
|
|
},
|
|
"training_history": [
|
|
{
|
|
"bits_per_byte": 8.084436624250287,
|
|
"learning_rate": 7.499999999999999e-07,
|
|
"loss_nats": 5.603704452514648,
|
|
"step": 1,
|
|
"unclipped_grad_norm": 18.78145980834961
|
|
},
|
|
{
|
|
"bits_per_byte": 7.484355446790379,
|
|
"learning_rate": 7.499999999999999e-06,
|
|
"loss_nats": 5.187759876251221,
|
|
"step": 10,
|
|
"unclipped_grad_norm": 12.867269515991211
|
|
},
|
|
{
|
|
"bits_per_byte": 6.795359238834398,
|
|
"learning_rate": 1.4999999999999999e-05,
|
|
"loss_nats": 4.710184097290039,
|
|
"step": 20,
|
|
"unclipped_grad_norm": 4.549929141998291
|
|
},
|
|
{
|
|
"bits_per_byte": 6.560912489178218,
|
|
"learning_rate": 2.2499999999999998e-05,
|
|
"loss_nats": 4.547677993774414,
|
|
"step": 30,
|
|
"unclipped_grad_norm": 2.9244894981384277
|
|
},
|
|
{
|
|
"bits_per_byte": 6.418922236764676,
|
|
"learning_rate": 2.9999999999999997e-05,
|
|
"loss_nats": 4.449257850646973,
|
|
"step": 40,
|
|
"unclipped_grad_norm": 2.8415465354919434
|
|
},
|
|
{
|
|
"bits_per_byte": 6.252045406840682,
|
|
"learning_rate": 3.75e-05,
|
|
"loss_nats": 4.333587646484375,
|
|
"step": 50,
|
|
"unclipped_grad_norm": 2.8266561031341553
|
|
},
|
|
{
|
|
"bits_per_byte": 5.950088587414815,
|
|
"learning_rate": 4.4999999999999996e-05,
|
|
"loss_nats": 4.124287128448486,
|
|
"step": 60,
|
|
"unclipped_grad_norm": 2.6689200401306152
|
|
},
|
|
{
|
|
"bits_per_byte": 5.70102021674758,
|
|
"learning_rate": 5.2499999999999995e-05,
|
|
"loss_nats": 3.951646089553833,
|
|
"step": 70,
|
|
"unclipped_grad_norm": 2.5050199031829834
|
|
},
|
|
{
|
|
"bits_per_byte": 5.519442653793135,
|
|
"learning_rate": 5.9999999999999995e-05,
|
|
"loss_nats": 3.8257861137390137,
|
|
"step": 80,
|
|
"unclipped_grad_norm": 2.352375030517578
|
|
},
|
|
{
|
|
"bits_per_byte": 5.302846327435307,
|
|
"learning_rate": 6.749999999999999e-05,
|
|
"loss_nats": 3.6756529808044434,
|
|
"step": 90,
|
|
"unclipped_grad_norm": 2.2169463634490967
|
|
},
|
|
{
|
|
"bits_per_byte": 5.139077911943008,
|
|
"learning_rate": 7.5e-05,
|
|
"loss_nats": 3.5621373653411865,
|
|
"step": 100,
|
|
"unclipped_grad_norm": 2.0478744506835938
|
|
},
|
|
{
|
|
"bits_per_byte": 4.944703463903163,
|
|
"learning_rate": 8.249999999999999e-05,
|
|
"loss_nats": 3.4274072647094727,
|
|
"step": 110,
|
|
"unclipped_grad_norm": 1.971948504447937
|
|
},
|
|
{
|
|
"bits_per_byte": 4.730622899762156,
|
|
"learning_rate": 8.999999999999999e-05,
|
|
"loss_nats": 3.279017925262451,
|
|
"step": 120,
|
|
"unclipped_grad_norm": 1.6964443922042847
|
|
},
|
|
{
|
|
"bits_per_byte": 4.549973699157069,
|
|
"learning_rate": 9.75e-05,
|
|
"loss_nats": 3.153801441192627,
|
|
"step": 130,
|
|
"unclipped_grad_norm": 1.537685751914978
|
|
},
|
|
{
|
|
"bits_per_byte": 4.410896832849143,
|
|
"learning_rate": 0.00010499999999999999,
|
|
"loss_nats": 3.057400703430176,
|
|
"step": 140,
|
|
"unclipped_grad_norm": 1.355125904083252
|
|
},
|
|
{
|
|
"bits_per_byte": 4.261557417798932,
|
|
"learning_rate": 0.0001125,
|
|
"loss_nats": 2.9538865089416504,
|
|
"step": 150,
|
|
"unclipped_grad_norm": 1.2107940912246704
|
|
},
|
|
{
|
|
"bits_per_byte": 4.165243693662742,
|
|
"learning_rate": 0.00011999999999999999,
|
|
"loss_nats": 2.887126922607422,
|
|
"step": 160,
|
|
"unclipped_grad_norm": 0.8971752524375916
|
|
},
|
|
{
|
|
"bits_per_byte": 4.038897327136983,
|
|
"learning_rate": 0.00012749999999999998,
|
|
"loss_nats": 2.7995502948760986,
|
|
"step": 170,
|
|
"unclipped_grad_norm": 0.8723403215408325
|
|
},
|
|
{
|
|
"bits_per_byte": 3.967892225936462,
|
|
"learning_rate": 0.00013499999999999997,
|
|
"loss_nats": 2.750333309173584,
|
|
"step": 180,
|
|
"unclipped_grad_norm": 0.7337902784347534
|
|
},
|
|
{
|
|
"bits_per_byte": 3.8789046187617258,
|
|
"learning_rate": 0.0001425,
|
|
"loss_nats": 2.6886518001556396,
|
|
"step": 190,
|
|
"unclipped_grad_norm": 0.8901073336601257
|
|
},
|
|
{
|
|
"bits_per_byte": 3.824914105176364,
|
|
"learning_rate": 0.00015,
|
|
"loss_nats": 2.651228427886963,
|
|
"step": 200,
|
|
"unclipped_grad_norm": 0.642342746257782
|
|
},
|
|
{
|
|
"bits_per_byte": 3.7744440764541642,
|
|
"learning_rate": 0.0001575,
|
|
"loss_nats": 2.6162452697753906,
|
|
"step": 210,
|
|
"unclipped_grad_norm": 0.6421008110046387
|
|
},
|
|
{
|
|
"bits_per_byte": 3.68308276473222,
|
|
"learning_rate": 0.00016499999999999997,
|
|
"loss_nats": 2.5529184341430664,
|
|
"step": 220,
|
|
"unclipped_grad_norm": 0.7395935654640198
|
|
},
|
|
{
|
|
"bits_per_byte": 3.6743831943209755,
|
|
"learning_rate": 0.0001725,
|
|
"loss_nats": 2.5468883514404297,
|
|
"step": 230,
|
|
"unclipped_grad_norm": 0.6173842549324036
|
|
},
|
|
{
|
|
"bits_per_byte": 3.575872219837362,
|
|
"learning_rate": 0.00017999999999999998,
|
|
"loss_nats": 2.4786057472229004,
|
|
"step": 240,
|
|
"unclipped_grad_norm": 0.8246023058891296
|
|
},
|
|
{
|
|
"bits_per_byte": 3.6039487315560175,
|
|
"learning_rate": 0.0001875,
|
|
"loss_nats": 2.4980669021606445,
|
|
"step": 250,
|
|
"unclipped_grad_norm": 0.7212417721748352
|
|
},
|
|
{
|
|
"bits_per_byte": 3.6017153648519127,
|
|
"learning_rate": 0.000195,
|
|
"loss_nats": 2.496518850326538,
|
|
"step": 260,
|
|
"unclipped_grad_norm": 0.6610386967658997
|
|
},
|
|
{
|
|
"bits_per_byte": 3.6077832567396544,
|
|
"learning_rate": 0.00020249999999999996,
|
|
"loss_nats": 2.5007247924804688,
|
|
"step": 270,
|
|
"unclipped_grad_norm": 0.8672931790351868
|
|
},
|
|
{
|
|
"bits_per_byte": 3.524929582790591,
|
|
"learning_rate": 0.00020999999999999998,
|
|
"loss_nats": 2.4432950019836426,
|
|
"step": 280,
|
|
"unclipped_grad_norm": 0.7419575452804565
|
|
},
|
|
{
|
|
"bits_per_byte": 3.446865969633359,
|
|
"learning_rate": 0.00021749999999999997,
|
|
"loss_nats": 2.3891854286193848,
|
|
"step": 290,
|
|
"unclipped_grad_norm": 0.8831741809844971
|
|
},
|
|
{
|
|
"bits_per_byte": 3.491855255237863,
|
|
"learning_rate": 0.000225,
|
|
"loss_nats": 2.4203696250915527,
|
|
"step": 300,
|
|
"unclipped_grad_norm": 0.9954257607460022
|
|
},
|
|
{
|
|
"bits_per_byte": 3.448469191904702,
|
|
"learning_rate": 0.00023249999999999996,
|
|
"loss_nats": 2.390296697616577,
|
|
"step": 310,
|
|
"unclipped_grad_norm": 1.0782777070999146
|
|
},
|
|
{
|
|
"bits_per_byte": 3.4360496367552953,
|
|
"learning_rate": 0.00023999999999999998,
|
|
"loss_nats": 2.381688117980957,
|
|
"step": 320,
|
|
"unclipped_grad_norm": 0.9428723454475403
|
|
},
|
|
{
|
|
"bits_per_byte": 3.415760155502406,
|
|
"learning_rate": 0.0002475,
|
|
"loss_nats": 2.367624521255493,
|
|
"step": 330,
|
|
"unclipped_grad_norm": 1.0516632795333862
|
|
},
|
|
{
|
|
"bits_per_byte": 3.5107169365232886,
|
|
"learning_rate": 0.00025499999999999996,
|
|
"loss_nats": 2.433443546295166,
|
|
"step": 340,
|
|
"unclipped_grad_norm": 1.1474740505218506
|
|
},
|
|
{
|
|
"bits_per_byte": 3.4490202243180663,
|
|
"learning_rate": 0.0002625,
|
|
"loss_nats": 2.390678644180298,
|
|
"step": 350,
|
|
"unclipped_grad_norm": 1.2037273645401
|
|
},
|
|
{
|
|
"bits_per_byte": 3.401095538828636,
|
|
"learning_rate": 0.00026999999999999995,
|
|
"loss_nats": 2.357459783554077,
|
|
"step": 360,
|
|
"unclipped_grad_norm": 1.0446844100952148
|
|
},
|
|
{
|
|
"bits_per_byte": 3.376710806693402,
|
|
"learning_rate": 0.00027749999999999997,
|
|
"loss_nats": 2.34055757522583,
|
|
"step": 370,
|
|
"unclipped_grad_norm": 0.8781102895736694
|
|
},
|
|
{
|
|
"bits_per_byte": 3.422498435763211,
|
|
"learning_rate": 0.000285,
|
|
"loss_nats": 2.3722951412200928,
|
|
"step": 380,
|
|
"unclipped_grad_norm": 0.9067438244819641
|
|
},
|
|
{
|
|
"bits_per_byte": 3.35829834409203,
|
|
"learning_rate": 0.0002925,
|
|
"loss_nats": 2.3277950286865234,
|
|
"step": 390,
|
|
"unclipped_grad_norm": 1.0105773210525513
|
|
},
|
|
{
|
|
"bits_per_byte": 3.298973960572809,
|
|
"learning_rate": 0.0003,
|
|
"loss_nats": 2.2866744995117188,
|
|
"step": 400,
|
|
"unclipped_grad_norm": 1.1294934749603271
|
|
},
|
|
{
|
|
"bits_per_byte": 3.2943352445137375,
|
|
"learning_rate": 0.0002999988466112388,
|
|
"loss_nats": 2.283459186553955,
|
|
"step": 410,
|
|
"unclipped_grad_norm": 1.2423999309539795
|
|
},
|
|
{
|
|
"bits_per_byte": 3.324363071389082,
|
|
"learning_rate": 0.00029999538646466355,
|
|
"loss_nats": 2.3042728900909424,
|
|
"step": 420,
|
|
"unclipped_grad_norm": 1.016541838645935
|
|
},
|
|
{
|
|
"bits_per_byte": 3.29597423917646,
|
|
"learning_rate": 0.0002999896196193985,
|
|
"loss_nats": 2.284595251083374,
|
|
"step": 430,
|
|
"unclipped_grad_norm": 1.1856372356414795
|
|
},
|
|
{
|
|
"bits_per_byte": 3.3349637380229638,
|
|
"learning_rate": 0.00029998154617398326,
|
|
"loss_nats": 2.3116207122802734,
|
|
"step": 440,
|
|
"unclipped_grad_norm": 0.8890857696533203
|
|
},
|
|
{
|
|
"bits_per_byte": 3.2500166912475987,
|
|
"learning_rate": 0.0002999711662663707,
|
|
"loss_nats": 2.252739906311035,
|
|
"step": 450,
|
|
"unclipped_grad_norm": 1.1027717590332031
|
|
},
|
|
{
|
|
"bits_per_byte": 3.273506081702441,
|
|
"learning_rate": 0.0002999584800739247,
|
|
"loss_nats": 2.269021511077881,
|
|
"step": 460,
|
|
"unclipped_grad_norm": 0.9078090190887451
|
|
},
|
|
{
|
|
"bits_per_byte": 3.199572459923028,
|
|
"learning_rate": 0.0002999434878134173,
|
|
"loss_nats": 2.2177746295928955,
|
|
"step": 470,
|
|
"unclipped_grad_norm": 1.019633173942566
|
|
},
|
|
{
|
|
"bits_per_byte": 3.1991851549932853,
|
|
"learning_rate": 0.000299926189741025,
|
|
"loss_nats": 2.217506170272827,
|
|
"step": 480,
|
|
"unclipped_grad_norm": 0.8520210981369019
|
|
},
|
|
{
|
|
"bits_per_byte": 3.1319430341936423,
|
|
"learning_rate": 0.000299906586152324,
|
|
"loss_nats": 2.1708974838256836,
|
|
"step": 490,
|
|
"unclipped_grad_norm": 2.670640707015991
|
|
},
|
|
{
|
|
"bits_per_byte": 3.0465694704441373,
|
|
"learning_rate": 0.0002998846773822856,
|
|
"loss_nats": 2.1117210388183594,
|
|
"step": 500,
|
|
"unclipped_grad_norm": 0.961890697479248
|
|
},
|
|
{
|
|
"bits_per_byte": 3.080116406321437,
|
|
"learning_rate": 0.0002998604638052702,
|
|
"loss_nats": 2.1349740028381348,
|
|
"step": 510,
|
|
"unclipped_grad_norm": 1.4813798666000366
|
|
},
|
|
{
|
|
"bits_per_byte": 2.9958545384271087,
|
|
"learning_rate": 0.0002998339458350211,
|
|
"loss_nats": 2.076568126678467,
|
|
"step": 520,
|
|
"unclipped_grad_norm": 1.3385756015777588
|
|
},
|
|
{
|
|
"bits_per_byte": 3.0340161127923593,
|
|
"learning_rate": 0.0002998051239246572,
|
|
"loss_nats": 2.1030197143554688,
|
|
"step": 530,
|
|
"unclipped_grad_norm": 1.0723164081573486
|
|
},
|
|
{
|
|
"bits_per_byte": 2.9825217554016206,
|
|
"learning_rate": 0.00029977399856666537,
|
|
"loss_nats": 2.067326545715332,
|
|
"step": 540,
|
|
"unclipped_grad_norm": 1.0559418201446533
|
|
},
|
|
{
|
|
"bits_per_byte": 2.964598755424604,
|
|
"learning_rate": 0.0002997405702928922,
|
|
"loss_nats": 2.054903268814087,
|
|
"step": 550,
|
|
"unclipped_grad_norm": 1.3542453050613403
|
|
},
|
|
{
|
|
"bits_per_byte": 2.9389846913010023,
|
|
"learning_rate": 0.0002997048396745345,
|
|
"loss_nats": 2.037148952484131,
|
|
"step": 560,
|
|
"unclipped_grad_norm": 1.3118830919265747
|
|
},
|
|
{
|
|
"bits_per_byte": 2.888880297694537,
|
|
"learning_rate": 0.0002996668073221302,
|
|
"loss_nats": 2.0024192333221436,
|
|
"step": 570,
|
|
"unclipped_grad_norm": 1.1606483459472656
|
|
},
|
|
{
|
|
"bits_per_byte": 2.9506829512126904,
|
|
"learning_rate": 0.000299626473885547,
|
|
"loss_nats": 2.045257568359375,
|
|
"step": 580,
|
|
"unclipped_grad_norm": 1.3442156314849854
|
|
},
|
|
{
|
|
"bits_per_byte": 2.8292920927888927,
|
|
"learning_rate": 0.00029958384005397226,
|
|
"loss_nats": 1.961115837097168,
|
|
"step": 590,
|
|
"unclipped_grad_norm": 0.9916635751724243
|
|
},
|
|
{
|
|
"bits_per_byte": 2.904369399196543,
|
|
"learning_rate": 0.0002995389065559004,
|
|
"loss_nats": 2.013155460357666,
|
|
"step": 600,
|
|
"unclipped_grad_norm": 1.5107635259628296
|
|
},
|
|
{
|
|
"bits_per_byte": 2.8103640262002334,
|
|
"learning_rate": 0.000299491674159121,
|
|
"loss_nats": 1.947995901107788,
|
|
"step": 610,
|
|
"unclipped_grad_norm": 1.3907541036605835
|
|
},
|
|
{
|
|
"bits_per_byte": 2.8376170849865576,
|
|
"learning_rate": 0.00029944214367070535,
|
|
"loss_nats": 1.966886281967163,
|
|
"step": 620,
|
|
"unclipped_grad_norm": 1.3457273244857788
|
|
},
|
|
{
|
|
"bits_per_byte": 2.772177686333259,
|
|
"learning_rate": 0.00029939031593699284,
|
|
"loss_nats": 1.9215271472930908,
|
|
"step": 630,
|
|
"unclipped_grad_norm": 1.2430039644241333
|
|
},
|
|
{
|
|
"bits_per_byte": 2.7475067750669893,
|
|
"learning_rate": 0.00029933619184357654,
|
|
"loss_nats": 1.9044265747070312,
|
|
"step": 640,
|
|
"unclipped_grad_norm": 1.7454813718795776
|
|
},
|
|
{
|
|
"bits_per_byte": 2.664434339186028,
|
|
"learning_rate": 0.00029927977231528774,
|
|
"loss_nats": 1.8468451499938965,
|
|
"step": 650,
|
|
"unclipped_grad_norm": 1.3669214248657227
|
|
},
|
|
{
|
|
"bits_per_byte": 2.607502578305611,
|
|
"learning_rate": 0.00029922105831618057,
|
|
"loss_nats": 1.8073830604553223,
|
|
"step": 660,
|
|
"unclipped_grad_norm": 2.4166994094848633
|
|
},
|
|
{
|
|
"bits_per_byte": 2.640132846653812,
|
|
"learning_rate": 0.0002991600508495153,
|
|
"loss_nats": 1.830000638961792,
|
|
"step": 670,
|
|
"unclipped_grad_norm": 1.6203631162643433
|
|
},
|
|
{
|
|
"bits_per_byte": 2.577274563624662,
|
|
"learning_rate": 0.00029909675095774114,
|
|
"loss_nats": 1.7864305973052979,
|
|
"step": 680,
|
|
"unclipped_grad_norm": 1.443710207939148
|
|
},
|
|
{
|
|
"bits_per_byte": 2.648628101675831,
|
|
"learning_rate": 0.0002990311597224787,
|
|
"loss_nats": 1.8358891010284424,
|
|
"step": 690,
|
|
"unclipped_grad_norm": 1.1127547025680542
|
|
},
|
|
{
|
|
"bits_per_byte": 2.5587637269469963,
|
|
"learning_rate": 0.00029896327826450103,
|
|
"loss_nats": 1.7735998630523682,
|
|
"step": 700,
|
|
"unclipped_grad_norm": 1.3421220779418945
|
|
},
|
|
{
|
|
"bits_per_byte": 2.584466534118443,
|
|
"learning_rate": 0.0002988931077437152,
|
|
"loss_nats": 1.7914156913757324,
|
|
"step": 710,
|
|
"unclipped_grad_norm": 1.5519825220108032
|
|
},
|
|
{
|
|
"bits_per_byte": 2.61796909847182,
|
|
"learning_rate": 0.00029882064935914164,
|
|
"loss_nats": 1.8146378993988037,
|
|
"step": 720,
|
|
"unclipped_grad_norm": 2.2321839332580566
|
|
},
|
|
{
|
|
"bits_per_byte": 2.574501859327452,
|
|
"learning_rate": 0.0002987459043488943,
|
|
"loss_nats": 1.7845087051391602,
|
|
"step": 730,
|
|
"unclipped_grad_norm": 1.835390567779541
|
|
},
|
|
{
|
|
"bits_per_byte": 2.509336257054327,
|
|
"learning_rate": 0.0002986688739901591,
|
|
"loss_nats": 1.7393393516540527,
|
|
"step": 740,
|
|
"unclipped_grad_norm": 1.5465850830078125
|
|
},
|
|
{
|
|
"bits_per_byte": 2.4645351204698667,
|
|
"learning_rate": 0.0002985895595991723,
|
|
"loss_nats": 1.7082855701446533,
|
|
"step": 750,
|
|
"unclipped_grad_norm": 1.1801629066467285
|
|
},
|
|
{
|
|
"bits_per_byte": 2.4095216540771736,
|
|
"learning_rate": 0.00029850796253119824,
|
|
"loss_nats": 1.6701531410217285,
|
|
"step": 760,
|
|
"unclipped_grad_norm": 1.296120524406433
|
|
},
|
|
{
|
|
"bits_per_byte": 2.584656058999694,
|
|
"learning_rate": 0.00029842408418050544,
|
|
"loss_nats": 1.7915470600128174,
|
|
"step": 770,
|
|
"unclipped_grad_norm": 2.048929452896118
|
|
},
|
|
{
|
|
"bits_per_byte": 2.4472219709728096,
|
|
"learning_rate": 0.0002983379259803436,
|
|
"loss_nats": 1.6962850093841553,
|
|
"step": 780,
|
|
"unclipped_grad_norm": 1.45075261592865
|
|
},
|
|
{
|
|
"bits_per_byte": 2.4007569022412523,
|
|
"learning_rate": 0.0002982494894029186,
|
|
"loss_nats": 1.664077877998352,
|
|
"step": 790,
|
|
"unclipped_grad_norm": 1.5680437088012695
|
|
},
|
|
{
|
|
"bits_per_byte": 2.3632249563608103,
|
|
"learning_rate": 0.0002981587759593675,
|
|
"loss_nats": 1.6380627155303955,
|
|
"step": 800,
|
|
"unclipped_grad_norm": 1.358572006225586
|
|
},
|
|
{
|
|
"bits_per_byte": 2.4340216145884854,
|
|
"learning_rate": 0.00029806578719973283,
|
|
"loss_nats": 1.6871352195739746,
|
|
"step": 810,
|
|
"unclipped_grad_norm": 1.6542795896530151
|
|
},
|
|
{
|
|
"bits_per_byte": 2.3610528154804133,
|
|
"learning_rate": 0.00029797052471293596,
|
|
"loss_nats": 1.6365571022033691,
|
|
"step": 820,
|
|
"unclipped_grad_norm": 1.4101909399032593
|
|
},
|
|
{
|
|
"bits_per_byte": 2.429125956449025,
|
|
"learning_rate": 0.0002978729901267499,
|
|
"loss_nats": 1.683741807937622,
|
|
"step": 830,
|
|
"unclipped_grad_norm": 1.4574730396270752
|
|
},
|
|
{
|
|
"bits_per_byte": 2.509321810511654,
|
|
"learning_rate": 0.0002977731851077716,
|
|
"loss_nats": 1.7393293380737305,
|
|
"step": 840,
|
|
"unclipped_grad_norm": 1.71798837184906
|
|
},
|
|
{
|
|
"bits_per_byte": 2.3876714302677042,
|
|
"learning_rate": 0.0002976711113613935,
|
|
"loss_nats": 1.6550077199935913,
|
|
"step": 850,
|
|
"unclipped_grad_norm": 1.7788947820663452
|
|
},
|
|
{
|
|
"bits_per_byte": 2.402646303643631,
|
|
"learning_rate": 0.0002975667706317743,
|
|
"loss_nats": 1.665387511253357,
|
|
"step": 860,
|
|
"unclipped_grad_norm": 1.3990782499313354
|
|
},
|
|
{
|
|
"bits_per_byte": 2.3188210996998437,
|
|
"learning_rate": 0.00029746016470180915,
|
|
"loss_nats": 1.6072843074798584,
|
|
"step": 870,
|
|
"unclipped_grad_norm": 1.5185577869415283
|
|
},
|
|
{
|
|
"bits_per_byte": 2.342377219423221,
|
|
"learning_rate": 0.00029735129539309926,
|
|
"loss_nats": 1.6236121654510498,
|
|
"step": 880,
|
|
"unclipped_grad_norm": 1.4461203813552856
|
|
},
|
|
{
|
|
"bits_per_byte": 2.2667829331670766,
|
|
"learning_rate": 0.0002972401645659207,
|
|
"loss_nats": 1.571214199066162,
|
|
"step": 890,
|
|
"unclipped_grad_norm": 1.2836962938308716
|
|
},
|
|
{
|
|
"bits_per_byte": 2.284366783356592,
|
|
"learning_rate": 0.00029712677411919283,
|
|
"loss_nats": 1.583402395248413,
|
|
"step": 900,
|
|
"unclipped_grad_norm": 1.337744116783142
|
|
},
|
|
{
|
|
"bits_per_byte": 2.3096057533186616,
|
|
"learning_rate": 0.0002970111259904454,
|
|
"loss_nats": 1.6008967161178589,
|
|
"step": 910,
|
|
"unclipped_grad_norm": 1.3567826747894287
|
|
},
|
|
{
|
|
"bits_per_byte": 2.2867041995644684,
|
|
"learning_rate": 0.0002968932221557859,
|
|
"loss_nats": 1.5850225687026978,
|
|
"step": 920,
|
|
"unclipped_grad_norm": 1.568986177444458
|
|
},
|
|
{
|
|
"bits_per_byte": 2.304558234498489,
|
|
"learning_rate": 0.0002967730646298655,
|
|
"loss_nats": 1.597398042678833,
|
|
"step": 930,
|
|
"unclipped_grad_norm": 1.5530638694763184
|
|
},
|
|
{
|
|
"bits_per_byte": 2.297734650842856,
|
|
"learning_rate": 0.0002966506554658449,
|
|
"loss_nats": 1.5926682949066162,
|
|
"step": 940,
|
|
"unclipped_grad_norm": 1.5203993320465088
|
|
},
|
|
{
|
|
"bits_per_byte": 2.280084587332759,
|
|
"learning_rate": 0.00029652599675535876,
|
|
"loss_nats": 1.5804342031478882,
|
|
"step": 950,
|
|
"unclipped_grad_norm": 1.4065279960632324
|
|
},
|
|
{
|
|
"bits_per_byte": 2.325858113825197,
|
|
"learning_rate": 0.0002963990906284808,
|
|
"loss_nats": 1.6121619939804077,
|
|
"step": 960,
|
|
"unclipped_grad_norm": 1.3770049810409546
|
|
},
|
|
{
|
|
"bits_per_byte": 2.231583072032335,
|
|
"learning_rate": 0.0002962699392536863,
|
|
"loss_nats": 1.5468155145645142,
|
|
"step": 970,
|
|
"unclipped_grad_norm": 1.3417105674743652
|
|
},
|
|
{
|
|
"bits_per_byte": 2.3223450242160224,
|
|
"learning_rate": 0.0002961385448378161,
|
|
"loss_nats": 1.609726905822754,
|
|
"step": 980,
|
|
"unclipped_grad_norm": 1.444604754447937
|
|
},
|
|
{
|
|
"bits_per_byte": 2.2731906627729233,
|
|
"learning_rate": 0.00029600490962603817,
|
|
"loss_nats": 1.5756556987762451,
|
|
"step": 990,
|
|
"unclipped_grad_norm": 1.3634493350982666
|
|
},
|
|
{
|
|
"bits_per_byte": 2.2346108266007745,
|
|
"learning_rate": 0.00029586903590180956,
|
|
"loss_nats": 1.5489141941070557,
|
|
"step": 1000,
|
|
"unclipped_grad_norm": 1.291415810585022
|
|
},
|
|
{
|
|
"bits_per_byte": 2.2324534762283514,
|
|
"learning_rate": 0.00029573092598683735,
|
|
"loss_nats": 1.5474188327789307,
|
|
"step": 1010,
|
|
"unclipped_grad_norm": 1.811213493347168
|
|
},
|
|
{
|
|
"bits_per_byte": 2.198316295893291,
|
|
"learning_rate": 0.00029559058224103895,
|
|
"loss_nats": 1.523756742477417,
|
|
"step": 1020,
|
|
"unclipped_grad_norm": 1.5226773023605347
|
|
},
|
|
{
|
|
"bits_per_byte": 2.1998723949182977,
|
|
"learning_rate": 0.0002954480070625017,
|
|
"loss_nats": 1.5248353481292725,
|
|
"step": 1030,
|
|
"unclipped_grad_norm": 1.3728047609329224
|
|
},
|
|
{
|
|
"bits_per_byte": 2.256233517363152,
|
|
"learning_rate": 0.00029530320288744205,
|
|
"loss_nats": 1.5639019012451172,
|
|
"step": 1040,
|
|
"unclipped_grad_norm": 1.3665989637374878
|
|
},
|
|
{
|
|
"bits_per_byte": 2.2423672441546874,
|
|
"learning_rate": 0.0002951561721901638,
|
|
"loss_nats": 1.554290533065796,
|
|
"step": 1050,
|
|
"unclipped_grad_norm": 1.274046778678894
|
|
},
|
|
{
|
|
"bits_per_byte": 2.1489992388624755,
|
|
"learning_rate": 0.00029500691748301594,
|
|
"loss_nats": 1.4895727634429932,
|
|
"step": 1060,
|
|
"unclipped_grad_norm": 1.2373967170715332
|
|
},
|
|
{
|
|
"bits_per_byte": 2.2260787672914706,
|
|
"learning_rate": 0.00029485544131634963,
|
|
"loss_nats": 1.5430002212524414,
|
|
"step": 1070,
|
|
"unclipped_grad_norm": 1.3982683420181274
|
|
},
|
|
{
|
|
"bits_per_byte": 2.206438348562938,
|
|
"learning_rate": 0.0002947017462784745,
|
|
"loss_nats": 1.5293865203857422,
|
|
"step": 1080,
|
|
"unclipped_grad_norm": 1.334183692932129
|
|
},
|
|
{
|
|
"bits_per_byte": 2.1702291212502822,
|
|
"learning_rate": 0.00029454583499561476,
|
|
"loss_nats": 1.5042881965637207,
|
|
"step": 1090,
|
|
"unclipped_grad_norm": 1.4794831275939941
|
|
},
|
|
{
|
|
"bits_per_byte": 2.2378767771406594,
|
|
"learning_rate": 0.00029438771013186413,
|
|
"loss_nats": 1.551177978515625,
|
|
"step": 1100,
|
|
"unclipped_grad_norm": 1.3411470651626587
|
|
},
|
|
{
|
|
"bits_per_byte": 2.087035609583114,
|
|
"learning_rate": 0.0002942273743891401,
|
|
"loss_nats": 1.4466228485107422,
|
|
"step": 1110,
|
|
"unclipped_grad_norm": 1.452073574066162
|
|
},
|
|
{
|
|
"bits_per_byte": 2.1415479185311934,
|
|
"learning_rate": 0.00029406483050713824,
|
|
"loss_nats": 1.484407901763916,
|
|
"step": 1120,
|
|
"unclipped_grad_norm": 1.1817387342453003
|
|
},
|
|
{
|
|
"bits_per_byte": 2.079985696758946,
|
|
"learning_rate": 0.00029390008126328504,
|
|
"loss_nats": 1.4417362213134766,
|
|
"step": 1130,
|
|
"unclipped_grad_norm": 1.5416163206100464
|
|
},
|
|
{
|
|
"bits_per_byte": 2.2251825656978244,
|
|
"learning_rate": 0.00029373312947269045,
|
|
"loss_nats": 1.5423790216445923,
|
|
"step": 1140,
|
|
"unclipped_grad_norm": 1.5179954767227173
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0668831985029628,
|
|
"learning_rate": 0.00029356397798809995,
|
|
"loss_nats": 1.4326542615890503,
|
|
"step": 1150,
|
|
"unclipped_grad_norm": 1.186457633972168
|
|
},
|
|
{
|
|
"bits_per_byte": 2.104958265594829,
|
|
"learning_rate": 0.0002933926296998457,
|
|
"loss_nats": 1.4590458869934082,
|
|
"step": 1160,
|
|
"unclipped_grad_norm": 1.5302215814590454
|
|
},
|
|
{
|
|
"bits_per_byte": 2.146566716248672,
|
|
"learning_rate": 0.0002932190875357972,
|
|
"loss_nats": 1.487886667251587,
|
|
"step": 1170,
|
|
"unclipped_grad_norm": 1.187764048576355
|
|
},
|
|
{
|
|
"bits_per_byte": 2.1225221657968443,
|
|
"learning_rate": 0.00029304335446131126,
|
|
"loss_nats": 1.4712202548980713,
|
|
"step": 1180,
|
|
"unclipped_grad_norm": 1.1978200674057007
|
|
},
|
|
{
|
|
"bits_per_byte": 2.061979457178911,
|
|
"learning_rate": 0.00029286543347918116,
|
|
"loss_nats": 1.4292552471160889,
|
|
"step": 1190,
|
|
"unclipped_grad_norm": 1.333662509918213
|
|
},
|
|
{
|
|
"bits_per_byte": 2.04061714814994,
|
|
"learning_rate": 0.0002926853276295856,
|
|
"loss_nats": 1.4144480228424072,
|
|
"step": 1200,
|
|
"unclipped_grad_norm": 1.397094964981079
|
|
},
|
|
{
|
|
"bits_per_byte": 2.105093787923709,
|
|
"learning_rate": 0.0002925030399900368,
|
|
"loss_nats": 1.4591398239135742,
|
|
"step": 1210,
|
|
"unclipped_grad_norm": 1.355148196220398
|
|
},
|
|
{
|
|
"bits_per_byte": 2.113284805636352,
|
|
"learning_rate": 0.00029231857367532726,
|
|
"loss_nats": 1.4648174047470093,
|
|
"step": 1220,
|
|
"unclipped_grad_norm": 1.2704886198043823
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0731149898569767,
|
|
"learning_rate": 0.0002921319318374776,
|
|
"loss_nats": 1.4369738101959229,
|
|
"step": 1230,
|
|
"unclipped_grad_norm": 1.2257503271102905
|
|
},
|
|
{
|
|
"bits_per_byte": 2.02175856255223,
|
|
"learning_rate": 0.00029194311766568174,
|
|
"loss_nats": 1.4013762474060059,
|
|
"step": 1240,
|
|
"unclipped_grad_norm": 1.311407208442688
|
|
},
|
|
{
|
|
"bits_per_byte": 2.10808387829161,
|
|
"learning_rate": 0.000291752134386253,
|
|
"loss_nats": 1.461212396621704,
|
|
"step": 1250,
|
|
"unclipped_grad_norm": 1.183249831199646
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0709103442555663,
|
|
"learning_rate": 0.00029155898526256856,
|
|
"loss_nats": 1.4354456663131714,
|
|
"step": 1260,
|
|
"unclipped_grad_norm": 1.2188260555267334
|
|
},
|
|
{
|
|
"bits_per_byte": 2.073173119992968,
|
|
"learning_rate": 0.0002913636735950141,
|
|
"loss_nats": 1.437014102935791,
|
|
"step": 1270,
|
|
"unclipped_grad_norm": 1.2204123735427856
|
|
},
|
|
{
|
|
"bits_per_byte": 1.983587529141182,
|
|
"learning_rate": 0.0002911662027209272,
|
|
"loss_nats": 1.3749181032180786,
|
|
"step": 1280,
|
|
"unclipped_grad_norm": 1.4762879610061646
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0899118474361438,
|
|
"learning_rate": 0.00029096657601454013,
|
|
"loss_nats": 1.4486165046691895,
|
|
"step": 1290,
|
|
"unclipped_grad_norm": 1.5250427722930908
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0451232655851967,
|
|
"learning_rate": 0.00029076479688692266,
|
|
"loss_nats": 1.4175714254379272,
|
|
"step": 1300,
|
|
"unclipped_grad_norm": 1.2546813488006592
|
|
},
|
|
{
|
|
"bits_per_byte": 2.037817958524497,
|
|
"learning_rate": 0.00029056086878592323,
|
|
"loss_nats": 1.4125077724456787,
|
|
"step": 1310,
|
|
"unclipped_grad_norm": 1.2255067825317383
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0185500542177333,
|
|
"learning_rate": 0.00029035479519611045,
|
|
"loss_nats": 1.3991522789001465,
|
|
"step": 1320,
|
|
"unclipped_grad_norm": 1.1987844705581665
|
|
},
|
|
{
|
|
"bits_per_byte": 2.097532054730573,
|
|
"learning_rate": 0.0002901465796387134,
|
|
"loss_nats": 1.4538984298706055,
|
|
"step": 1330,
|
|
"unclipped_grad_norm": 1.270599603652954
|
|
},
|
|
{
|
|
"bits_per_byte": 2.036339251692379,
|
|
"learning_rate": 0.00028993622567156154,
|
|
"loss_nats": 1.411482810974121,
|
|
"step": 1340,
|
|
"unclipped_grad_norm": 1.31504225730896
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0528777913299496,
|
|
"learning_rate": 0.0002897237368890237,
|
|
"loss_nats": 1.4229464530944824,
|
|
"step": 1350,
|
|
"unclipped_grad_norm": 1.3823131322860718
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0989251142025616,
|
|
"learning_rate": 0.0002895091169219469,
|
|
"loss_nats": 1.4548640251159668,
|
|
"step": 1360,
|
|
"unclipped_grad_norm": 1.3448033332824707
|
|
},
|
|
{
|
|
"bits_per_byte": 2.081831758533307,
|
|
"learning_rate": 0.00028929236943759425,
|
|
"loss_nats": 1.4430158138275146,
|
|
"step": 1370,
|
|
"unclipped_grad_norm": 1.1824978590011597
|
|
},
|
|
{
|
|
"bits_per_byte": 2.037543130248418,
|
|
"learning_rate": 0.00028907349813958225,
|
|
"loss_nats": 1.4123172760009766,
|
|
"step": 1380,
|
|
"unclipped_grad_norm": 1.241772174835205
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0465340392702243,
|
|
"learning_rate": 0.0002888525067678175,
|
|
"loss_nats": 1.4185492992401123,
|
|
"step": 1390,
|
|
"unclipped_grad_norm": 1.1579062938690186
|
|
},
|
|
{
|
|
"bits_per_byte": 1.983804227281269,
|
|
"learning_rate": 0.00028862939909843273,
|
|
"loss_nats": 1.3750683069229126,
|
|
"step": 1400,
|
|
"unclipped_grad_norm": 1.2093477249145508
|
|
},
|
|
{
|
|
"bits_per_byte": 2.004750510277803,
|
|
"learning_rate": 0.0002884041789437225,
|
|
"loss_nats": 1.389587163925171,
|
|
"step": 1410,
|
|
"unclipped_grad_norm": 1.0995537042617798
|
|
},
|
|
{
|
|
"bits_per_byte": 2.044002282666873,
|
|
"learning_rate": 0.0002881768501520778,
|
|
"loss_nats": 1.4167944192886353,
|
|
"step": 1420,
|
|
"unclipped_grad_norm": 1.377259612083435
|
|
},
|
|
{
|
|
"bits_per_byte": 2.072302027866348,
|
|
"learning_rate": 0.0002879474166079205,
|
|
"loss_nats": 1.4364103078842163,
|
|
"step": 1430,
|
|
"unclipped_grad_norm": 1.2491836547851562
|
|
},
|
|
{
|
|
"bits_per_byte": 2.010299358525241,
|
|
"learning_rate": 0.00028771588223163694,
|
|
"loss_nats": 1.3934333324432373,
|
|
"step": 1440,
|
|
"unclipped_grad_norm": 1.1953747272491455
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9653831655473537,
|
|
"learning_rate": 0.00028748225097951054,
|
|
"loss_nats": 1.3622997999191284,
|
|
"step": 1450,
|
|
"unclipped_grad_norm": 1.1474745273590088
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0178239434657903,
|
|
"learning_rate": 0.000287246526843655,
|
|
"loss_nats": 1.398648977279663,
|
|
"step": 1460,
|
|
"unclipped_grad_norm": 1.309095859527588
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9352492253764266,
|
|
"learning_rate": 0.00028700871385194533,
|
|
"loss_nats": 1.3414125442504883,
|
|
"step": 1470,
|
|
"unclipped_grad_norm": 1.2327831983566284
|
|
},
|
|
{
|
|
"bits_per_byte": 2.02495382822261,
|
|
"learning_rate": 0.00028676881606794945,
|
|
"loss_nats": 1.4035910367965698,
|
|
"step": 1480,
|
|
"unclipped_grad_norm": 1.2556331157684326
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0395678819970264,
|
|
"learning_rate": 0.00028652683759085854,
|
|
"loss_nats": 1.413720726966858,
|
|
"step": 1490,
|
|
"unclipped_grad_norm": 1.1320099830627441
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0169458000504847,
|
|
"learning_rate": 0.0002862827825554171,
|
|
"loss_nats": 1.3980402946472168,
|
|
"step": 1500,
|
|
"unclipped_grad_norm": 1.2224371433258057
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0172811662196675,
|
|
"learning_rate": 0.00028603665513185245,
|
|
"loss_nats": 1.3982727527618408,
|
|
"step": 1510,
|
|
"unclipped_grad_norm": 1.1516478061676025
|
|
},
|
|
{
|
|
"bits_per_byte": 1.959494651564462,
|
|
"learning_rate": 0.000285788459525803,
|
|
"loss_nats": 1.3582181930541992,
|
|
"step": 1520,
|
|
"unclipped_grad_norm": 0.9697362780570984
|
|
},
|
|
{
|
|
"bits_per_byte": 2.05711750763902,
|
|
"learning_rate": 0.00028553819997824707,
|
|
"loss_nats": 1.4258852005004883,
|
|
"step": 1530,
|
|
"unclipped_grad_norm": 1.1719471216201782
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0050322178599163,
|
|
"learning_rate": 0.00028528588076542964,
|
|
"loss_nats": 1.389782428741455,
|
|
"step": 1540,
|
|
"unclipped_grad_norm": 1.2557746171951294
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0500259750133427,
|
|
"learning_rate": 0.00028503150619878996,
|
|
"loss_nats": 1.4209697246551514,
|
|
"step": 1550,
|
|
"unclipped_grad_norm": 1.1871017217636108
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9591589414299777,
|
|
"learning_rate": 0.0002847750806248876,
|
|
"loss_nats": 1.357985496520996,
|
|
"step": 1560,
|
|
"unclipped_grad_norm": 1.1881691217422485
|
|
},
|
|
{
|
|
"bits_per_byte": 2.048241998975943,
|
|
"learning_rate": 0.000284516608425328,
|
|
"loss_nats": 1.4197331666946411,
|
|
"step": 1570,
|
|
"unclipped_grad_norm": 1.3408379554748535
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0003424229535387,
|
|
"learning_rate": 0.000284256094016688,
|
|
"loss_nats": 1.3865317106246948,
|
|
"step": 1580,
|
|
"unclipped_grad_norm": 1.160094141960144
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9972685770346676,
|
|
"learning_rate": 0.0002839935418504399,
|
|
"loss_nats": 1.3844010829925537,
|
|
"step": 1590,
|
|
"unclipped_grad_norm": 1.1695513725280762
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0280931995314595,
|
|
"learning_rate": 0.000283728956412876,
|
|
"loss_nats": 1.4057670831680298,
|
|
"step": 1600,
|
|
"unclipped_grad_norm": 1.1151546239852905
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9648931869750457,
|
|
"learning_rate": 0.00028346234222503136,
|
|
"loss_nats": 1.3619601726531982,
|
|
"step": 1610,
|
|
"unclipped_grad_norm": 1.1052403450012207
|
|
},
|
|
{
|
|
"bits_per_byte": 1.934925382044852,
|
|
"learning_rate": 0.00028319370384260675,
|
|
"loss_nats": 1.3411880731582642,
|
|
"step": 1620,
|
|
"unclipped_grad_norm": 1.0654687881469727
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9529704897039815,
|
|
"learning_rate": 0.00028292304585589105,
|
|
"loss_nats": 1.3536959886550903,
|
|
"step": 1630,
|
|
"unclipped_grad_norm": 1.1012754440307617
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9931929321745208,
|
|
"learning_rate": 0.0002826503728896824,
|
|
"loss_nats": 1.3815760612487793,
|
|
"step": 1640,
|
|
"unclipped_grad_norm": 1.143923282623291
|
|
},
|
|
{
|
|
"bits_per_byte": 2.059868542122221,
|
|
"learning_rate": 0.0002823756896032095,
|
|
"loss_nats": 1.4277920722961426,
|
|
"step": 1650,
|
|
"unclipped_grad_norm": 1.2584717273712158
|
|
},
|
|
{
|
|
"bits_per_byte": 1.992070745377641,
|
|
"learning_rate": 0.0002820990006900518,
|
|
"loss_nats": 1.3807982206344604,
|
|
"step": 1660,
|
|
"unclipped_grad_norm": 1.177032232284546
|
|
},
|
|
{
|
|
"bits_per_byte": 2.009216383772758,
|
|
"learning_rate": 0.00028182031087805947,
|
|
"loss_nats": 1.392682671546936,
|
|
"step": 1670,
|
|
"unclipped_grad_norm": 1.1299933195114136
|
|
},
|
|
{
|
|
"bits_per_byte": 2.027195106128654,
|
|
"learning_rate": 0.0002815396249292725,
|
|
"loss_nats": 1.4051445722579956,
|
|
"step": 1680,
|
|
"unclipped_grad_norm": 1.2568069696426392
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0647518175108197,
|
|
"learning_rate": 0.0002812569476398391,
|
|
"loss_nats": 1.4311769008636475,
|
|
"step": 1690,
|
|
"unclipped_grad_norm": 1.1674096584320068
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9602546428986247,
|
|
"learning_rate": 0.00028097228383993446,
|
|
"loss_nats": 1.3587449789047241,
|
|
"step": 1700,
|
|
"unclipped_grad_norm": 1.178646206855774
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8971180919403792,
|
|
"learning_rate": 0.0002806856383936774,
|
|
"loss_nats": 1.3149820566177368,
|
|
"step": 1710,
|
|
"unclipped_grad_norm": 1.0156482458114624
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8941140709777584,
|
|
"learning_rate": 0.00028039701619904774,
|
|
"loss_nats": 1.3128998279571533,
|
|
"step": 1720,
|
|
"unclipped_grad_norm": 1.0928010940551758
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8690832000232327,
|
|
"learning_rate": 0.0002801064221878024,
|
|
"loss_nats": 1.295549750328064,
|
|
"step": 1730,
|
|
"unclipped_grad_norm": 1.0898301601409912
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0020670649763908,
|
|
"learning_rate": 0.00027981386132539134,
|
|
"loss_nats": 1.38772714138031,
|
|
"step": 1740,
|
|
"unclipped_grad_norm": 1.1916359663009644
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9965376507685006,
|
|
"learning_rate": 0.0002795193386108724,
|
|
"loss_nats": 1.383894443511963,
|
|
"step": 1750,
|
|
"unclipped_grad_norm": 1.1260547637939453
|
|
},
|
|
{
|
|
"bits_per_byte": 2.010018682839033,
|
|
"learning_rate": 0.00027922285907682625,
|
|
"loss_nats": 1.3932387828826904,
|
|
"step": 1760,
|
|
"unclipped_grad_norm": 1.2356810569763184
|
|
},
|
|
{
|
|
"bits_per_byte": 2.000046096846102,
|
|
"learning_rate": 0.00027892442778927005,
|
|
"loss_nats": 1.3863263130187988,
|
|
"step": 1770,
|
|
"unclipped_grad_norm": 1.1323615312576294
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8857641412957182,
|
|
"learning_rate": 0.00027862404984757116,
|
|
"loss_nats": 1.3071120977401733,
|
|
"step": 1780,
|
|
"unclipped_grad_norm": 1.1074084043502808
|
|
},
|
|
{
|
|
"bits_per_byte": 1.98148022572016,
|
|
"learning_rate": 0.0002783217303843597,
|
|
"loss_nats": 1.373457431793213,
|
|
"step": 1790,
|
|
"unclipped_grad_norm": 1.1702499389648438
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8942642118319617,
|
|
"learning_rate": 0.00027801747456544134,
|
|
"loss_nats": 1.3130038976669312,
|
|
"step": 1800,
|
|
"unclipped_grad_norm": 1.2193527221679688
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9583697130451683,
|
|
"learning_rate": 0.00027771128758970843,
|
|
"loss_nats": 1.3574384450912476,
|
|
"step": 1810,
|
|
"unclipped_grad_norm": 1.4279322624206543
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9039834673801714,
|
|
"learning_rate": 0.00027740317468905165,
|
|
"loss_nats": 1.3197407722473145,
|
|
"step": 1820,
|
|
"unclipped_grad_norm": 1.2399523258209229
|
|
},
|
|
{
|
|
"bits_per_byte": 1.902950023631137,
|
|
"learning_rate": 0.0002770931411282704,
|
|
"loss_nats": 1.3190244436264038,
|
|
"step": 1830,
|
|
"unclipped_grad_norm": 1.066636562347412
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9139311158887273,
|
|
"learning_rate": 0.0002767811922049827,
|
|
"loss_nats": 1.3266359567642212,
|
|
"step": 1840,
|
|
"unclipped_grad_norm": 1.129875898361206
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9166544611651397,
|
|
"learning_rate": 0.00027646733324953503,
|
|
"loss_nats": 1.3285236358642578,
|
|
"step": 1850,
|
|
"unclipped_grad_norm": 1.1898164749145508
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9000117000411458,
|
|
"learning_rate": 0.00027615156962491097,
|
|
"loss_nats": 1.3169877529144287,
|
|
"step": 1860,
|
|
"unclipped_grad_norm": 1.1322225332260132
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9091849106728662,
|
|
"learning_rate": 0.0002758339067266396,
|
|
"loss_nats": 1.3233461380004883,
|
|
"step": 1870,
|
|
"unclipped_grad_norm": 1.1632918119430542
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9066440389890191,
|
|
"learning_rate": 0.0002755143499827035,
|
|
"loss_nats": 1.321584939956665,
|
|
"step": 1880,
|
|
"unclipped_grad_norm": 1.0685805082321167
|
|
},
|
|
{
|
|
"bits_per_byte": 1.915992327959318,
|
|
"learning_rate": 0.0002751929048534456,
|
|
"loss_nats": 1.3280646800994873,
|
|
"step": 1890,
|
|
"unclipped_grad_norm": 1.1415059566497803
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9305143710155228,
|
|
"learning_rate": 0.0002748695768314763,
|
|
"loss_nats": 1.3381305932998657,
|
|
"step": 1900,
|
|
"unclipped_grad_norm": 1.0947742462158203
|
|
},
|
|
{
|
|
"bits_per_byte": 1.863847876148318,
|
|
"learning_rate": 0.00027454437144157947,
|
|
"loss_nats": 1.2919209003448486,
|
|
"step": 1910,
|
|
"unclipped_grad_norm": 1.0498101711273193
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8709641742757193,
|
|
"learning_rate": 0.00027421729424061787,
|
|
"loss_nats": 1.2968535423278809,
|
|
"step": 1920,
|
|
"unclipped_grad_norm": 1.227995753288269
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9694514151535134,
|
|
"learning_rate": 0.0002738883508174385,
|
|
"loss_nats": 1.3651196956634521,
|
|
"step": 1930,
|
|
"unclipped_grad_norm": 1.1106438636779785
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8401163341210607,
|
|
"learning_rate": 0.0002735575467927768,
|
|
"loss_nats": 1.2754714488983154,
|
|
"step": 1940,
|
|
"unclipped_grad_norm": 1.1273587942123413
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9228790292478175,
|
|
"learning_rate": 0.0002732248878191609,
|
|
"loss_nats": 1.3328381776809692,
|
|
"step": 1950,
|
|
"unclipped_grad_norm": 1.0416061878204346
|
|
},
|
|
{
|
|
"bits_per_byte": 1.840930843955547,
|
|
"learning_rate": 0.0002728903795808148,
|
|
"loss_nats": 1.276036024093628,
|
|
"step": 1960,
|
|
"unclipped_grad_norm": 1.0370304584503174
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9309775202942963,
|
|
"learning_rate": 0.0002725540277935614,
|
|
"loss_nats": 1.338451623916626,
|
|
"step": 1970,
|
|
"unclipped_grad_norm": 1.147440791130066
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9639140897436835,
|
|
"learning_rate": 0.00027221583820472456,
|
|
"loss_nats": 1.3612815141677856,
|
|
"step": 1980,
|
|
"unclipped_grad_norm": 1.2305821180343628
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9535571225260744,
|
|
"learning_rate": 0.0002718758165930314,
|
|
"loss_nats": 1.354102611541748,
|
|
"step": 1990,
|
|
"unclipped_grad_norm": 1.3296585083007812
|
|
},
|
|
{
|
|
"bits_per_byte": 1.92549075778382,
|
|
"learning_rate": 0.0002715339687685131,
|
|
"loss_nats": 1.3346484899520874,
|
|
"step": 2000,
|
|
"unclipped_grad_norm": 1.0916969776153564
|
|
},
|
|
{
|
|
"bits_per_byte": 1.880516778635243,
|
|
"learning_rate": 0.00027119030057240576,
|
|
"loss_nats": 1.3034749031066895,
|
|
"step": 2010,
|
|
"unclipped_grad_norm": 1.0219805240631104
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9332846675556208,
|
|
"learning_rate": 0.0002708448178770506,
|
|
"loss_nats": 1.3400508165359497,
|
|
"step": 2020,
|
|
"unclipped_grad_norm": 1.1069045066833496
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9929170720025369,
|
|
"learning_rate": 0.00027049752658579376,
|
|
"loss_nats": 1.3813848495483398,
|
|
"step": 2030,
|
|
"unclipped_grad_norm": 1.0479846000671387
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9184877962233373,
|
|
"learning_rate": 0.00027014843263288514,
|
|
"loss_nats": 1.3297944068908691,
|
|
"step": 2040,
|
|
"unclipped_grad_norm": 1.0204336643218994
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9692321372736634,
|
|
"learning_rate": 0.00026979754198337724,
|
|
"loss_nats": 1.364967703819275,
|
|
"step": 2050,
|
|
"unclipped_grad_norm": 1.0853239297866821
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9303957029864274,
|
|
"learning_rate": 0.00026944486063302305,
|
|
"loss_nats": 1.3380483388900757,
|
|
"step": 2060,
|
|
"unclipped_grad_norm": 1.0730464458465576
|
|
},
|
|
{
|
|
"bits_per_byte": 1.889977716241857,
|
|
"learning_rate": 0.0002690903946081738,
|
|
"loss_nats": 1.3100327253341675,
|
|
"step": 2070,
|
|
"unclipped_grad_norm": 1.0691337585449219
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8844801188243765,
|
|
"learning_rate": 0.00026873414996567596,
|
|
"loss_nats": 1.3062220811843872,
|
|
"step": 2080,
|
|
"unclipped_grad_norm": 1.0554076433181763
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8957609768424206,
|
|
"learning_rate": 0.00026837613279276736,
|
|
"loss_nats": 1.3140413761138916,
|
|
"step": 2090,
|
|
"unclipped_grad_norm": 1.1196725368499756
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8758003264179814,
|
|
"learning_rate": 0.0002680163492069739,
|
|
"loss_nats": 1.3002057075500488,
|
|
"step": 2100,
|
|
"unclipped_grad_norm": 1.0327236652374268
|
|
},
|
|
{
|
|
"bits_per_byte": 1.914194593309849,
|
|
"learning_rate": 0.00026765480535600414,
|
|
"loss_nats": 1.326818585395813,
|
|
"step": 2110,
|
|
"unclipped_grad_norm": 1.0782309770584106
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8761728408397502,
|
|
"learning_rate": 0.0002672915074176452,
|
|
"loss_nats": 1.3004639148712158,
|
|
"step": 2120,
|
|
"unclipped_grad_norm": 1.0318117141723633
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8463771905430704,
|
|
"learning_rate": 0.00026692646159965624,
|
|
"loss_nats": 1.279811143875122,
|
|
"step": 2130,
|
|
"unclipped_grad_norm": 1.1898446083068848
|
|
},
|
|
{
|
|
"bits_per_byte": 1.984834747325239,
|
|
"learning_rate": 0.0002665596741396633,
|
|
"loss_nats": 1.3757826089859009,
|
|
"step": 2140,
|
|
"unclipped_grad_norm": 1.1598726511001587
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8562880347643413,
|
|
"learning_rate": 0.000266191151305052,
|
|
"loss_nats": 1.286680817604065,
|
|
"step": 2150,
|
|
"unclipped_grad_norm": 1.0410785675048828
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8990754264898486,
|
|
"learning_rate": 0.00026582089939286075,
|
|
"loss_nats": 1.3163387775421143,
|
|
"step": 2160,
|
|
"unclipped_grad_norm": 1.0785906314849854
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9583294691048665,
|
|
"learning_rate": 0.0002654489247296732,
|
|
"loss_nats": 1.3574105501174927,
|
|
"step": 2170,
|
|
"unclipped_grad_norm": 1.1421548128128052
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8472800994601,
|
|
"learning_rate": 0.00026507523367151007,
|
|
"loss_nats": 1.2804369926452637,
|
|
"step": 2180,
|
|
"unclipped_grad_norm": 1.0929888486862183
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8586782496460328,
|
|
"learning_rate": 0.00026469983260372037,
|
|
"loss_nats": 1.2883375883102417,
|
|
"step": 2190,
|
|
"unclipped_grad_norm": 1.058180570602417
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8910505440179393,
|
|
"learning_rate": 0.0002643227279408727,
|
|
"loss_nats": 1.3107763528823853,
|
|
"step": 2200,
|
|
"unclipped_grad_norm": 0.9802089333534241
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8399546704292495,
|
|
"learning_rate": 0.0002639439261266453,
|
|
"loss_nats": 1.2753593921661377,
|
|
"step": 2210,
|
|
"unclipped_grad_norm": 1.060943365097046
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8476158095945843,
|
|
"learning_rate": 0.000263563433633716,
|
|
"loss_nats": 1.2806696891784668,
|
|
"step": 2220,
|
|
"unclipped_grad_norm": 1.0172474384307861
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9309195621409554,
|
|
"learning_rate": 0.00026318125696365156,
|
|
"loss_nats": 1.3384114503860474,
|
|
"step": 2230,
|
|
"unclipped_grad_norm": 1.1571786403656006
|
|
},
|
|
{
|
|
"bits_per_byte": 1.848552427111183,
|
|
"learning_rate": 0.000262797402646797,
|
|
"loss_nats": 1.2813189029693604,
|
|
"step": 2240,
|
|
"unclipped_grad_norm": 1.085301160812378
|
|
},
|
|
{
|
|
"bits_per_byte": 1.864913308670413,
|
|
"learning_rate": 0.00026241187724216334,
|
|
"loss_nats": 1.2926594018936157,
|
|
"step": 2250,
|
|
"unclipped_grad_norm": 1.0694295167922974
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8611869605741684,
|
|
"learning_rate": 0.0002620246873373162,
|
|
"loss_nats": 1.290076494216919,
|
|
"step": 2260,
|
|
"unclipped_grad_norm": 1.1977293491363525
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8337584794839643,
|
|
"learning_rate": 0.00026163583954826264,
|
|
"loss_nats": 1.2710645198822021,
|
|
"step": 2270,
|
|
"unclipped_grad_norm": 1.1360434293746948
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8587281246147829,
|
|
"learning_rate": 0.0002612453405193386,
|
|
"loss_nats": 1.2883721590042114,
|
|
"step": 2280,
|
|
"unclipped_grad_norm": 1.113946795463562
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8907026231152437,
|
|
"learning_rate": 0.000260853196923095,
|
|
"loss_nats": 1.310535192489624,
|
|
"step": 2290,
|
|
"unclipped_grad_norm": 1.1474530696868896
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8085284526197394,
|
|
"learning_rate": 0.0002604594154601839,
|
|
"loss_nats": 1.253576397895813,
|
|
"step": 2300,
|
|
"unclipped_grad_norm": 0.96898353099823
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8446171200941404,
|
|
"learning_rate": 0.0002600640028592441,
|
|
"loss_nats": 1.2785911560058594,
|
|
"step": 2310,
|
|
"unclipped_grad_norm": 0.9442379474639893
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8195852172437095,
|
|
"learning_rate": 0.0002596669658767859,
|
|
"loss_nats": 1.2612403631210327,
|
|
"step": 2320,
|
|
"unclipped_grad_norm": 0.9633188843727112
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8903022475040352,
|
|
"learning_rate": 0.000259268311297076,
|
|
"loss_nats": 1.3102576732635498,
|
|
"step": 2330,
|
|
"unclipped_grad_norm": 1.0167381763458252
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8286817236131452,
|
|
"learning_rate": 0.00025886804593202114,
|
|
"loss_nats": 1.2675455808639526,
|
|
"step": 2340,
|
|
"unclipped_grad_norm": 1.1636918783187866
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8687012265556664,
|
|
"learning_rate": 0.00025846617662105193,
|
|
"loss_nats": 1.2952849864959717,
|
|
"step": 2350,
|
|
"unclipped_grad_norm": 1.0884090662002563
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8708572010668825,
|
|
"learning_rate": 0.00025806271023100637,
|
|
"loss_nats": 1.2967793941497803,
|
|
"step": 2360,
|
|
"unclipped_grad_norm": 1.0467369556427002
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8979811008824086,
|
|
"learning_rate": 0.00025765765365601165,
|
|
"loss_nats": 1.3155802488327026,
|
|
"step": 2370,
|
|
"unclipped_grad_norm": 1.0231177806854248
|
|
},
|
|
{
|
|
"bits_per_byte": 1.853912954355926,
|
|
"learning_rate": 0.00025725101381736707,
|
|
"loss_nats": 1.2850345373153687,
|
|
"step": 2380,
|
|
"unclipped_grad_norm": 1.0531508922576904
|
|
},
|
|
{
|
|
"bits_per_byte": 1.909142430958103,
|
|
"learning_rate": 0.00025684279766342567,
|
|
"loss_nats": 1.3233166933059692,
|
|
"step": 2390,
|
|
"unclipped_grad_norm": 1.0615708827972412
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8518789155441715,
|
|
"learning_rate": 0.00025643301216947504,
|
|
"loss_nats": 1.2836246490478516,
|
|
"step": 2400,
|
|
"unclipped_grad_norm": 1.0747002363204956
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8430525939192415,
|
|
"learning_rate": 0.00025602166433761866,
|
|
"loss_nats": 1.277506709098816,
|
|
"step": 2410,
|
|
"unclipped_grad_norm": 1.035906434059143
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8218237434273397,
|
|
"learning_rate": 0.0002556087611966561,
|
|
"loss_nats": 1.2627919912338257,
|
|
"step": 2420,
|
|
"unclipped_grad_norm": 0.9908418655395508
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8454151195941437,
|
|
"learning_rate": 0.00025519430980196277,
|
|
"loss_nats": 1.279144287109375,
|
|
"step": 2430,
|
|
"unclipped_grad_norm": 1.1172900199890137
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9008382486611926,
|
|
"learning_rate": 0.0002547783172353695,
|
|
"loss_nats": 1.3175606727600098,
|
|
"step": 2440,
|
|
"unclipped_grad_norm": 1.0292961597442627
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8510478953752025,
|
|
"learning_rate": 0.0002543607906050415,
|
|
"loss_nats": 1.2830486297607422,
|
|
"step": 2450,
|
|
"unclipped_grad_norm": 1.0116122961044312
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8498311181203482,
|
|
"learning_rate": 0.0002539417370453568,
|
|
"loss_nats": 1.2822052240371704,
|
|
"step": 2460,
|
|
"unclipped_grad_norm": 0.9883264303207397
|
|
},
|
|
{
|
|
"bits_per_byte": 1.849598081628429,
|
|
"learning_rate": 0.00025352116371678464,
|
|
"loss_nats": 1.282043695449829,
|
|
"step": 2470,
|
|
"unclipped_grad_norm": 1.0902615785598755
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8701792454571813,
|
|
"learning_rate": 0.0002530990778057627,
|
|
"loss_nats": 1.296309471130371,
|
|
"step": 2480,
|
|
"unclipped_grad_norm": 1.4108390808105469
|
|
},
|
|
{
|
|
"bits_per_byte": 1.81769770765049,
|
|
"learning_rate": 0.00025267548652457475,
|
|
"loss_nats": 1.259932041168213,
|
|
"step": 2490,
|
|
"unclipped_grad_norm": 1.0036945343017578
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8526576529872785,
|
|
"learning_rate": 0.00025225039711122696,
|
|
"loss_nats": 1.2841644287109375,
|
|
"step": 2500,
|
|
"unclipped_grad_norm": 1.0499781370162964
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8174889207123426,
|
|
"learning_rate": 0.00025182381682932464,
|
|
"loss_nats": 1.2597873210906982,
|
|
"step": 2510,
|
|
"unclipped_grad_norm": 1.112528681755066
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8559390819657409,
|
|
"learning_rate": 0.0002513957529679477,
|
|
"loss_nats": 1.2864389419555664,
|
|
"step": 2520,
|
|
"unclipped_grad_norm": 1.043527603149414
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8664243482408935,
|
|
"learning_rate": 0.00025096621284152656,
|
|
"loss_nats": 1.2937067747116089,
|
|
"step": 2530,
|
|
"unclipped_grad_norm": 1.047589898109436
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8385335777851701,
|
|
"learning_rate": 0.0002505352037897168,
|
|
"loss_nats": 1.2743743658065796,
|
|
"step": 2540,
|
|
"unclipped_grad_norm": 1.114875078201294
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8303919190933249,
|
|
"learning_rate": 0.0002501027331772739,
|
|
"loss_nats": 1.2687309980392456,
|
|
"step": 2550,
|
|
"unclipped_grad_norm": 1.026410460472107
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8414952910156785,
|
|
"learning_rate": 0.00024966880839392746,
|
|
"loss_nats": 1.2764272689819336,
|
|
"step": 2560,
|
|
"unclipped_grad_norm": 1.0519986152648926
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8285018297603426,
|
|
"learning_rate": 0.0002492334368542548,
|
|
"loss_nats": 1.2674208879470825,
|
|
"step": 2570,
|
|
"unclipped_grad_norm": 0.9845532178878784
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8621820521920607,
|
|
"learning_rate": 0.00024879662599755416,
|
|
"loss_nats": 1.2907662391662598,
|
|
"step": 2580,
|
|
"unclipped_grad_norm": 1.1312247514724731
|
|
},
|
|
{
|
|
"bits_per_byte": 1.821957545929711,
|
|
"learning_rate": 0.00024835838328771803,
|
|
"loss_nats": 1.2628847360610962,
|
|
"step": 2590,
|
|
"unclipped_grad_norm": 0.9848816990852356
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8336463467956017,
|
|
"learning_rate": 0.0002479187162131051,
|
|
"loss_nats": 1.270986795425415,
|
|
"step": 2600,
|
|
"unclipped_grad_norm": 1.0540134906768799
|
|
},
|
|
{
|
|
"bits_per_byte": 1.762311382870726,
|
|
"learning_rate": 0.00024747763228641264,
|
|
"loss_nats": 1.221541166305542,
|
|
"step": 2610,
|
|
"unclipped_grad_norm": 0.9237543940544128
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8236500271968519,
|
|
"learning_rate": 0.00024703513904454795,
|
|
"loss_nats": 1.2640578746795654,
|
|
"step": 2620,
|
|
"unclipped_grad_norm": 0.9258099794387817
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8442078013850869,
|
|
"learning_rate": 0.00024659124404849956,
|
|
"loss_nats": 1.2783074378967285,
|
|
"step": 2630,
|
|
"unclipped_grad_norm": 1.1065376996994019
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6990233151970702,
|
|
"learning_rate": 0.00024614595488320833,
|
|
"loss_nats": 1.1776732206344604,
|
|
"step": 2640,
|
|
"unclipped_grad_norm": 0.975441038608551
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7897238695744,
|
|
"learning_rate": 0.00024569927915743745,
|
|
"loss_nats": 1.2405420541763306,
|
|
"step": 2650,
|
|
"unclipped_grad_norm": 1.024839997291565
|
|
},
|
|
{
|
|
"bits_per_byte": 1.829891621561965,
|
|
"learning_rate": 0.0002452512245036427,
|
|
"loss_nats": 1.2683842182159424,
|
|
"step": 2660,
|
|
"unclipped_grad_norm": 1.0799146890640259
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8374299651145836,
|
|
"learning_rate": 0.00024480179857784177,
|
|
"loss_nats": 1.2736093997955322,
|
|
"step": 2670,
|
|
"unclipped_grad_norm": 0.9827171564102173
|
|
},
|
|
{
|
|
"bits_per_byte": 1.757754702536116,
|
|
"learning_rate": 0.00024435100905948387,
|
|
"loss_nats": 1.218382716178894,
|
|
"step": 2680,
|
|
"unclipped_grad_norm": 0.9623468518257141
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9223347041578367,
|
|
"learning_rate": 0.00024389886365131793,
|
|
"loss_nats": 1.332460880279541,
|
|
"step": 2690,
|
|
"unclipped_grad_norm": 1.0313293933868408
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8426768118271062,
|
|
"learning_rate": 0.00024344537007926163,
|
|
"loss_nats": 1.2772462368011475,
|
|
"step": 2700,
|
|
"unclipped_grad_norm": 1.154735803604126
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7467827253590213,
|
|
"learning_rate": 0.00024299053609226878,
|
|
"loss_nats": 1.2107775211334229,
|
|
"step": 2710,
|
|
"unclipped_grad_norm": 0.9442266225814819
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7668878412101854,
|
|
"learning_rate": 0.00024253436946219733,
|
|
"loss_nats": 1.2247133255004883,
|
|
"step": 2720,
|
|
"unclipped_grad_norm": 0.9805414080619812
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8017586995338266,
|
|
"learning_rate": 0.0002420768779836764,
|
|
"loss_nats": 1.2488839626312256,
|
|
"step": 2730,
|
|
"unclipped_grad_norm": 1.027875542640686
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7992866209103247,
|
|
"learning_rate": 0.00024161806947397305,
|
|
"loss_nats": 1.2471704483032227,
|
|
"step": 2740,
|
|
"unclipped_grad_norm": 1.0912469625473022
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8291928560515094,
|
|
"learning_rate": 0.00024115795177285886,
|
|
"loss_nats": 1.2678998708724976,
|
|
"step": 2750,
|
|
"unclipped_grad_norm": 1.0327590703964233
|
|
},
|
|
{
|
|
"bits_per_byte": 1.831356913747316,
|
|
"learning_rate": 0.0002406965327424758,
|
|
"loss_nats": 1.269399881362915,
|
|
"step": 2760,
|
|
"unclipped_grad_norm": 1.0535632371902466
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7175185014535075,
|
|
"learning_rate": 0.00024023382026720197,
|
|
"loss_nats": 1.190493106842041,
|
|
"step": 2770,
|
|
"unclipped_grad_norm": 0.9521624445915222
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7419930085324922,
|
|
"learning_rate": 0.00023976982225351698,
|
|
"loss_nats": 1.2074575424194336,
|
|
"step": 2780,
|
|
"unclipped_grad_norm": 0.9253908395767212
|
|
},
|
|
{
|
|
"bits_per_byte": 1.829489182158946,
|
|
"learning_rate": 0.0002393045466298666,
|
|
"loss_nats": 1.2681052684783936,
|
|
"step": 2790,
|
|
"unclipped_grad_norm": 1.0019073486328125
|
|
},
|
|
{
|
|
"bits_per_byte": 1.900268298156233,
|
|
"learning_rate": 0.00023883800134652763,
|
|
"loss_nats": 1.3171656131744385,
|
|
"step": 2800,
|
|
"unclipped_grad_norm": 1.1026051044464111
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8251356133350047,
|
|
"learning_rate": 0.0002383701943754715,
|
|
"loss_nats": 1.265087604522705,
|
|
"step": 2810,
|
|
"unclipped_grad_norm": 1.095402479171753
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8078448215825598,
|
|
"learning_rate": 0.00023790113371022884,
|
|
"loss_nats": 1.2531025409698486,
|
|
"step": 2820,
|
|
"unclipped_grad_norm": 1.0277965068817139
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8903005276775264,
|
|
"learning_rate": 0.0002374308273657521,
|
|
"loss_nats": 1.3102564811706543,
|
|
"step": 2830,
|
|
"unclipped_grad_norm": 1.0477416515350342
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7718302786306808,
|
|
"learning_rate": 0.00023695928337827911,
|
|
"loss_nats": 1.2281391620635986,
|
|
"step": 2840,
|
|
"unclipped_grad_norm": 0.9842959642410278
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7997145137456714,
|
|
"learning_rate": 0.00023648650980519564,
|
|
"loss_nats": 1.247467041015625,
|
|
"step": 2850,
|
|
"unclipped_grad_norm": 1.0457379817962646
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7627234533021934,
|
|
"learning_rate": 0.00023601251472489757,
|
|
"loss_nats": 1.2218267917633057,
|
|
"step": 2860,
|
|
"unclipped_grad_norm": 1.0218563079833984
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8575106594293251,
|
|
"learning_rate": 0.00023553730623665305,
|
|
"loss_nats": 1.2875282764434814,
|
|
"step": 2870,
|
|
"unclipped_grad_norm": 1.1067496538162231
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8563188196588458,
|
|
"learning_rate": 0.00023506089246046403,
|
|
"loss_nats": 1.2867021560668945,
|
|
"step": 2880,
|
|
"unclipped_grad_norm": 1.0256601572036743
|
|
},
|
|
{
|
|
"bits_per_byte": 1.76771198207312,
|
|
"learning_rate": 0.00023458328153692736,
|
|
"loss_nats": 1.2252845764160156,
|
|
"step": 2890,
|
|
"unclipped_grad_norm": 1.0549262762069702
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7222432088380106,
|
|
"learning_rate": 0.0002341044816270961,
|
|
"loss_nats": 1.19376802444458,
|
|
"step": 2900,
|
|
"unclipped_grad_norm": 1.0038959980010986
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7887707417233183,
|
|
"learning_rate": 0.00023362450091233953,
|
|
"loss_nats": 1.2398813962936401,
|
|
"step": 2910,
|
|
"unclipped_grad_norm": 1.020102620124817
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7699561836842288,
|
|
"learning_rate": 0.00023314334759420388,
|
|
"loss_nats": 1.2268401384353638,
|
|
"step": 2920,
|
|
"unclipped_grad_norm": 0.9710083603858948
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7845655939270717,
|
|
"learning_rate": 0.0002326610298942717,
|
|
"loss_nats": 1.236966609954834,
|
|
"step": 2930,
|
|
"unclipped_grad_norm": 1.063584566116333
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8044884081683212,
|
|
"learning_rate": 0.00023217755605402175,
|
|
"loss_nats": 1.2507760524749756,
|
|
"step": 2940,
|
|
"unclipped_grad_norm": 1.1001172065734863
|
|
},
|
|
{
|
|
"bits_per_byte": 1.788769881810064,
|
|
"learning_rate": 0.00023169293433468802,
|
|
"loss_nats": 1.2398808002471924,
|
|
"step": 2950,
|
|
"unclipped_grad_norm": 1.052484393119812
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8266576597951407,
|
|
"learning_rate": 0.00023120717301711857,
|
|
"loss_nats": 1.2661426067352295,
|
|
"step": 2960,
|
|
"unclipped_grad_norm": 1.0507400035858154
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7942014378896132,
|
|
"learning_rate": 0.00023072028040163408,
|
|
"loss_nats": 1.2436456680297852,
|
|
"step": 2970,
|
|
"unclipped_grad_norm": 1.016513466835022
|
|
},
|
|
{
|
|
"bits_per_byte": 1.776544667056132,
|
|
"learning_rate": 0.00023023226480788593,
|
|
"loss_nats": 1.2314069271087646,
|
|
"step": 2980,
|
|
"unclipped_grad_norm": 1.0853108167648315
|
|
},
|
|
{
|
|
"bits_per_byte": 1.797620281006115,
|
|
"learning_rate": 0.00022974313457471416,
|
|
"loss_nats": 1.2460154294967651,
|
|
"step": 2990,
|
|
"unclipped_grad_norm": 1.025924563407898
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8018763356670169,
|
|
"learning_rate": 0.0002292528980600049,
|
|
"loss_nats": 1.2489655017852783,
|
|
"step": 3000,
|
|
"unclipped_grad_norm": 1.066794514656067
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8036527444677788,
|
|
"learning_rate": 0.00022876156364054758,
|
|
"loss_nats": 1.2501968145370483,
|
|
"step": 3010,
|
|
"unclipped_grad_norm": 0.9892268180847168
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7646690930314042,
|
|
"learning_rate": 0.0002282691397118918,
|
|
"loss_nats": 1.2231754064559937,
|
|
"step": 3020,
|
|
"unclipped_grad_norm": 1.1210085153579712
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8358253669820335,
|
|
"learning_rate": 0.00022777563468820376,
|
|
"loss_nats": 1.2724971771240234,
|
|
"step": 3030,
|
|
"unclipped_grad_norm": 1.0379256010055542
|
|
},
|
|
{
|
|
"bits_per_byte": 1.763304066731506,
|
|
"learning_rate": 0.00022728105700212282,
|
|
"loss_nats": 1.222229242324829,
|
|
"step": 3040,
|
|
"unclipped_grad_norm": 1.0503100156784058
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7804524568490367,
|
|
"learning_rate": 0.00022678541510461698,
|
|
"loss_nats": 1.2341156005859375,
|
|
"step": 3050,
|
|
"unclipped_grad_norm": 1.0255935192108154
|
|
},
|
|
{
|
|
"bits_per_byte": 1.779026720673384,
|
|
"learning_rate": 0.00022628871746483876,
|
|
"loss_nats": 1.2331273555755615,
|
|
"step": 3060,
|
|
"unclipped_grad_norm": 1.0667622089385986
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7089128335696342,
|
|
"learning_rate": 0.0002257909725699804,
|
|
"loss_nats": 1.184528112411499,
|
|
"step": 3070,
|
|
"unclipped_grad_norm": 0.9528610110282898
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7514905784437398,
|
|
"learning_rate": 0.00022529218892512886,
|
|
"loss_nats": 1.214040756225586,
|
|
"step": 3080,
|
|
"unclipped_grad_norm": 0.9759883880615234
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8129533942437885,
|
|
"learning_rate": 0.00022479237505312034,
|
|
"loss_nats": 1.256643533706665,
|
|
"step": 3090,
|
|
"unclipped_grad_norm": 1.074378490447998
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7348117009630646,
|
|
"learning_rate": 0.00022429153949439496,
|
|
"loss_nats": 1.2024798393249512,
|
|
"step": 3100,
|
|
"unclipped_grad_norm": 1.0595066547393799
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8111885082806345,
|
|
"learning_rate": 0.00022378969080685053,
|
|
"loss_nats": 1.255420207977295,
|
|
"step": 3110,
|
|
"unclipped_grad_norm": 1.0304162502288818
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7739528885076294,
|
|
"learning_rate": 0.0002232868375656965,
|
|
"loss_nats": 1.2296104431152344,
|
|
"step": 3120,
|
|
"unclipped_grad_norm": 0.9790826439857483
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8333825254091782,
|
|
"learning_rate": 0.00022278298836330733,
|
|
"loss_nats": 1.2708039283752441,
|
|
"step": 3130,
|
|
"unclipped_grad_norm": 1.0250012874603271
|
|
},
|
|
{
|
|
"bits_per_byte": 1.807222588351738,
|
|
"learning_rate": 0.00022227815180907572,
|
|
"loss_nats": 1.252671241760254,
|
|
"step": 3140,
|
|
"unclipped_grad_norm": 1.065406322479248
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7667856835155729,
|
|
"learning_rate": 0.00022177233652926546,
|
|
"loss_nats": 1.2246425151824951,
|
|
"step": 3150,
|
|
"unclipped_grad_norm": 1.1064149141311646
|
|
},
|
|
{
|
|
"bits_per_byte": 1.77101886448391,
|
|
"learning_rate": 0.00022126555116686416,
|
|
"loss_nats": 1.227576732635498,
|
|
"step": 3160,
|
|
"unclipped_grad_norm": 0.9769210815429688
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6904848925296843,
|
|
"learning_rate": 0.00022075780438143532,
|
|
"loss_nats": 1.1717548370361328,
|
|
"step": 3170,
|
|
"unclipped_grad_norm": 1.003720998764038
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7897881910858227,
|
|
"learning_rate": 0.00022024910484897065,
|
|
"loss_nats": 1.2405866384506226,
|
|
"step": 3180,
|
|
"unclipped_grad_norm": 0.9989861249923706
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7613923075845153,
|
|
"learning_rate": 0.00021973946126174158,
|
|
"loss_nats": 1.2209041118621826,
|
|
"step": 3190,
|
|
"unclipped_grad_norm": 1.0040642023086548
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7417902409871249,
|
|
"learning_rate": 0.00021922888232815086,
|
|
"loss_nats": 1.2073169946670532,
|
|
"step": 3200,
|
|
"unclipped_grad_norm": 0.9894367456436157
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7268853645500994,
|
|
"learning_rate": 0.0002187173767725837,
|
|
"loss_nats": 1.1969857215881348,
|
|
"step": 3210,
|
|
"unclipped_grad_norm": 1.0016205310821533
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7365394386736324,
|
|
"learning_rate": 0.0002182049533352588,
|
|
"loss_nats": 1.2036774158477783,
|
|
"step": 3220,
|
|
"unclipped_grad_norm": 0.9552999138832092
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8281382584364185,
|
|
"learning_rate": 0.00021769162077207884,
|
|
"loss_nats": 1.2671688795089722,
|
|
"step": 3230,
|
|
"unclipped_grad_norm": 1.0519078969955444
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7008661093010653,
|
|
"learning_rate": 0.00021717738785448102,
|
|
"loss_nats": 1.178950548171997,
|
|
"step": 3240,
|
|
"unclipped_grad_norm": 0.956534206867218
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8618967329742793,
|
|
"learning_rate": 0.00021666226336928708,
|
|
"loss_nats": 1.290568470954895,
|
|
"step": 3250,
|
|
"unclipped_grad_norm": 1.118624210357666
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7141249397866825,
|
|
"learning_rate": 0.0002161462561185532,
|
|
"loss_nats": 1.188140869140625,
|
|
"step": 3260,
|
|
"unclipped_grad_norm": 0.9860926270484924
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7454612106697915,
|
|
"learning_rate": 0.00021562937491941957,
|
|
"loss_nats": 1.2098615169525146,
|
|
"step": 3270,
|
|
"unclipped_grad_norm": 0.968685507774353
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8017138120619516,
|
|
"learning_rate": 0.00021511162860395975,
|
|
"loss_nats": 1.2488528490066528,
|
|
"step": 3280,
|
|
"unclipped_grad_norm": 1.0125019550323486
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7954340375483469,
|
|
"learning_rate": 0.0002145930260190298,
|
|
"loss_nats": 1.2445000410079956,
|
|
"step": 3290,
|
|
"unclipped_grad_norm": 1.0358432531356812
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7138430602219183,
|
|
"learning_rate": 0.00021407357602611698,
|
|
"loss_nats": 1.1879454851150513,
|
|
"step": 3300,
|
|
"unclipped_grad_norm": 1.0642298460006714
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7570346111769533,
|
|
"learning_rate": 0.0002135532875011885,
|
|
"loss_nats": 1.217883586883545,
|
|
"step": 3310,
|
|
"unclipped_grad_norm": 1.017012596130371
|
|
},
|
|
{
|
|
"bits_per_byte": 1.753239298037713,
|
|
"learning_rate": 0.0002130321693345397,
|
|
"loss_nats": 1.2152528762817383,
|
|
"step": 3320,
|
|
"unclipped_grad_norm": 0.9849429130554199
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7669095110241941,
|
|
"learning_rate": 0.00021251023043064224,
|
|
"loss_nats": 1.2247283458709717,
|
|
"step": 3330,
|
|
"unclipped_grad_norm": 0.9692639708518982
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7337540076602584,
|
|
"learning_rate": 0.00021198747970799188,
|
|
"loss_nats": 1.2017467021942139,
|
|
"step": 3340,
|
|
"unclipped_grad_norm": 0.9712040424346924
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7289699662612077,
|
|
"learning_rate": 0.00021146392609895618,
|
|
"loss_nats": 1.1984306573867798,
|
|
"step": 3350,
|
|
"unclipped_grad_norm": 0.9344873428344727
|
|
},
|
|
{
|
|
"bits_per_byte": 1.771012673108479,
|
|
"learning_rate": 0.00021093957854962178,
|
|
"loss_nats": 1.2275724411010742,
|
|
"step": 3360,
|
|
"unclipped_grad_norm": 1.0185227394104004
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7121330367243894,
|
|
"learning_rate": 0.00021041444601964157,
|
|
"loss_nats": 1.1867601871490479,
|
|
"step": 3370,
|
|
"unclipped_grad_norm": 1.017809271812439
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7444269070075025,
|
|
"learning_rate": 0.00020988853748208164,
|
|
"loss_nats": 1.2091445922851562,
|
|
"step": 3380,
|
|
"unclipped_grad_norm": 0.9594104886054993
|
|
},
|
|
{
|
|
"bits_per_byte": 1.782493202984175,
|
|
"learning_rate": 0.00020936186192326782,
|
|
"loss_nats": 1.235530138015747,
|
|
"step": 3390,
|
|
"unclipped_grad_norm": 1.0056157112121582
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8303659497130447,
|
|
"learning_rate": 0.00020883442834263225,
|
|
"loss_nats": 1.2687129974365234,
|
|
"step": 3400,
|
|
"unclipped_grad_norm": 0.9542431235313416
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8696024156461875,
|
|
"learning_rate": 0.0002083062457525596,
|
|
"loss_nats": 1.2959096431732178,
|
|
"step": 3410,
|
|
"unclipped_grad_norm": 1.0261633396148682
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7306273630675726,
|
|
"learning_rate": 0.00020777732317823306,
|
|
"loss_nats": 1.1995794773101807,
|
|
"step": 3420,
|
|
"unclipped_grad_norm": 1.0138797760009766
|
|
},
|
|
{
|
|
"bits_per_byte": 1.663148250175041,
|
|
"learning_rate": 0.00020724766965748005,
|
|
"loss_nats": 1.1528065204620361,
|
|
"step": 3430,
|
|
"unclipped_grad_norm": 0.9656930565834045
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7509053214828536,
|
|
"learning_rate": 0.00020671729424061788,
|
|
"loss_nats": 1.2136350870132446,
|
|
"step": 3440,
|
|
"unclipped_grad_norm": 1.1262567043304443
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8269866626062412,
|
|
"learning_rate": 0.00020618620599029906,
|
|
"loss_nats": 1.2663706541061401,
|
|
"step": 3450,
|
|
"unclipped_grad_norm": 1.0454469919204712
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7413009503454202,
|
|
"learning_rate": 0.00020565441398135649,
|
|
"loss_nats": 1.2069778442382812,
|
|
"step": 3460,
|
|
"unclipped_grad_norm": 1.1041505336761475
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7427603951206418,
|
|
"learning_rate": 0.00020512192730064827,
|
|
"loss_nats": 1.2079894542694092,
|
|
"step": 3470,
|
|
"unclipped_grad_norm": 1.0516799688339233
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6734345325231454,
|
|
"learning_rate": 0.00020458875504690273,
|
|
"loss_nats": 1.1599364280700684,
|
|
"step": 3480,
|
|
"unclipped_grad_norm": 0.9458639621734619
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8872512752777288,
|
|
"learning_rate": 0.00020405490633056245,
|
|
"loss_nats": 1.308142900466919,
|
|
"step": 3490,
|
|
"unclipped_grad_norm": 1.0662349462509155
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7517492403506374,
|
|
"learning_rate": 0.00020352039027362917,
|
|
"loss_nats": 1.2142200469970703,
|
|
"step": 3500,
|
|
"unclipped_grad_norm": 1.0241658687591553
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7728004327641977,
|
|
"learning_rate": 0.0002029852160095075,
|
|
"loss_nats": 1.2288116216659546,
|
|
"step": 3510,
|
|
"unclipped_grad_norm": 1.0039124488830566
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7731452579791778,
|
|
"learning_rate": 0.000202449392682849,
|
|
"loss_nats": 1.229050636291504,
|
|
"step": 3520,
|
|
"unclipped_grad_norm": 0.9908767342567444
|
|
},
|
|
{
|
|
"bits_per_byte": 1.743414617124524,
|
|
"learning_rate": 0.0002019129294493959,
|
|
"loss_nats": 1.2084429264068604,
|
|
"step": 3530,
|
|
"unclipped_grad_norm": 0.9787898063659668
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7481145590073028,
|
|
"learning_rate": 0.00020137583547582478,
|
|
"loss_nats": 1.211700677871704,
|
|
"step": 3540,
|
|
"unclipped_grad_norm": 1.0219446420669556
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7531405799961175,
|
|
"learning_rate": 0.00020083811993958973,
|
|
"loss_nats": 1.2151844501495361,
|
|
"step": 3550,
|
|
"unclipped_grad_norm": 1.2019073963165283
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6952811446969462,
|
|
"learning_rate": 0.00020029979202876563,
|
|
"loss_nats": 1.175079345703125,
|
|
"step": 3560,
|
|
"unclipped_grad_norm": 0.9977114796638489
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7182931113129936,
|
|
"learning_rate": 0.00019976086094189118,
|
|
"loss_nats": 1.1910300254821777,
|
|
"step": 3570,
|
|
"unclipped_grad_norm": 1.0032203197479248
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7239327664000867,
|
|
"learning_rate": 0.00019922133588781175,
|
|
"loss_nats": 1.194939136505127,
|
|
"step": 3580,
|
|
"unclipped_grad_norm": 1.0535842180252075
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7655023489748345,
|
|
"learning_rate": 0.0001986812260855219,
|
|
"loss_nats": 1.2237529754638672,
|
|
"step": 3590,
|
|
"unclipped_grad_norm": 1.055069923400879
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7734857836278861,
|
|
"learning_rate": 0.00019814054076400787,
|
|
"loss_nats": 1.2292866706848145,
|
|
"step": 3600,
|
|
"unclipped_grad_norm": 1.0577540397644043
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7140950148054324,
|
|
"learning_rate": 0.00019759928916209004,
|
|
"loss_nats": 1.1881201267242432,
|
|
"step": 3610,
|
|
"unclipped_grad_norm": 1.0167864561080933
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6696412831757155,
|
|
"learning_rate": 0.00019705748052826493,
|
|
"loss_nats": 1.1573071479797363,
|
|
"step": 3620,
|
|
"unclipped_grad_norm": 0.9244848489761353
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7374531824976664,
|
|
"learning_rate": 0.00019651512412054723,
|
|
"loss_nats": 1.2043107748031616,
|
|
"step": 3630,
|
|
"unclipped_grad_norm": 0.9926364421844482
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7074227758825589,
|
|
"learning_rate": 0.00019597222920631144,
|
|
"loss_nats": 1.183495283126831,
|
|
"step": 3640,
|
|
"unclipped_grad_norm": 0.9468497633934021
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7634653864580154,
|
|
"learning_rate": 0.0001954288050621338,
|
|
"loss_nats": 1.2223410606384277,
|
|
"step": 3650,
|
|
"unclipped_grad_norm": 0.9865193367004395
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6666551484087844,
|
|
"learning_rate": 0.00019488486097363363,
|
|
"loss_nats": 1.1552373170852661,
|
|
"step": 3660,
|
|
"unclipped_grad_norm": 1.0027393102645874
|
|
},
|
|
{
|
|
"bits_per_byte": 1.749947206134897,
|
|
"learning_rate": 0.00019434040623531452,
|
|
"loss_nats": 1.2129709720611572,
|
|
"step": 3670,
|
|
"unclipped_grad_norm": 1.0161325931549072
|
|
},
|
|
{
|
|
"bits_per_byte": 1.700321268263132,
|
|
"learning_rate": 0.0001937954501504058,
|
|
"loss_nats": 1.1785728931427002,
|
|
"step": 3680,
|
|
"unclipped_grad_norm": 1.0996551513671875
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6689437215438159,
|
|
"learning_rate": 0.00019325000203070333,
|
|
"loss_nats": 1.1568236351013184,
|
|
"step": 3690,
|
|
"unclipped_grad_norm": 0.9738581776618958
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6837737855277157,
|
|
"learning_rate": 0.0001927040711964106,
|
|
"loss_nats": 1.1671030521392822,
|
|
"step": 3700,
|
|
"unclipped_grad_norm": 1.0792776346206665
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7280950905162684,
|
|
"learning_rate": 0.00019215766697597923,
|
|
"loss_nats": 1.197824239730835,
|
|
"step": 3710,
|
|
"unclipped_grad_norm": 0.9678228497505188
|
|
},
|
|
{
|
|
"bits_per_byte": 1.705079684247204,
|
|
"learning_rate": 0.00019161079870594978,
|
|
"loss_nats": 1.1818711757659912,
|
|
"step": 3720,
|
|
"unclipped_grad_norm": 0.9917497634887695
|
|
},
|
|
{
|
|
"bits_per_byte": 1.649159353336511,
|
|
"learning_rate": 0.0001910634757307922,
|
|
"loss_nats": 1.1431101560592651,
|
|
"step": 3730,
|
|
"unclipped_grad_norm": 0.9426941275596619
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6954601786364945,
|
|
"learning_rate": 0.0001905157074027459,
|
|
"loss_nats": 1.1752034425735474,
|
|
"step": 3740,
|
|
"unclipped_grad_norm": 1.0086418390274048
|
|
},
|
|
{
|
|
"bits_per_byte": 1.721657263946521,
|
|
"learning_rate": 0.00018996750308166037,
|
|
"loss_nats": 1.1933618783950806,
|
|
"step": 3750,
|
|
"unclipped_grad_norm": 0.9619976878166199
|
|
},
|
|
{
|
|
"bits_per_byte": 1.808596729732132,
|
|
"learning_rate": 0.00018941887213483482,
|
|
"loss_nats": 1.2536237239837646,
|
|
"step": 3760,
|
|
"unclipped_grad_norm": 1.012074589729309
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7638779728374354,
|
|
"learning_rate": 0.00018886982393685845,
|
|
"loss_nats": 1.22262704372406,
|
|
"step": 3770,
|
|
"unclipped_grad_norm": 1.050723910331726
|
|
},
|
|
{
|
|
"bits_per_byte": 1.589899463311364,
|
|
"learning_rate": 0.00018832036786945008,
|
|
"loss_nats": 1.102034330368042,
|
|
"step": 3780,
|
|
"unclipped_grad_norm": 0.9615285992622375
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6942863970443558,
|
|
"learning_rate": 0.0001877705133212979,
|
|
"loss_nats": 1.1743898391723633,
|
|
"step": 3790,
|
|
"unclipped_grad_norm": 0.9870805144309998
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6994689222454558,
|
|
"learning_rate": 0.00018722026968789906,
|
|
"loss_nats": 1.1779820919036865,
|
|
"step": 3800,
|
|
"unclipped_grad_norm": 0.9850614070892334
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7199389852817506,
|
|
"learning_rate": 0.00018666964637139907,
|
|
"loss_nats": 1.1921708583831787,
|
|
"step": 3810,
|
|
"unclipped_grad_norm": 0.9774236083030701
|
|
},
|
|
{
|
|
"bits_per_byte": 1.718822817877651,
|
|
"learning_rate": 0.00018611865278043115,
|
|
"loss_nats": 1.1913971900939941,
|
|
"step": 3820,
|
|
"unclipped_grad_norm": 1.0395604372024536
|
|
},
|
|
{
|
|
"bits_per_byte": 1.762180848038721,
|
|
"learning_rate": 0.00018556729832995555,
|
|
"loss_nats": 1.221450686454773,
|
|
"step": 3830,
|
|
"unclipped_grad_norm": 1.0331096649169922
|
|
},
|
|
{
|
|
"bits_per_byte": 1.68565630762406,
|
|
"learning_rate": 0.00018501559244109864,
|
|
"loss_nats": 1.168407917022705,
|
|
"step": 3840,
|
|
"unclipped_grad_norm": 1.0491844415664673
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6311636046981792,
|
|
"learning_rate": 0.00018446354454099178,
|
|
"loss_nats": 1.13063645362854,
|
|
"step": 3850,
|
|
"unclipped_grad_norm": 0.9598016738891602
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7078809376644575,
|
|
"learning_rate": 0.00018391116406261048,
|
|
"loss_nats": 1.1838128566741943,
|
|
"step": 3860,
|
|
"unclipped_grad_norm": 1.0586371421813965
|
|
},
|
|
{
|
|
"bits_per_byte": 1.778100422115837,
|
|
"learning_rate": 0.00018335846044461305,
|
|
"loss_nats": 1.232485294342041,
|
|
"step": 3870,
|
|
"unclipped_grad_norm": 0.9732621908187866
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6634643542873266,
|
|
"learning_rate": 0.0001828054431311793,
|
|
"loss_nats": 1.1530256271362305,
|
|
"step": 3880,
|
|
"unclipped_grad_norm": 0.9620565176010132
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6362388127251406,
|
|
"learning_rate": 0.00018225212157184934,
|
|
"loss_nats": 1.1341543197631836,
|
|
"step": 3890,
|
|
"unclipped_grad_norm": 0.9925653338432312
|
|
},
|
|
{
|
|
"bits_per_byte": 1.721069255263221,
|
|
"learning_rate": 0.00018169850522136192,
|
|
"loss_nats": 1.1929543018341064,
|
|
"step": 3900,
|
|
"unclipped_grad_norm": 1.121357798576355
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7172674067832479,
|
|
"learning_rate": 0.000181144603539493,
|
|
"loss_nats": 1.1903190612792969,
|
|
"step": 3910,
|
|
"unclipped_grad_norm": 1.0240309238433838
|
|
},
|
|
{
|
|
"bits_per_byte": 1.671385531220766,
|
|
"learning_rate": 0.00018059042599089412,
|
|
"loss_nats": 1.1585161685943604,
|
|
"step": 3920,
|
|
"unclipped_grad_norm": 0.9621630311012268
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7072636919305109,
|
|
"learning_rate": 0.00018003598204493048,
|
|
"loss_nats": 1.1833850145339966,
|
|
"step": 3930,
|
|
"unclipped_grad_norm": 0.9956491589546204
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7254322831329596,
|
|
"learning_rate": 0.00017948128117551936,
|
|
"loss_nats": 1.1959785223007202,
|
|
"step": 3940,
|
|
"unclipped_grad_norm": 0.9730517268180847
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7623370082857046,
|
|
"learning_rate": 0.00017892633286096816,
|
|
"loss_nats": 1.221558928489685,
|
|
"step": 3950,
|
|
"unclipped_grad_norm": 0.9878451228141785
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7644881672826966,
|
|
"learning_rate": 0.00017837114658381247,
|
|
"loss_nats": 1.2230499982833862,
|
|
"step": 3960,
|
|
"unclipped_grad_norm": 1.003400444984436
|
|
},
|
|
{
|
|
"bits_per_byte": 1.682019734471566,
|
|
"learning_rate": 0.00017781573183065386,
|
|
"loss_nats": 1.1658872365951538,
|
|
"step": 3970,
|
|
"unclipped_grad_norm": 1.0887466669082642
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7137744391442242,
|
|
"learning_rate": 0.0001772600980919981,
|
|
"loss_nats": 1.1878979206085205,
|
|
"step": 3980,
|
|
"unclipped_grad_norm": 1.0592182874679565
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7370323409510051,
|
|
"learning_rate": 0.0001767042548620927,
|
|
"loss_nats": 1.2040190696716309,
|
|
"step": 3990,
|
|
"unclipped_grad_norm": 1.0668442249298096
|
|
},
|
|
{
|
|
"bits_per_byte": 1.620528197568823,
|
|
"learning_rate": 0.00017614821163876486,
|
|
"loss_nats": 1.1232645511627197,
|
|
"step": 4000,
|
|
"unclipped_grad_norm": 0.979034423828125
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7224021208074078,
|
|
"learning_rate": 0.00017559197792325906,
|
|
"loss_nats": 1.193878173828125,
|
|
"step": 4010,
|
|
"unclipped_grad_norm": 1.0592741966247559
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6779786581242426,
|
|
"learning_rate": 0.0001750355632200748,
|
|
"loss_nats": 1.163086175918579,
|
|
"step": 4020,
|
|
"unclipped_grad_norm": 1.0186548233032227
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7290010951210137,
|
|
"learning_rate": 0.00017447897703680418,
|
|
"loss_nats": 1.1984522342681885,
|
|
"step": 4030,
|
|
"unclipped_grad_norm": 1.0059210062026978
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6595417739864478,
|
|
"learning_rate": 0.0001739222288839694,
|
|
"loss_nats": 1.1503067016601562,
|
|
"step": 4040,
|
|
"unclipped_grad_norm": 1.009366750717163
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6963183720643,
|
|
"learning_rate": 0.00017336532827486026,
|
|
"loss_nats": 1.1757982969284058,
|
|
"step": 4050,
|
|
"unclipped_grad_norm": 1.0469119548797607
|
|
},
|
|
{
|
|
"bits_per_byte": 1.706078903448717,
|
|
"learning_rate": 0.00017280828472537163,
|
|
"loss_nats": 1.1825637817382812,
|
|
"step": 4060,
|
|
"unclipped_grad_norm": 1.0410232543945312
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7226642223673228,
|
|
"learning_rate": 0.0001722511077538408,
|
|
"loss_nats": 1.1940598487854004,
|
|
"step": 4070,
|
|
"unclipped_grad_norm": 1.0424585342407227
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7834826191745887,
|
|
"learning_rate": 0.00017169380688088497,
|
|
"loss_nats": 1.2362159490585327,
|
|
"step": 4080,
|
|
"unclipped_grad_norm": 1.0965845584869385
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6886807945221334,
|
|
"learning_rate": 0.0001711363916292384,
|
|
"loss_nats": 1.1705043315887451,
|
|
"step": 4090,
|
|
"unclipped_grad_norm": 1.0564258098602295
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6871728506393684,
|
|
"learning_rate": 0.00017057887152358974,
|
|
"loss_nats": 1.1694591045379639,
|
|
"step": 4100,
|
|
"unclipped_grad_norm": 1.032038688659668
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6672990514536148,
|
|
"learning_rate": 0.0001700212560904194,
|
|
"loss_nats": 1.1556836366653442,
|
|
"step": 4110,
|
|
"unclipped_grad_norm": 1.0212913751602173
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7325631997856843,
|
|
"learning_rate": 0.00016946355485783656,
|
|
"loss_nats": 1.2009212970733643,
|
|
"step": 4120,
|
|
"unclipped_grad_norm": 1.1421489715576172
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6623953101295634,
|
|
"learning_rate": 0.0001689057773554166,
|
|
"loss_nats": 1.1522846221923828,
|
|
"step": 4130,
|
|
"unclipped_grad_norm": 1.0443917512893677
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7315172013031366,
|
|
"learning_rate": 0.00016834793311403806,
|
|
"loss_nats": 1.2001962661743164,
|
|
"step": 4140,
|
|
"unclipped_grad_norm": 1.0389643907546997
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6632394009799982,
|
|
"learning_rate": 0.00016779003166571981,
|
|
"loss_nats": 1.152869701385498,
|
|
"step": 4150,
|
|
"unclipped_grad_norm": 0.9869729280471802
|
|
},
|
|
{
|
|
"bits_per_byte": 1.697900440469587,
|
|
"learning_rate": 0.00016723208254345838,
|
|
"loss_nats": 1.1768949031829834,
|
|
"step": 4160,
|
|
"unclipped_grad_norm": 1.018755316734314
|
|
},
|
|
{
|
|
"bits_per_byte": 1.74134876152236,
|
|
"learning_rate": 0.00016667409528106481,
|
|
"loss_nats": 1.2070109844207764,
|
|
"step": 4170,
|
|
"unclipped_grad_norm": 1.0534698963165283
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7750522016119445,
|
|
"learning_rate": 0.00016611607941300184,
|
|
"loss_nats": 1.230372428894043,
|
|
"step": 4180,
|
|
"unclipped_grad_norm": 1.1320860385894775
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6878769476120008,
|
|
"learning_rate": 0.0001655580444742211,
|
|
"loss_nats": 1.1699471473693848,
|
|
"step": 4190,
|
|
"unclipped_grad_norm": 1.0554869174957275
|
|
},
|
|
{
|
|
"bits_per_byte": 1.725129765650092,
|
|
"learning_rate": 0.00016499999999999997,
|
|
"loss_nats": 1.1957688331604004,
|
|
"step": 4200,
|
|
"unclipped_grad_norm": 1.0452762842178345
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7145272072070508,
|
|
"learning_rate": 0.0001644419555257789,
|
|
"loss_nats": 1.1884196996688843,
|
|
"step": 4210,
|
|
"unclipped_grad_norm": 0.9850034117698669
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7283826455085112,
|
|
"learning_rate": 0.0001638839205869981,
|
|
"loss_nats": 1.1980235576629639,
|
|
"step": 4220,
|
|
"unclipped_grad_norm": 1.0109167098999023
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7140664656853892,
|
|
"learning_rate": 0.0001633259047189352,
|
|
"loss_nats": 1.1881003379821777,
|
|
"step": 4230,
|
|
"unclipped_grad_norm": 1.0496208667755127
|
|
},
|
|
{
|
|
"bits_per_byte": 1.659261958213494,
|
|
"learning_rate": 0.0001627679174565416,
|
|
"loss_nats": 1.1501127481460571,
|
|
"step": 4240,
|
|
"unclipped_grad_norm": 0.9920905232429504
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6850357942197471,
|
|
"learning_rate": 0.00016220996833428016,
|
|
"loss_nats": 1.1679778099060059,
|
|
"step": 4250,
|
|
"unclipped_grad_norm": 1.0322012901306152
|
|
},
|
|
{
|
|
"bits_per_byte": 1.667805196395104,
|
|
"learning_rate": 0.00016165206688596191,
|
|
"loss_nats": 1.1560344696044922,
|
|
"step": 4260,
|
|
"unclipped_grad_norm": 1.0430845022201538
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6625848350108143,
|
|
"learning_rate": 0.00016109422264458335,
|
|
"loss_nats": 1.1524159908294678,
|
|
"step": 4270,
|
|
"unclipped_grad_norm": 1.0265511274337769
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6648945620119018,
|
|
"learning_rate": 0.00016053644514216336,
|
|
"loss_nats": 1.1540169715881348,
|
|
"step": 4280,
|
|
"unclipped_grad_norm": 0.9929917454719543
|
|
},
|
|
{
|
|
"bits_per_byte": 1.72171487813456,
|
|
"learning_rate": 0.00015997874390958058,
|
|
"loss_nats": 1.19340181350708,
|
|
"step": 4290,
|
|
"unclipped_grad_norm": 1.0577914714813232
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6424783432984433,
|
|
"learning_rate": 0.00015942112847641018,
|
|
"loss_nats": 1.138479232788086,
|
|
"step": 4300,
|
|
"unclipped_grad_norm": 1.0359195470809937
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7237528725472844,
|
|
"learning_rate": 0.00015886360837076156,
|
|
"loss_nats": 1.1948144435882568,
|
|
"step": 4310,
|
|
"unclipped_grad_norm": 1.124787449836731
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7081155220002342,
|
|
"learning_rate": 0.00015830619311911495,
|
|
"loss_nats": 1.1839754581451416,
|
|
"step": 4320,
|
|
"unclipped_grad_norm": 1.0496878623962402
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6609448084521865,
|
|
"learning_rate": 0.00015774889224615915,
|
|
"loss_nats": 1.1512792110443115,
|
|
"step": 4330,
|
|
"unclipped_grad_norm": 1.0365654230117798
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6981388084236828,
|
|
"learning_rate": 0.00015719171527462834,
|
|
"loss_nats": 1.1770601272583008,
|
|
"step": 4340,
|
|
"unclipped_grad_norm": 1.0712857246398926
|
|
},
|
|
{
|
|
"bits_per_byte": 1.679871155214337,
|
|
"learning_rate": 0.0001566346717251397,
|
|
"loss_nats": 1.164397954940796,
|
|
"step": 4350,
|
|
"unclipped_grad_norm": 1.0368938446044922
|
|
},
|
|
{
|
|
"bits_per_byte": 1.658068570599157,
|
|
"learning_rate": 0.00015607777111603056,
|
|
"loss_nats": 1.1492855548858643,
|
|
"step": 4360,
|
|
"unclipped_grad_norm": 1.054489254951477
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6791636185886873,
|
|
"learning_rate": 0.00015552102296319576,
|
|
"loss_nats": 1.163907527923584,
|
|
"step": 4370,
|
|
"unclipped_grad_norm": 1.05704927444458
|
|
},
|
|
{
|
|
"bits_per_byte": 1.733114060216398,
|
|
"learning_rate": 0.00015496443677992516,
|
|
"loss_nats": 1.2013031244277954,
|
|
"step": 4380,
|
|
"unclipped_grad_norm": 1.066345453262329
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6866214742607022,
|
|
"learning_rate": 0.0001544080220767409,
|
|
"loss_nats": 1.169076919555664,
|
|
"step": 4390,
|
|
"unclipped_grad_norm": 1.039746880531311
|
|
},
|
|
{
|
|
"bits_per_byte": 1.656244522604106,
|
|
"learning_rate": 0.00015385178836123511,
|
|
"loss_nats": 1.1480212211608887,
|
|
"step": 4400,
|
|
"unclipped_grad_norm": 1.0477714538574219
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7190543065257124,
|
|
"learning_rate": 0.00015329574513790726,
|
|
"loss_nats": 1.1915576457977295,
|
|
"step": 4410,
|
|
"unclipped_grad_norm": 1.0500718355178833
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6606173534849438,
|
|
"learning_rate": 0.00015273990190800186,
|
|
"loss_nats": 1.1510522365570068,
|
|
"step": 4420,
|
|
"unclipped_grad_norm": 1.0457862615585327
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7152531459763425,
|
|
"learning_rate": 0.0001521842681693461,
|
|
"loss_nats": 1.1889228820800781,
|
|
"step": 4430,
|
|
"unclipped_grad_norm": 1.0248607397079468
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6480127450032085,
|
|
"learning_rate": 0.00015162885341618752,
|
|
"loss_nats": 1.14231538772583,
|
|
"step": 4440,
|
|
"unclipped_grad_norm": 1.026747703552246
|
|
},
|
|
{
|
|
"bits_per_byte": 1.64553137931656,
|
|
"learning_rate": 0.00015107366713903178,
|
|
"loss_nats": 1.1405954360961914,
|
|
"step": 4450,
|
|
"unclipped_grad_norm": 1.0325950384140015
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6368648295742814,
|
|
"learning_rate": 0.0001505187188244806,
|
|
"loss_nats": 1.1345882415771484,
|
|
"step": 4460,
|
|
"unclipped_grad_norm": 1.0030871629714966
|
|
},
|
|
{
|
|
"bits_per_byte": 1.686227977955528,
|
|
"learning_rate": 0.00014996401795506952,
|
|
"loss_nats": 1.1688041687011719,
|
|
"step": 4470,
|
|
"unclipped_grad_norm": 0.9844985604286194
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7063241507088474,
|
|
"learning_rate": 0.00014940957400910585,
|
|
"loss_nats": 1.1827337741851807,
|
|
"step": 4480,
|
|
"unclipped_grad_norm": 1.1298258304595947
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6590796566035795,
|
|
"learning_rate": 0.00014885539646050699,
|
|
"loss_nats": 1.1499863862991333,
|
|
"step": 4490,
|
|
"unclipped_grad_norm": 0.9567995071411133
|
|
},
|
|
{
|
|
"bits_per_byte": 1.699403568838128,
|
|
"learning_rate": 0.00014830149477863805,
|
|
"loss_nats": 1.1779367923736572,
|
|
"step": 4500,
|
|
"unclipped_grad_norm": 1.0041329860687256
|
|
},
|
|
{
|
|
"bits_per_byte": 1.630450564627702,
|
|
"learning_rate": 0.00014774787842815066,
|
|
"loss_nats": 1.1301422119140625,
|
|
"step": 4510,
|
|
"unclipped_grad_norm": 1.0097919702529907
|
|
},
|
|
{
|
|
"bits_per_byte": 1.677602188101504,
|
|
"learning_rate": 0.00014719455686882066,
|
|
"loss_nats": 1.1628252267837524,
|
|
"step": 4520,
|
|
"unclipped_grad_norm": 1.0421855449676514
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6955814264053528,
|
|
"learning_rate": 0.00014664153955538695,
|
|
"loss_nats": 1.1752874851226807,
|
|
"step": 4530,
|
|
"unclipped_grad_norm": 1.0619962215423584
|
|
},
|
|
{
|
|
"bits_per_byte": 1.738972993183341,
|
|
"learning_rate": 0.00014608883593738946,
|
|
"loss_nats": 1.2053642272949219,
|
|
"step": 4540,
|
|
"unclipped_grad_norm": 1.0259038209915161
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6138776284599583,
|
|
"learning_rate": 0.00014553645545900817,
|
|
"loss_nats": 1.118654727935791,
|
|
"step": 4550,
|
|
"unclipped_grad_norm": 1.1216750144958496
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6847919228208237,
|
|
"learning_rate": 0.0001449844075589013,
|
|
"loss_nats": 1.1678087711334229,
|
|
"step": 4560,
|
|
"unclipped_grad_norm": 1.1666667461395264
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6566538413131595,
|
|
"learning_rate": 0.0001444327016700444,
|
|
"loss_nats": 1.1483049392700195,
|
|
"step": 4570,
|
|
"unclipped_grad_norm": 0.9492276310920715
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6127442627907724,
|
|
"learning_rate": 0.0001438813472195688,
|
|
"loss_nats": 1.1178691387176514,
|
|
"step": 4580,
|
|
"unclipped_grad_norm": 1.0188902616500854
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6539726317862082,
|
|
"learning_rate": 0.00014333035362860093,
|
|
"loss_nats": 1.1464464664459229,
|
|
"step": 4590,
|
|
"unclipped_grad_norm": 1.0299646854400635
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6561746976478557,
|
|
"learning_rate": 0.00014277973031210088,
|
|
"loss_nats": 1.147972822189331,
|
|
"step": 4600,
|
|
"unclipped_grad_norm": 1.0103821754455566
|
|
},
|
|
{
|
|
"bits_per_byte": 1.719434732149421,
|
|
"learning_rate": 0.00014222948667870207,
|
|
"loss_nats": 1.1918213367462158,
|
|
"step": 4610,
|
|
"unclipped_grad_norm": 1.0808284282684326
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7013596995090414,
|
|
"learning_rate": 0.00014167963213054991,
|
|
"loss_nats": 1.1792926788330078,
|
|
"step": 4620,
|
|
"unclipped_grad_norm": 1.0917437076568604
|
|
},
|
|
{
|
|
"bits_per_byte": 1.683035292024911,
|
|
"learning_rate": 0.00014113017606314152,
|
|
"loss_nats": 1.1665911674499512,
|
|
"step": 4630,
|
|
"unclipped_grad_norm": 1.0124032497406006
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7275268598378177,
|
|
"learning_rate": 0.00014058112786516518,
|
|
"loss_nats": 1.1974303722381592,
|
|
"step": 4640,
|
|
"unclipped_grad_norm": 1.0515177249908447
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6549154406782383,
|
|
"learning_rate": 0.0001400324969183396,
|
|
"loss_nats": 1.1470999717712402,
|
|
"step": 4650,
|
|
"unclipped_grad_norm": 1.1102732419967651
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6181718632693518,
|
|
"learning_rate": 0.00013948429259725406,
|
|
"loss_nats": 1.1216312646865845,
|
|
"step": 4660,
|
|
"unclipped_grad_norm": 1.066238522529602
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7593243881905407,
|
|
"learning_rate": 0.00013893652426920776,
|
|
"loss_nats": 1.219470739364624,
|
|
"step": 4670,
|
|
"unclipped_grad_norm": 1.1023285388946533
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7532637195741354,
|
|
"learning_rate": 0.00013838920129405016,
|
|
"loss_nats": 1.2152698040008545,
|
|
"step": 4680,
|
|
"unclipped_grad_norm": 1.0341681241989136
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7266088164475122,
|
|
"learning_rate": 0.0001378423330240207,
|
|
"loss_nats": 1.196794033050537,
|
|
"step": 4690,
|
|
"unclipped_grad_norm": 1.0434808731079102
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6483247215318737,
|
|
"learning_rate": 0.00013729592880358936,
|
|
"loss_nats": 1.1425316333770752,
|
|
"step": 4700,
|
|
"unclipped_grad_norm": 1.045780897140503
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7162458298371228,
|
|
"learning_rate": 0.00013674999796929659,
|
|
"loss_nats": 1.1896109580993652,
|
|
"step": 4710,
|
|
"unclipped_grad_norm": 1.1512850522994995
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6600910865733036,
|
|
"learning_rate": 0.00013620454984959417,
|
|
"loss_nats": 1.1506874561309814,
|
|
"step": 4720,
|
|
"unclipped_grad_norm": 1.0870006084442139
|
|
},
|
|
{
|
|
"bits_per_byte": 1.579696248583626,
|
|
"learning_rate": 0.00013565959376468542,
|
|
"loss_nats": 1.0949620008468628,
|
|
"step": 4730,
|
|
"unclipped_grad_norm": 1.066159725189209
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6295564268258662,
|
|
"learning_rate": 0.00013511513902636634,
|
|
"loss_nats": 1.129522442817688,
|
|
"step": 4740,
|
|
"unclipped_grad_norm": 1.0375505685806274
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6749376608916864,
|
|
"learning_rate": 0.0001345711949378662,
|
|
"loss_nats": 1.1609783172607422,
|
|
"step": 4750,
|
|
"unclipped_grad_norm": 1.0134141445159912
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6434221840863785,
|
|
"learning_rate": 0.00013402777079368853,
|
|
"loss_nats": 1.1391334533691406,
|
|
"step": 4760,
|
|
"unclipped_grad_norm": 1.0448397397994995
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6362174868764336,
|
|
"learning_rate": 0.0001334848758794528,
|
|
"loss_nats": 1.1341395378112793,
|
|
"step": 4770,
|
|
"unclipped_grad_norm": 1.055105447769165
|
|
},
|
|
{
|
|
"bits_per_byte": 1.666649472981306,
|
|
"learning_rate": 0.00013294251947173504,
|
|
"loss_nats": 1.155233383178711,
|
|
"step": 4780,
|
|
"unclipped_grad_norm": 1.0775461196899414
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6314788488972107,
|
|
"learning_rate": 0.00013240071083790996,
|
|
"loss_nats": 1.1308549642562866,
|
|
"step": 4790,
|
|
"unclipped_grad_norm": 1.065148949623108
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6269109896902945,
|
|
"learning_rate": 0.0001318594592359921,
|
|
"loss_nats": 1.1276887655258179,
|
|
"step": 4800,
|
|
"unclipped_grad_norm": 1.1373785734176636
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6901620810940148,
|
|
"learning_rate": 0.0001313187739144781,
|
|
"loss_nats": 1.171531081199646,
|
|
"step": 4810,
|
|
"unclipped_grad_norm": 1.1410114765167236
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6112782826748178,
|
|
"learning_rate": 0.0001307786641121882,
|
|
"loss_nats": 1.1168529987335205,
|
|
"step": 4820,
|
|
"unclipped_grad_norm": 1.0628036260604858
|
|
},
|
|
{
|
|
"bits_per_byte": 1.649350942009572,
|
|
"learning_rate": 0.00013023913905810876,
|
|
"loss_nats": 1.1432429552078247,
|
|
"step": 4830,
|
|
"unclipped_grad_norm": 1.1659979820251465
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6571300612733986,
|
|
"learning_rate": 0.00012970020797123432,
|
|
"loss_nats": 1.1486350297927856,
|
|
"step": 4840,
|
|
"unclipped_grad_norm": 1.1067216396331787
|
|
},
|
|
{
|
|
"bits_per_byte": 1.668376866726572,
|
|
"learning_rate": 0.00012916188006041027,
|
|
"loss_nats": 1.156430721282959,
|
|
"step": 4850,
|
|
"unclipped_grad_norm": 0.9945695996284485
|
|
},
|
|
{
|
|
"bits_per_byte": 1.600552412671108,
|
|
"learning_rate": 0.00012862416452417516,
|
|
"loss_nats": 1.1094183921813965,
|
|
"step": 4860,
|
|
"unclipped_grad_norm": 1.0181769132614136
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6771526254521485,
|
|
"learning_rate": 0.00012808707055060405,
|
|
"loss_nats": 1.1625136137008667,
|
|
"step": 4870,
|
|
"unclipped_grad_norm": 1.0462807416915894
|
|
},
|
|
{
|
|
"bits_per_byte": 1.539689988305221,
|
|
"learning_rate": 0.000127550607317151,
|
|
"loss_nats": 1.0672317743301392,
|
|
"step": 4880,
|
|
"unclipped_grad_norm": 1.0612690448760986
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5967718900398415,
|
|
"learning_rate": 0.00012701478399049248,
|
|
"loss_nats": 1.1067979335784912,
|
|
"step": 4890,
|
|
"unclipped_grad_norm": 1.0629737377166748
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6387246499607118,
|
|
"learning_rate": 0.0001264796097263708,
|
|
"loss_nats": 1.1358773708343506,
|
|
"step": 4900,
|
|
"unclipped_grad_norm": 1.0522006750106812
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7368791044090865,
|
|
"learning_rate": 0.0001259450936694375,
|
|
"loss_nats": 1.2039128541946411,
|
|
"step": 4910,
|
|
"unclipped_grad_norm": 1.1116063594818115
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6773633041994556,
|
|
"learning_rate": 0.00012541124495309727,
|
|
"loss_nats": 1.1626596450805664,
|
|
"step": 4920,
|
|
"unclipped_grad_norm": 1.0861767530441284
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6375162998557495,
|
|
"learning_rate": 0.00012487807269935167,
|
|
"loss_nats": 1.1350398063659668,
|
|
"step": 4930,
|
|
"unclipped_grad_norm": 1.0190138816833496
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6159926711002692,
|
|
"learning_rate": 0.00012434558601864349,
|
|
"loss_nats": 1.1201207637786865,
|
|
"step": 4940,
|
|
"unclipped_grad_norm": 1.0380074977874756
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5895295286293578,
|
|
"learning_rate": 0.0001238137940097009,
|
|
"loss_nats": 1.1017779111862183,
|
|
"step": 4950,
|
|
"unclipped_grad_norm": 1.0411629676818848
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5717379233976003,
|
|
"learning_rate": 0.0001232827057593821,
|
|
"loss_nats": 1.08944571018219,
|
|
"step": 4960,
|
|
"unclipped_grad_norm": 1.0460548400878906
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6461665112461963,
|
|
"learning_rate": 0.00012275233034251992,
|
|
"loss_nats": 1.1410356760025024,
|
|
"step": 4970,
|
|
"unclipped_grad_norm": 1.112047553062439
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7016685803499911,
|
|
"learning_rate": 0.0001222226768217669,
|
|
"loss_nats": 1.179506778717041,
|
|
"step": 4980,
|
|
"unclipped_grad_norm": 1.062510371208191
|
|
},
|
|
{
|
|
"bits_per_byte": 1.636959248049605,
|
|
"learning_rate": 0.00012169375424744034,
|
|
"loss_nats": 1.1346536874771118,
|
|
"step": 4990,
|
|
"unclipped_grad_norm": 1.1910533905029297
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6019551031715449,
|
|
"learning_rate": 0.00012116557165736772,
|
|
"loss_nats": 1.1103906631469727,
|
|
"step": 5000,
|
|
"unclipped_grad_norm": 1.0508692264556885
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5837816964548719,
|
|
"learning_rate": 0.00012063813807673214,
|
|
"loss_nats": 1.0977938175201416,
|
|
"step": 5010,
|
|
"unclipped_grad_norm": 1.0227967500686646
|
|
},
|
|
{
|
|
"bits_per_byte": 1.639255904369227,
|
|
"learning_rate": 0.00012011146251791832,
|
|
"loss_nats": 1.136245608329773,
|
|
"step": 5020,
|
|
"unclipped_grad_norm": 1.0390269756317139
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6173587292960725,
|
|
"learning_rate": 0.0001195855539803584,
|
|
"loss_nats": 1.1210676431655884,
|
|
"step": 5030,
|
|
"unclipped_grad_norm": 1.083649754524231
|
|
},
|
|
{
|
|
"bits_per_byte": 1.591486175248224,
|
|
"learning_rate": 0.00011906042145037817,
|
|
"loss_nats": 1.1031341552734375,
|
|
"step": 5040,
|
|
"unclipped_grad_norm": 1.0160164833068848
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5612364907532663,
|
|
"learning_rate": 0.0001185360739010438,
|
|
"loss_nats": 1.0821666717529297,
|
|
"step": 5050,
|
|
"unclipped_grad_norm": 0.9697359800338745
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5996653261579574,
|
|
"learning_rate": 0.00011801252029200808,
|
|
"loss_nats": 1.1088035106658936,
|
|
"step": 5060,
|
|
"unclipped_grad_norm": 1.0825191736221313
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6612003746713688,
|
|
"learning_rate": 0.00011748976956935777,
|
|
"loss_nats": 1.151456356048584,
|
|
"step": 5070,
|
|
"unclipped_grad_norm": 1.051671028137207
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6135078657606032,
|
|
"learning_rate": 0.00011696783066546024,
|
|
"loss_nats": 1.1183984279632568,
|
|
"step": 5080,
|
|
"unclipped_grad_norm": 1.0758769512176514
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6053440213067967,
|
|
"learning_rate": 0.00011644671249881147,
|
|
"loss_nats": 1.1127396821975708,
|
|
"step": 5090,
|
|
"unclipped_grad_norm": 1.0736517906188965
|
|
},
|
|
{
|
|
"bits_per_byte": 1.620226196033908,
|
|
"learning_rate": 0.00011592642397388293,
|
|
"loss_nats": 1.1230552196502686,
|
|
"step": 5100,
|
|
"unclipped_grad_norm": 1.0084763765335083
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5469340695422136,
|
|
"learning_rate": 0.00011540697398097015,
|
|
"loss_nats": 1.0722529888153076,
|
|
"step": 5110,
|
|
"unclipped_grad_norm": 1.001043438911438
|
|
},
|
|
{
|
|
"bits_per_byte": 1.627192525289757,
|
|
"learning_rate": 0.00011488837139604016,
|
|
"loss_nats": 1.1278839111328125,
|
|
"step": 5120,
|
|
"unclipped_grad_norm": 1.0516539812088013
|
|
},
|
|
{
|
|
"bits_per_byte": 1.642070228467946,
|
|
"learning_rate": 0.00011437062508058039,
|
|
"loss_nats": 1.138196349143982,
|
|
"step": 5130,
|
|
"unclipped_grad_norm": 1.082858681678772
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6022958008029042,
|
|
"learning_rate": 0.00011385374388144673,
|
|
"loss_nats": 1.1106268167495728,
|
|
"step": 5140,
|
|
"unclipped_grad_norm": 1.0696102380752563
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6803482350878305,
|
|
"learning_rate": 0.00011333773663071286,
|
|
"loss_nats": 1.1647286415100098,
|
|
"step": 5150,
|
|
"unclipped_grad_norm": 1.045096755027771
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5608273440268638,
|
|
"learning_rate": 0.00011282261214551897,
|
|
"loss_nats": 1.0818830728530884,
|
|
"step": 5160,
|
|
"unclipped_grad_norm": 1.0852984189987183
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7284662290768305,
|
|
"learning_rate": 0.00011230837922792112,
|
|
"loss_nats": 1.1980814933776855,
|
|
"step": 5170,
|
|
"unclipped_grad_norm": 1.2249319553375244
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6274811121779047,
|
|
"learning_rate": 0.0001117950466647412,
|
|
"loss_nats": 1.1280839443206787,
|
|
"step": 5180,
|
|
"unclipped_grad_norm": 1.1194853782653809
|
|
},
|
|
{
|
|
"bits_per_byte": 1.567900990456851,
|
|
"learning_rate": 0.00011128262322741627,
|
|
"loss_nats": 1.086786150932312,
|
|
"step": 5190,
|
|
"unclipped_grad_norm": 1.0312306880950928
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5985697966719614,
|
|
"learning_rate": 0.00011077111767184913,
|
|
"loss_nats": 1.108044147491455,
|
|
"step": 5200,
|
|
"unclipped_grad_norm": 1.137611985206604
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6023676895509649,
|
|
"learning_rate": 0.00011026053873825837,
|
|
"loss_nats": 1.110676646232605,
|
|
"step": 5210,
|
|
"unclipped_grad_norm": 1.031370759010315
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6946726700781938,
|
|
"learning_rate": 0.0001097508951510293,
|
|
"loss_nats": 1.1746575832366943,
|
|
"step": 5220,
|
|
"unclipped_grad_norm": 1.113584280014038
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5914005278880945,
|
|
"learning_rate": 0.0001092421956185646,
|
|
"loss_nats": 1.1030747890472412,
|
|
"step": 5230,
|
|
"unclipped_grad_norm": 1.1469851732254028
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6053940682581977,
|
|
"learning_rate": 0.00010873444883313581,
|
|
"loss_nats": 1.11277437210083,
|
|
"step": 5240,
|
|
"unclipped_grad_norm": 1.0413556098937988
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5800591319769466,
|
|
"learning_rate": 0.00010822766347073447,
|
|
"loss_nats": 1.095213532447815,
|
|
"step": 5250,
|
|
"unclipped_grad_norm": 1.0926955938339233
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5918442431273205,
|
|
"learning_rate": 0.00010772184819092424,
|
|
"loss_nats": 1.1033823490142822,
|
|
"step": 5260,
|
|
"unclipped_grad_norm": 1.0704180002212524
|
|
},
|
|
{
|
|
"bits_per_byte": 1.589662299235824,
|
|
"learning_rate": 0.0001072170116366926,
|
|
"loss_nats": 1.1018699407577515,
|
|
"step": 5270,
|
|
"unclipped_grad_norm": 1.0476908683776855
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6108980290337602,
|
|
"learning_rate": 0.00010671316243430345,
|
|
"loss_nats": 1.1165894269943237,
|
|
"step": 5280,
|
|
"unclipped_grad_norm": 1.1081280708312988
|
|
},
|
|
{
|
|
"bits_per_byte": 1.608212003992585,
|
|
"learning_rate": 0.00010621030919314944,
|
|
"loss_nats": 1.1147276163101196,
|
|
"step": 5290,
|
|
"unclipped_grad_norm": 1.1043791770935059
|
|
},
|
|
{
|
|
"bits_per_byte": 1.614489714714379,
|
|
"learning_rate": 0.000105708460505605,
|
|
"loss_nats": 1.1190789937973022,
|
|
"step": 5300,
|
|
"unclipped_grad_norm": 1.0849814414978027
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6119760162893682,
|
|
"learning_rate": 0.00010520762494687964,
|
|
"loss_nats": 1.117336630821228,
|
|
"step": 5310,
|
|
"unclipped_grad_norm": 1.1325145959854126
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5916898027068456,
|
|
"learning_rate": 0.00010470781107487111,
|
|
"loss_nats": 1.1032752990722656,
|
|
"step": 5320,
|
|
"unclipped_grad_norm": 1.072629451751709
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6015805249579658,
|
|
"learning_rate": 0.00010420902743001955,
|
|
"loss_nats": 1.110131025314331,
|
|
"step": 5330,
|
|
"unclipped_grad_norm": 1.1142079830169678
|
|
},
|
|
{
|
|
"bits_per_byte": 1.553969879789011,
|
|
"learning_rate": 0.00010371128253516117,
|
|
"loss_nats": 1.07712984085083,
|
|
"step": 5340,
|
|
"unclipped_grad_norm": 1.0742131471633911
|
|
},
|
|
{
|
|
"bits_per_byte": 1.626766180298268,
|
|
"learning_rate": 0.000103214584895383,
|
|
"loss_nats": 1.1275883913040161,
|
|
"step": 5350,
|
|
"unclipped_grad_norm": 1.127328634262085
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5508274127924457,
|
|
"learning_rate": 0.00010271894299787715,
|
|
"loss_nats": 1.0749516487121582,
|
|
"step": 5360,
|
|
"unclipped_grad_norm": 1.1100437641143799
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5306884163590617,
|
|
"learning_rate": 0.00010222436531179622,
|
|
"loss_nats": 1.0609923601150513,
|
|
"step": 5370,
|
|
"unclipped_grad_norm": 1.0785573720932007
|
|
},
|
|
{
|
|
"bits_per_byte": 1.633070548330946,
|
|
"learning_rate": 0.0001017308602881082,
|
|
"loss_nats": 1.131958246231079,
|
|
"step": 5380,
|
|
"unclipped_grad_norm": 1.0940150022506714
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5790724675089465,
|
|
"learning_rate": 0.00010123843635945242,
|
|
"loss_nats": 1.094529628753662,
|
|
"step": 5390,
|
|
"unclipped_grad_norm": 1.1135237216949463
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6121983898569339,
|
|
"learning_rate": 0.00010074710193999505,
|
|
"loss_nats": 1.1174907684326172,
|
|
"step": 5400,
|
|
"unclipped_grad_norm": 1.1351183652877808
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6233261833157102,
|
|
"learning_rate": 0.0001002568654252858,
|
|
"loss_nats": 1.1252039670944214,
|
|
"step": 5410,
|
|
"unclipped_grad_norm": 1.1545859575271606
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5857118577455054,
|
|
"learning_rate": 9.976773519211404e-05,
|
|
"loss_nats": 1.09913170337677,
|
|
"step": 5420,
|
|
"unclipped_grad_norm": 1.1009039878845215
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6290494219711227,
|
|
"learning_rate": 9.92797195983659e-05,
|
|
"loss_nats": 1.1291710138320923,
|
|
"step": 5430,
|
|
"unclipped_grad_norm": 1.1000012159347534
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5753132707263873,
|
|
"learning_rate": 9.879282698288142e-05,
|
|
"loss_nats": 1.0919239521026611,
|
|
"step": 5440,
|
|
"unclipped_grad_norm": 1.0844659805297852
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5681787424379945,
|
|
"learning_rate": 9.830706566531197e-05,
|
|
"loss_nats": 1.0869786739349365,
|
|
"step": 5450,
|
|
"unclipped_grad_norm": 1.1008447408676147
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5540281819076536,
|
|
"learning_rate": 9.782244394597826e-05,
|
|
"loss_nats": 1.0771702527999878,
|
|
"step": 5460,
|
|
"unclipped_grad_norm": 1.077485203742981
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5997471898997682,
|
|
"learning_rate": 9.733897010572829e-05,
|
|
"loss_nats": 1.1088602542877197,
|
|
"step": 5470,
|
|
"unclipped_grad_norm": 1.0731958150863647
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6110182449067132,
|
|
"learning_rate": 9.685665240579612e-05,
|
|
"loss_nats": 1.1166727542877197,
|
|
"step": 5480,
|
|
"unclipped_grad_norm": 1.098453164100647
|
|
},
|
|
{
|
|
"bits_per_byte": 1.627262694211309,
|
|
"learning_rate": 9.637549908766041e-05,
|
|
"loss_nats": 1.1279325485229492,
|
|
"step": 5490,
|
|
"unclipped_grad_norm": 1.2330873012542725
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5907870657724668,
|
|
"learning_rate": 9.589551837290389e-05,
|
|
"loss_nats": 1.1026495695114136,
|
|
"step": 5500,
|
|
"unclipped_grad_norm": 1.1288633346557617
|
|
},
|
|
{
|
|
"bits_per_byte": 1.677719824234694,
|
|
"learning_rate": 9.541671846307257e-05,
|
|
"loss_nats": 1.1629067659378052,
|
|
"step": 5510,
|
|
"unclipped_grad_norm": 1.1779037714004517
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6037174093949362,
|
|
"learning_rate": 9.493910753953597e-05,
|
|
"loss_nats": 1.1116122007369995,
|
|
"step": 5520,
|
|
"unclipped_grad_norm": 1.07362699508667
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5749782485225066,
|
|
"learning_rate": 9.446269376334689e-05,
|
|
"loss_nats": 1.0916917324066162,
|
|
"step": 5530,
|
|
"unclipped_grad_norm": 1.0827255249023438
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6475040203219564,
|
|
"learning_rate": 9.39874852751024e-05,
|
|
"loss_nats": 1.1419627666473389,
|
|
"step": 5540,
|
|
"unclipped_grad_norm": 1.2362996339797974
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5801301608117528,
|
|
"learning_rate": 9.351349019480433e-05,
|
|
"loss_nats": 1.0952627658843994,
|
|
"step": 5550,
|
|
"unclipped_grad_norm": 1.0816401243209839
|
|
},
|
|
{
|
|
"bits_per_byte": 1.566390638816974,
|
|
"learning_rate": 9.304071662172086e-05,
|
|
"loss_nats": 1.085739254951477,
|
|
"step": 5560,
|
|
"unclipped_grad_norm": 1.1727906465530396
|
|
},
|
|
{
|
|
"bits_per_byte": 1.545297310653952,
|
|
"learning_rate": 9.256917263424787e-05,
|
|
"loss_nats": 1.0711184740066528,
|
|
"step": 5570,
|
|
"unclipped_grad_norm": 1.096352458000183
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6197793851069664,
|
|
"learning_rate": 9.209886628977114e-05,
|
|
"loss_nats": 1.1227455139160156,
|
|
"step": 5580,
|
|
"unclipped_grad_norm": 1.098443865776062
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5488499562728253,
|
|
"learning_rate": 9.162980562452843e-05,
|
|
"loss_nats": 1.0735809803009033,
|
|
"step": 5590,
|
|
"unclipped_grad_norm": 1.106048822402954
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6060310919969936,
|
|
"learning_rate": 9.116199865347238e-05,
|
|
"loss_nats": 1.1132159233093262,
|
|
"step": 5600,
|
|
"unclipped_grad_norm": 1.1421784162521362
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6012358717256368,
|
|
"learning_rate": 9.069545337013333e-05,
|
|
"loss_nats": 1.1098921298980713,
|
|
"step": 5610,
|
|
"unclipped_grad_norm": 1.0477343797683716
|
|
},
|
|
{
|
|
"bits_per_byte": 1.612595669780427,
|
|
"learning_rate": 9.0230177746483e-05,
|
|
"loss_nats": 1.1177661418914795,
|
|
"step": 5620,
|
|
"unclipped_grad_norm": 1.0961613655090332
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5887812321154537,
|
|
"learning_rate": 8.976617973279797e-05,
|
|
"loss_nats": 1.1012592315673828,
|
|
"step": 5630,
|
|
"unclipped_grad_norm": 1.0803630352020264
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5675833385007074,
|
|
"learning_rate": 8.930346725752419e-05,
|
|
"loss_nats": 1.0865659713745117,
|
|
"step": 5640,
|
|
"unclipped_grad_norm": 1.0926858186721802
|
|
},
|
|
{
|
|
"bits_per_byte": 1.598304599424331,
|
|
"learning_rate": 8.88420482271411e-05,
|
|
"loss_nats": 1.1078603267669678,
|
|
"step": 5650,
|
|
"unclipped_grad_norm": 1.1493103504180908
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6562716958629424,
|
|
"learning_rate": 8.838193052602691e-05,
|
|
"loss_nats": 1.1480400562286377,
|
|
"step": 5660,
|
|
"unclipped_grad_norm": 1.1589452028274536
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5621947780838739,
|
|
"learning_rate": 8.792312201632352e-05,
|
|
"loss_nats": 1.0828309059143066,
|
|
"step": 5670,
|
|
"unclipped_grad_norm": 1.118128776550293
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5797031278896605,
|
|
"learning_rate": 8.746563053780261e-05,
|
|
"loss_nats": 1.0949667692184448,
|
|
"step": 5680,
|
|
"unclipped_grad_norm": 1.079695701599121
|
|
},
|
|
{
|
|
"bits_per_byte": 1.650425317629512,
|
|
"learning_rate": 8.700946390773114e-05,
|
|
"loss_nats": 1.1439876556396484,
|
|
"step": 5690,
|
|
"unclipped_grad_norm": 1.1217325925827026
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5713879387030945,
|
|
"learning_rate": 8.655462992073833e-05,
|
|
"loss_nats": 1.089203119277954,
|
|
"step": 5700,
|
|
"unclipped_grad_norm": 1.0890491008758545
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5792038622542057,
|
|
"learning_rate": 8.610113634868199e-05,
|
|
"loss_nats": 1.094620704650879,
|
|
"step": 5710,
|
|
"unclipped_grad_norm": 1.0754131078720093
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6025209260928837,
|
|
"learning_rate": 8.564899094051613e-05,
|
|
"loss_nats": 1.1107828617095947,
|
|
"step": 5720,
|
|
"unclipped_grad_norm": 1.0796459913253784
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5944050647986676,
|
|
"learning_rate": 8.519820142215816e-05,
|
|
"loss_nats": 1.1051573753356934,
|
|
"step": 5730,
|
|
"unclipped_grad_norm": 1.0518229007720947
|
|
},
|
|
{
|
|
"bits_per_byte": 1.599884088089855,
|
|
"learning_rate": 8.474877549635728e-05,
|
|
"loss_nats": 1.1089551448822021,
|
|
"step": 5740,
|
|
"unclipped_grad_norm": 1.2038764953613281
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5972181850188307,
|
|
"learning_rate": 8.430072084256247e-05,
|
|
"loss_nats": 1.1071072816848755,
|
|
"step": 5750,
|
|
"unclipped_grad_norm": 1.1661580801010132
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6206038699352028,
|
|
"learning_rate": 8.385404511679161e-05,
|
|
"loss_nats": 1.123317003250122,
|
|
"step": 5760,
|
|
"unclipped_grad_norm": 1.1536771059036255
|
|
},
|
|
{
|
|
"bits_per_byte": 1.569340313261922,
|
|
"learning_rate": 8.340875595150034e-05,
|
|
"loss_nats": 1.0877838134765625,
|
|
"step": 5770,
|
|
"unclipped_grad_norm": 1.1234960556030273
|
|
},
|
|
{
|
|
"bits_per_byte": 1.548520437513772,
|
|
"learning_rate": 8.296486095545203e-05,
|
|
"loss_nats": 1.073352575302124,
|
|
"step": 5780,
|
|
"unclipped_grad_norm": 1.0824602842330933
|
|
},
|
|
{
|
|
"bits_per_byte": 1.575600653735979,
|
|
"learning_rate": 8.252236771358729e-05,
|
|
"loss_nats": 1.0921231508255005,
|
|
"step": 5790,
|
|
"unclipped_grad_norm": 1.0697625875473022
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5474534571478191,
|
|
"learning_rate": 8.208128378689484e-05,
|
|
"loss_nats": 1.072613000869751,
|
|
"step": 5800,
|
|
"unclipped_grad_norm": 1.121078372001648
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6281222635003212,
|
|
"learning_rate": 8.16416167122819e-05,
|
|
"loss_nats": 1.128528356552124,
|
|
"step": 5810,
|
|
"unclipped_grad_norm": 1.163888931274414
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5571622217543264,
|
|
"learning_rate": 8.120337400244578e-05,
|
|
"loss_nats": 1.0793426036834717,
|
|
"step": 5820,
|
|
"unclipped_grad_norm": 1.0646225214004517
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5715148618994312,
|
|
"learning_rate": 8.076656314574521e-05,
|
|
"loss_nats": 1.0892910957336426,
|
|
"step": 5830,
|
|
"unclipped_grad_norm": 1.1465011835098267
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5656843060698804,
|
|
"learning_rate": 8.03311916060725e-05,
|
|
"loss_nats": 1.085249662399292,
|
|
"step": 5840,
|
|
"unclipped_grad_norm": 1.0994694232940674
|
|
},
|
|
{
|
|
"bits_per_byte": 1.529561070082656,
|
|
"learning_rate": 7.989726682272606e-05,
|
|
"loss_nats": 1.060210943222046,
|
|
"step": 5850,
|
|
"unclipped_grad_norm": 1.123828649520874
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5867750544931392,
|
|
"learning_rate": 7.946479621028318e-05,
|
|
"loss_nats": 1.099868655204773,
|
|
"step": 5860,
|
|
"unclipped_grad_norm": 1.1375004053115845
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5937161022993112,
|
|
"learning_rate": 7.903378715847343e-05,
|
|
"loss_nats": 1.104679822921753,
|
|
"step": 5870,
|
|
"unclipped_grad_norm": 1.0862858295440674
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6139739387444414,
|
|
"learning_rate": 7.860424703205224e-05,
|
|
"loss_nats": 1.1187214851379395,
|
|
"step": 5880,
|
|
"unclipped_grad_norm": 1.1279133558273315
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6231863614205586,
|
|
"learning_rate": 7.817618317067534e-05,
|
|
"loss_nats": 1.1251070499420166,
|
|
"step": 5890,
|
|
"unclipped_grad_norm": 1.1251599788665771
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6124319422968056,
|
|
"learning_rate": 7.774960288877295e-05,
|
|
"loss_nats": 1.1176526546478271,
|
|
"step": 5900,
|
|
"unclipped_grad_norm": 1.1988507509231567
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6244132856518139,
|
|
"learning_rate": 7.732451347542522e-05,
|
|
"loss_nats": 1.1259574890136719,
|
|
"step": 5910,
|
|
"unclipped_grad_norm": 1.1206265687942505
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5302899325570125,
|
|
"learning_rate": 7.690092219423722e-05,
|
|
"loss_nats": 1.060716152191162,
|
|
"step": 5920,
|
|
"unclipped_grad_norm": 1.1038360595703125
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6585005910181243,
|
|
"learning_rate": 7.647883628321536e-05,
|
|
"loss_nats": 1.1495850086212158,
|
|
"step": 5930,
|
|
"unclipped_grad_norm": 1.2028963565826416
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5036174872000014,
|
|
"learning_rate": 7.605826295464316e-05,
|
|
"loss_nats": 1.0422282218933105,
|
|
"step": 5940,
|
|
"unclipped_grad_norm": 1.0828943252563477
|
|
},
|
|
{
|
|
"bits_per_byte": 1.564500033536039,
|
|
"learning_rate": 7.563920939495851e-05,
|
|
"loss_nats": 1.0844287872314453,
|
|
"step": 5950,
|
|
"unclipped_grad_norm": 1.1633635759353638
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6026038217305996,
|
|
"learning_rate": 7.522168276463047e-05,
|
|
"loss_nats": 1.1108403205871582,
|
|
"step": 5960,
|
|
"unclipped_grad_norm": 1.1734113693237305
|
|
},
|
|
{
|
|
"bits_per_byte": 1.582973034030515,
|
|
"learning_rate": 7.480569019803719e-05,
|
|
"loss_nats": 1.0972332954406738,
|
|
"step": 5970,
|
|
"unclipped_grad_norm": 1.255017638206482
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6250308753510623,
|
|
"learning_rate": 7.439123880334388e-05,
|
|
"loss_nats": 1.1263855695724487,
|
|
"step": 5980,
|
|
"unclipped_grad_norm": 1.1516860723495483
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5794843659577629,
|
|
"learning_rate": 7.39783356623813e-05,
|
|
"loss_nats": 1.0948151350021362,
|
|
"step": 5990,
|
|
"unclipped_grad_norm": 1.1278661489486694
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5522483334538744,
|
|
"learning_rate": 7.356698783052497e-05,
|
|
"loss_nats": 1.0759365558624268,
|
|
"step": 6000,
|
|
"unclipped_grad_norm": 1.067203402519226
|
|
},
|
|
{
|
|
"bits_per_byte": 1.577093635128119,
|
|
"learning_rate": 7.315720233657431e-05,
|
|
"loss_nats": 1.0931580066680908,
|
|
"step": 6010,
|
|
"unclipped_grad_norm": 1.1185256242752075
|
|
},
|
|
{
|
|
"bits_per_byte": 1.534402725669746,
|
|
"learning_rate": 7.274898618263286e-05,
|
|
"loss_nats": 1.0635669231414795,
|
|
"step": 6020,
|
|
"unclipped_grad_norm": 1.1662800312042236
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5661140907143865,
|
|
"learning_rate": 7.234234634398831e-05,
|
|
"loss_nats": 1.0855475664138794,
|
|
"step": 6030,
|
|
"unclipped_grad_norm": 1.1327592134475708
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6326177180112242,
|
|
"learning_rate": 7.193728976899362e-05,
|
|
"loss_nats": 1.131644368171692,
|
|
"step": 6040,
|
|
"unclipped_grad_norm": 1.1918420791625977
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5457630396724886,
|
|
"learning_rate": 7.153382337894797e-05,
|
|
"loss_nats": 1.0714412927627563,
|
|
"step": 6050,
|
|
"unclipped_grad_norm": 1.147464394569397
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5419047808830324,
|
|
"learning_rate": 7.113195406797887e-05,
|
|
"loss_nats": 1.0687669515609741,
|
|
"step": 6060,
|
|
"unclipped_grad_norm": 1.1692534685134888
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6009338701907216,
|
|
"learning_rate": 7.073168870292397e-05,
|
|
"loss_nats": 1.1096827983856201,
|
|
"step": 6070,
|
|
"unclipped_grad_norm": 1.1858196258544922
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5324048032146727,
|
|
"learning_rate": 7.033303412321404e-05,
|
|
"loss_nats": 1.062182068824768,
|
|
"step": 6080,
|
|
"unclipped_grad_norm": 1.1669044494628906
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5495642002218584,
|
|
"learning_rate": 6.993599714075586e-05,
|
|
"loss_nats": 1.0740760564804077,
|
|
"step": 6090,
|
|
"unclipped_grad_norm": 1.0681768655776978
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5432428059067451,
|
|
"learning_rate": 6.954058453981609e-05,
|
|
"loss_nats": 1.0696943998336792,
|
|
"step": 6100,
|
|
"unclipped_grad_norm": 1.086612582206726
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6119146184830102,
|
|
"learning_rate": 6.914680307690504e-05,
|
|
"loss_nats": 1.1172940731048584,
|
|
"step": 6110,
|
|
"unclipped_grad_norm": 1.1307899951934814
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6174417969164392,
|
|
"learning_rate": 6.875465948066139e-05,
|
|
"loss_nats": 1.1211252212524414,
|
|
"step": 6120,
|
|
"unclipped_grad_norm": 1.2087455987930298
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5350297744147916,
|
|
"learning_rate": 6.836416045173733e-05,
|
|
"loss_nats": 1.0640015602111816,
|
|
"step": 6130,
|
|
"unclipped_grad_norm": 1.1170084476470947
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5441861307467275,
|
|
"learning_rate": 6.797531266268377e-05,
|
|
"loss_nats": 1.0703482627868652,
|
|
"step": 6140,
|
|
"unclipped_grad_norm": 1.191177487373352
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6015168913771465,
|
|
"learning_rate": 6.758812275783663e-05,
|
|
"loss_nats": 1.1100869178771973,
|
|
"step": 6150,
|
|
"unclipped_grad_norm": 1.1631611585617065
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4840940166740553,
|
|
"learning_rate": 6.720259735320298e-05,
|
|
"loss_nats": 1.0286955833435059,
|
|
"step": 6160,
|
|
"unclipped_grad_norm": 1.0670503377914429
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5765931656141083,
|
|
"learning_rate": 6.681874303634842e-05,
|
|
"loss_nats": 1.092811107635498,
|
|
"step": 6170,
|
|
"unclipped_grad_norm": 1.12460458278656
|
|
},
|
|
{
|
|
"bits_per_byte": 1.531391309453138,
|
|
"learning_rate": 6.643656636628402e-05,
|
|
"loss_nats": 1.0614795684814453,
|
|
"step": 6180,
|
|
"unclipped_grad_norm": 1.1588267087936401
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5676526475090051,
|
|
"learning_rate": 6.605607387335466e-05,
|
|
"loss_nats": 1.0866140127182007,
|
|
"step": 6190,
|
|
"unclipped_grad_norm": 1.2090874910354614
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5723472579296074,
|
|
"learning_rate": 6.567727205912724e-05,
|
|
"loss_nats": 1.0898680686950684,
|
|
"step": 6200,
|
|
"unclipped_grad_norm": 1.1369835138320923
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5728810920778855,
|
|
"learning_rate": 6.530016739627959e-05,
|
|
"loss_nats": 1.090238094329834,
|
|
"step": 6210,
|
|
"unclipped_grad_norm": 1.2529284954071045
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5681563846933824,
|
|
"learning_rate": 6.492476632848993e-05,
|
|
"loss_nats": 1.086963176727295,
|
|
"step": 6220,
|
|
"unclipped_grad_norm": 1.0888316631317139
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5207944264375752,
|
|
"learning_rate": 6.455107527032678e-05,
|
|
"loss_nats": 1.0541343688964844,
|
|
"step": 6230,
|
|
"unclipped_grad_norm": 1.15282142162323
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5617125387308546,
|
|
"learning_rate": 6.417910060713926e-05,
|
|
"loss_nats": 1.0824966430664062,
|
|
"step": 6240,
|
|
"unclipped_grad_norm": 1.227396845817566
|
|
},
|
|
{
|
|
"bits_per_byte": 1.595514008931431,
|
|
"learning_rate": 6.380884869494799e-05,
|
|
"loss_nats": 1.1059260368347168,
|
|
"step": 6250,
|
|
"unclipped_grad_norm": 1.1354495286941528
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4806825688115406,
|
|
"learning_rate": 6.34403258603367e-05,
|
|
"loss_nats": 1.0263309478759766,
|
|
"step": 6260,
|
|
"unclipped_grad_norm": 1.1806098222732544
|
|
},
|
|
{
|
|
"bits_per_byte": 1.561176984756068,
|
|
"learning_rate": 6.30735384003437e-05,
|
|
"loss_nats": 1.0821254253387451,
|
|
"step": 6270,
|
|
"unclipped_grad_norm": 1.1072839498519897
|
|
},
|
|
{
|
|
"bits_per_byte": 1.557884892853252,
|
|
"learning_rate": 6.270849258235484e-05,
|
|
"loss_nats": 1.079843521118164,
|
|
"step": 6280,
|
|
"unclipped_grad_norm": 1.16572904586792
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5084749651909706,
|
|
"learning_rate": 6.234519464399581e-05,
|
|
"loss_nats": 1.0455951690673828,
|
|
"step": 6290,
|
|
"unclipped_grad_norm": 1.1536295413970947
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5632667459467018,
|
|
"learning_rate": 6.198365079302614e-05,
|
|
"loss_nats": 1.0835739374160767,
|
|
"step": 6300,
|
|
"unclipped_grad_norm": 1.1973258256912231
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5420322200273218,
|
|
"learning_rate": 6.16238672072326e-05,
|
|
"loss_nats": 1.0688552856445312,
|
|
"step": 6310,
|
|
"unclipped_grad_norm": 1.146715760231018
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4920311879940247,
|
|
"learning_rate": 6.126585003432406e-05,
|
|
"loss_nats": 1.034197211265564,
|
|
"step": 6320,
|
|
"unclipped_grad_norm": 1.1094690561294556
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5741831727275677,
|
|
"learning_rate": 6.090960539182614e-05,
|
|
"loss_nats": 1.091140627861023,
|
|
"step": 6330,
|
|
"unclipped_grad_norm": 1.2368261814117432
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5350222071781536,
|
|
"learning_rate": 6.0555139366976935e-05,
|
|
"loss_nats": 1.0639963150024414,
|
|
"step": 6340,
|
|
"unclipped_grad_norm": 1.105275273323059
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6462146663884378,
|
|
"learning_rate": 6.020245801662274e-05,
|
|
"loss_nats": 1.1410690546035767,
|
|
"step": 6350,
|
|
"unclipped_grad_norm": 1.3086775541305542
|
|
},
|
|
{
|
|
"bits_per_byte": 1.52795922367252,
|
|
"learning_rate": 5.985156736711483e-05,
|
|
"loss_nats": 1.05910062789917,
|
|
"step": 6360,
|
|
"unclipped_grad_norm": 1.1954522132873535
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5079418189732958,
|
|
"learning_rate": 5.950247341420618e-05,
|
|
"loss_nats": 1.0452256202697754,
|
|
"step": 6370,
|
|
"unclipped_grad_norm": 1.1107361316680908
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5318044117805105,
|
|
"learning_rate": 5.915518212294935e-05,
|
|
"loss_nats": 1.0617659091949463,
|
|
"step": 6380,
|
|
"unclipped_grad_norm": 1.0837970972061157
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5828646849604715,
|
|
"learning_rate": 5.880969942759425e-05,
|
|
"loss_nats": 1.0971581935882568,
|
|
"step": 6390,
|
|
"unclipped_grad_norm": 1.1240276098251343
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5799186221511916,
|
|
"learning_rate": 5.846603123148688e-05,
|
|
"loss_nats": 1.095116138458252,
|
|
"step": 6400,
|
|
"unclipped_grad_norm": 1.1799321174621582
|
|
},
|
|
{
|
|
"bits_per_byte": 1.552409309215082,
|
|
"learning_rate": 5.812418340696858e-05,
|
|
"loss_nats": 1.0760481357574463,
|
|
"step": 6410,
|
|
"unclipped_grad_norm": 1.1518590450286865
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5442817531006072,
|
|
"learning_rate": 5.778416179527541e-05,
|
|
"loss_nats": 1.0704145431518555,
|
|
"step": 6420,
|
|
"unclipped_grad_norm": 1.1955360174179077
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5805626971786728,
|
|
"learning_rate": 5.7445972206438635e-05,
|
|
"loss_nats": 1.0955625772476196,
|
|
"step": 6430,
|
|
"unclipped_grad_norm": 1.156053066253662
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5364046037257888,
|
|
"learning_rate": 5.710962041918516e-05,
|
|
"loss_nats": 1.0649545192718506,
|
|
"step": 6440,
|
|
"unclipped_grad_norm": 1.1582752466201782
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5944274225432797,
|
|
"learning_rate": 5.677511218083908e-05,
|
|
"loss_nats": 1.105172872543335,
|
|
"step": 6450,
|
|
"unclipped_grad_norm": 1.2264126539230347
|
|
},
|
|
{
|
|
"bits_per_byte": 1.645741886081216,
|
|
"learning_rate": 5.644245320722316e-05,
|
|
"loss_nats": 1.1407413482666016,
|
|
"step": 6460,
|
|
"unclipped_grad_norm": 1.2444238662719727
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6345733327341851,
|
|
"learning_rate": 5.61116491825615e-05,
|
|
"loss_nats": 1.1329998970031738,
|
|
"step": 6470,
|
|
"unclipped_grad_norm": 1.2292388677597046
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5111716531564994,
|
|
"learning_rate": 5.5782705759382104e-05,
|
|
"loss_nats": 1.047464370727539,
|
|
"step": 6480,
|
|
"unclipped_grad_norm": 1.2222721576690674
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6197842006211907,
|
|
"learning_rate": 5.5455628558420546e-05,
|
|
"loss_nats": 1.122748851776123,
|
|
"step": 6490,
|
|
"unclipped_grad_norm": 1.2203607559204102
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5564085937782457,
|
|
"learning_rate": 5.513042316852368e-05,
|
|
"loss_nats": 1.0788202285766602,
|
|
"step": 6500,
|
|
"unclipped_grad_norm": 1.1637120246887207
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5288421826020495,
|
|
"learning_rate": 5.480709514655443e-05,
|
|
"loss_nats": 1.0597126483917236,
|
|
"step": 6510,
|
|
"unclipped_grad_norm": 1.1861555576324463
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5068566084463515,
|
|
"learning_rate": 5.448565001729653e-05,
|
|
"loss_nats": 1.04447340965271,
|
|
"step": 6520,
|
|
"unclipped_grad_norm": 1.1399540901184082
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5271775625243484,
|
|
"learning_rate": 5.41660932733604e-05,
|
|
"loss_nats": 1.0585588216781616,
|
|
"step": 6530,
|
|
"unclipped_grad_norm": 1.200762152671814
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5654438743239742,
|
|
"learning_rate": 5.384843037508902e-05,
|
|
"loss_nats": 1.0850830078125,
|
|
"step": 6540,
|
|
"unclipped_grad_norm": 1.1505160331726074
|
|
},
|
|
{
|
|
"bits_per_byte": 1.505103417303456,
|
|
"learning_rate": 5.353266675046498e-05,
|
|
"loss_nats": 1.0432581901550293,
|
|
"step": 6550,
|
|
"unclipped_grad_norm": 1.1259007453918457
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5822820077393482,
|
|
"learning_rate": 5.321880779501729e-05,
|
|
"loss_nats": 1.0967543125152588,
|
|
"step": 6560,
|
|
"unclipped_grad_norm": 1.1489042043685913
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5307443107205922,
|
|
"learning_rate": 5.290685887172962e-05,
|
|
"loss_nats": 1.0610311031341553,
|
|
"step": 6570,
|
|
"unclipped_grad_norm": 1.2110720872879028
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5820749406277095,
|
|
"learning_rate": 5.2596825310948295e-05,
|
|
"loss_nats": 1.0966107845306396,
|
|
"step": 6580,
|
|
"unclipped_grad_norm": 1.2832127809524536
|
|
},
|
|
{
|
|
"bits_per_byte": 1.555234640203456,
|
|
"learning_rate": 5.228871241029154e-05,
|
|
"loss_nats": 1.0780065059661865,
|
|
"step": 6590,
|
|
"unclipped_grad_norm": 1.1875557899475098
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4893780116391642,
|
|
"learning_rate": 5.198252543455865e-05,
|
|
"loss_nats": 1.032358169555664,
|
|
"step": 6600,
|
|
"unclipped_grad_norm": 1.1434928178787231
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4980550523231455,
|
|
"learning_rate": 5.167826961564027e-05,
|
|
"loss_nats": 1.0383726358413696,
|
|
"step": 6610,
|
|
"unclipped_grad_norm": 1.1379001140594482
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4935792038344409,
|
|
"learning_rate": 5.137595015242884e-05,
|
|
"loss_nats": 1.0352702140808105,
|
|
"step": 6620,
|
|
"unclipped_grad_norm": 1.1951154470443726
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5329076804857955,
|
|
"learning_rate": 5.1075572210729925e-05,
|
|
"loss_nats": 1.0625306367874146,
|
|
"step": 6630,
|
|
"unclipped_grad_norm": 1.144928216934204
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5874349519244997,
|
|
"learning_rate": 5.07771409231737e-05,
|
|
"loss_nats": 1.1003260612487793,
|
|
"step": 6640,
|
|
"unclipped_grad_norm": 1.2067164182662964
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5760562357781147,
|
|
"learning_rate": 5.048066138912756e-05,
|
|
"loss_nats": 1.0924389362335205,
|
|
"step": 6650,
|
|
"unclipped_grad_norm": 1.1507362127304077
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5807757836830918,
|
|
"learning_rate": 5.018613867460863e-05,
|
|
"loss_nats": 1.095710277557373,
|
|
"step": 6660,
|
|
"unclipped_grad_norm": 1.134096622467041
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5557022610311522,
|
|
"learning_rate": 4.989357781219755e-05,
|
|
"loss_nats": 1.078330636024475,
|
|
"step": 6670,
|
|
"unclipped_grad_norm": 1.2659887075424194
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5265064862206816,
|
|
"learning_rate": 4.960298380095222e-05,
|
|
"loss_nats": 1.0580936670303345,
|
|
"step": 6680,
|
|
"unclipped_grad_norm": 1.196817398071289
|
|
},
|
|
{
|
|
"bits_per_byte": 1.558342366704547,
|
|
"learning_rate": 4.931436160632256e-05,
|
|
"loss_nats": 1.0801606178283691,
|
|
"step": 6690,
|
|
"unclipped_grad_norm": 1.191389799118042
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6202516494662358,
|
|
"learning_rate": 4.9027716160065485e-05,
|
|
"loss_nats": 1.123072862625122,
|
|
"step": 6700,
|
|
"unclipped_grad_norm": 1.2432118654251099
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5747254340257382,
|
|
"learning_rate": 4.874305236016084e-05,
|
|
"loss_nats": 1.0915164947509766,
|
|
"step": 6710,
|
|
"unclipped_grad_norm": 1.1514227390289307
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5896156919374402,
|
|
"learning_rate": 4.846037507072752e-05,
|
|
"loss_nats": 1.1018376350402832,
|
|
"step": 6720,
|
|
"unclipped_grad_norm": 1.1839321851730347
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5521088555240246,
|
|
"learning_rate": 4.817968912194049e-05,
|
|
"loss_nats": 1.075839877128601,
|
|
"step": 6730,
|
|
"unclipped_grad_norm": 1.2353639602661133
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4976266435398462,
|
|
"learning_rate": 4.790099930994813e-05,
|
|
"loss_nats": 1.0380756855010986,
|
|
"step": 6740,
|
|
"unclipped_grad_norm": 1.1758193969726562
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5596673210467942,
|
|
"learning_rate": 4.762431039679047e-05,
|
|
"loss_nats": 1.0810790061950684,
|
|
"step": 6750,
|
|
"unclipped_grad_norm": 1.1156134605407715
|
|
},
|
|
{
|
|
"bits_per_byte": 1.536689063030316,
|
|
"learning_rate": 4.734962711031755e-05,
|
|
"loss_nats": 1.0651516914367676,
|
|
"step": 6760,
|
|
"unclipped_grad_norm": 1.2122220993041992
|
|
},
|
|
{
|
|
"bits_per_byte": 1.443431470599958,
|
|
"learning_rate": 4.707695414410892e-05,
|
|
"loss_nats": 1.0005104541778564,
|
|
"step": 6770,
|
|
"unclipped_grad_norm": 1.1533452272415161
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5402759331967106,
|
|
"learning_rate": 4.680629615739322e-05,
|
|
"loss_nats": 1.0676379203796387,
|
|
"step": 6780,
|
|
"unclipped_grad_norm": 1.1691333055496216
|
|
},
|
|
{
|
|
"bits_per_byte": 1.523325667092974,
|
|
"learning_rate": 4.653765777496863e-05,
|
|
"loss_nats": 1.0558888912200928,
|
|
"step": 6790,
|
|
"unclipped_grad_norm": 1.2405608892440796
|
|
},
|
|
{
|
|
"bits_per_byte": 1.57861671348416,
|
|
"learning_rate": 4.6271043587123986e-05,
|
|
"loss_nats": 1.0942137241363525,
|
|
"step": 6800,
|
|
"unclipped_grad_norm": 1.2281029224395752
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5738859867068768,
|
|
"learning_rate": 4.600645814956006e-05,
|
|
"loss_nats": 1.0909346342086792,
|
|
"step": 6810,
|
|
"unclipped_grad_norm": 1.2371455430984497
|
|
},
|
|
{
|
|
"bits_per_byte": 1.582858837550342,
|
|
"learning_rate": 4.574390598331204e-05,
|
|
"loss_nats": 1.097154140472412,
|
|
"step": 6820,
|
|
"unclipped_grad_norm": 1.3370100259780884
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5273318309621724,
|
|
"learning_rate": 4.548339157467199e-05,
|
|
"loss_nats": 1.0586657524108887,
|
|
"step": 6830,
|
|
"unclipped_grad_norm": 1.196358561515808
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4954078953610652,
|
|
"learning_rate": 4.52249193751124e-05,
|
|
"loss_nats": 1.036537766456604,
|
|
"step": 6840,
|
|
"unclipped_grad_norm": 1.1750952005386353
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5476113372213112,
|
|
"learning_rate": 4.4968493801209985e-05,
|
|
"loss_nats": 1.0727224349975586,
|
|
"step": 6850,
|
|
"unclipped_grad_norm": 1.1924985647201538
|
|
},
|
|
{
|
|
"bits_per_byte": 1.526412067745358,
|
|
"learning_rate": 4.471411923457034e-05,
|
|
"loss_nats": 1.058028221130371,
|
|
"step": 6860,
|
|
"unclipped_grad_norm": 1.1877104043960571
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4832949852781467,
|
|
"learning_rate": 4.446180002175291e-05,
|
|
"loss_nats": 1.028141736984253,
|
|
"step": 6870,
|
|
"unclipped_grad_norm": 1.175843596458435
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4908706490660026,
|
|
"learning_rate": 4.421154047419693e-05,
|
|
"loss_nats": 1.0333927869796753,
|
|
"step": 6880,
|
|
"unclipped_grad_norm": 1.1246671676635742
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5010700801754213,
|
|
"learning_rate": 4.396334486814753e-05,
|
|
"loss_nats": 1.0404624938964844,
|
|
"step": 6890,
|
|
"unclipped_grad_norm": 1.1924952268600464
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6045640799851337,
|
|
"learning_rate": 4.371721744458285e-05,
|
|
"loss_nats": 1.112199068069458,
|
|
"step": 6900,
|
|
"unclipped_grad_norm": 1.2239227294921875
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5736668808096776,
|
|
"learning_rate": 4.347316240914145e-05,
|
|
"loss_nats": 1.0907827615737915,
|
|
"step": 6910,
|
|
"unclipped_grad_norm": 1.2664575576782227
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5080825007817016,
|
|
"learning_rate": 4.3231183932050546e-05,
|
|
"loss_nats": 1.045323133468628,
|
|
"step": 6920,
|
|
"unclipped_grad_norm": 1.1531251668930054
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5085007625886,
|
|
"learning_rate": 4.299128614805464e-05,
|
|
"loss_nats": 1.0456130504608154,
|
|
"step": 6930,
|
|
"unclipped_grad_norm": 1.2472591400146484
|
|
},
|
|
{
|
|
"bits_per_byte": 1.484484417291514,
|
|
"learning_rate": 4.2753473156344965e-05,
|
|
"loss_nats": 1.0289661884307861,
|
|
"step": 6940,
|
|
"unclipped_grad_norm": 1.187600016593933
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4929882714460763,
|
|
"learning_rate": 4.251774902048942e-05,
|
|
"loss_nats": 1.034860610961914,
|
|
"step": 6950,
|
|
"unclipped_grad_norm": 1.228901743888855
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5665772399931601,
|
|
"learning_rate": 4.228411776836306e-05,
|
|
"loss_nats": 1.0858685970306396,
|
|
"step": 6960,
|
|
"unclipped_grad_norm": 1.205407738685608
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5512240047853354,
|
|
"learning_rate": 4.2052583392079427e-05,
|
|
"loss_nats": 1.0752265453338623,
|
|
"step": 6970,
|
|
"unclipped_grad_norm": 1.2078452110290527
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5256931802647515,
|
|
"learning_rate": 4.182314984792214e-05,
|
|
"loss_nats": 1.0575299263000488,
|
|
"step": 6980,
|
|
"unclipped_grad_norm": 1.216407060623169
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5495781308165784,
|
|
"learning_rate": 4.159582105627748e-05,
|
|
"loss_nats": 1.0740857124328613,
|
|
"step": 6990,
|
|
"unclipped_grad_norm": 1.2360340356826782
|
|
},
|
|
{
|
|
"bits_per_byte": 1.579722561929208,
|
|
"learning_rate": 4.137060090156724e-05,
|
|
"loss_nats": 1.094980239868164,
|
|
"step": 7000,
|
|
"unclipped_grad_norm": 1.2281304597854614
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5775139607268278,
|
|
"learning_rate": 4.114749323218249e-05,
|
|
"loss_nats": 1.093449354171753,
|
|
"step": 7010,
|
|
"unclipped_grad_norm": 1.238048791885376
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5315163408403152,
|
|
"learning_rate": 4.0926501860417706e-05,
|
|
"loss_nats": 1.0615662336349487,
|
|
"step": 7020,
|
|
"unclipped_grad_norm": 1.2856208086013794
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5716388613907033,
|
|
"learning_rate": 4.070763056240571e-05,
|
|
"loss_nats": 1.0893770456314087,
|
|
"step": 7030,
|
|
"unclipped_grad_norm": 1.228158950805664
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5737668027298288,
|
|
"learning_rate": 4.049088307805306e-05,
|
|
"loss_nats": 1.0908520221710205,
|
|
"step": 7040,
|
|
"unclipped_grad_norm": 1.227662444114685
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5448513596402649,
|
|
"learning_rate": 4.027626311097629e-05,
|
|
"loss_nats": 1.0708093643188477,
|
|
"step": 7050,
|
|
"unclipped_grad_norm": 1.2028948068618774
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5624737339435735,
|
|
"learning_rate": 4.006377432843846e-05,
|
|
"loss_nats": 1.083024263381958,
|
|
"step": 7060,
|
|
"unclipped_grad_norm": 1.259883999824524
|
|
},
|
|
{
|
|
"bits_per_byte": 1.539107139101447,
|
|
"learning_rate": 3.985342036128656e-05,
|
|
"loss_nats": 1.0668277740478516,
|
|
"step": 7070,
|
|
"unclipped_grad_norm": 1.2382662296295166
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5001651074665814,
|
|
"learning_rate": 3.964520480388956e-05,
|
|
"loss_nats": 1.0398352146148682,
|
|
"step": 7080,
|
|
"unclipped_grad_norm": 1.1878225803375244
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5461551601164558,
|
|
"learning_rate": 3.943913121407678e-05,
|
|
"loss_nats": 1.0717130899429321,
|
|
"step": 7090,
|
|
"unclipped_grad_norm": 1.3277226686477661
|
|
},
|
|
{
|
|
"bits_per_byte": 1.525824231044709,
|
|
"learning_rate": 3.923520311307735e-05,
|
|
"loss_nats": 1.0576207637786865,
|
|
"step": 7100,
|
|
"unclipped_grad_norm": 1.2164647579193115
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5404795606553323,
|
|
"learning_rate": 3.9033423985459816e-05,
|
|
"loss_nats": 1.0677790641784668,
|
|
"step": 7110,
|
|
"unclipped_grad_norm": 1.1725497245788574
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5736082347257334,
|
|
"learning_rate": 3.88337972790728e-05,
|
|
"loss_nats": 1.0907421112060547,
|
|
"step": 7120,
|
|
"unclipped_grad_norm": 1.218058705329895
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5565896915096042,
|
|
"learning_rate": 3.8636326404985845e-05,
|
|
"loss_nats": 1.0789457559585571,
|
|
"step": 7130,
|
|
"unclipped_grad_norm": 1.3638907670974731
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5273908210114182,
|
|
"learning_rate": 3.844101473743143e-05,
|
|
"loss_nats": 1.0587066411972046,
|
|
"step": 7140,
|
|
"unclipped_grad_norm": 1.2031904458999634
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5130964829849558,
|
|
"learning_rate": 3.824786561374701e-05,
|
|
"loss_nats": 1.0487985610961914,
|
|
"step": 7150,
|
|
"unclipped_grad_norm": 1.1664750576019287
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5860308855628558,
|
|
"learning_rate": 3.805688233431824e-05,
|
|
"loss_nats": 1.0993528366088867,
|
|
"step": 7160,
|
|
"unclipped_grad_norm": 1.2491129636764526
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5206325907631133,
|
|
"learning_rate": 3.786806816252236e-05,
|
|
"loss_nats": 1.054022192955017,
|
|
"step": 7170,
|
|
"unclipped_grad_norm": 1.20036780834198
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5424085180674096,
|
|
"learning_rate": 3.768142632467269e-05,
|
|
"loss_nats": 1.0691161155700684,
|
|
"step": 7180,
|
|
"unclipped_grad_norm": 1.2460989952087402
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5104255924170564,
|
|
"learning_rate": 3.7496960009963236e-05,
|
|
"loss_nats": 1.0469472408294678,
|
|
"step": 7190,
|
|
"unclipped_grad_norm": 1.2477872371673584
|
|
},
|
|
{
|
|
"bits_per_byte": 1.550646659026389,
|
|
"learning_rate": 3.7314672370414325e-05,
|
|
"loss_nats": 1.0748263597488403,
|
|
"step": 7200,
|
|
"unclipped_grad_norm": 1.26192045211792
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5958559104413463,
|
|
"learning_rate": 3.713456652081882e-05,
|
|
"loss_nats": 1.1061630249023438,
|
|
"step": 7210,
|
|
"unclipped_grad_norm": 1.2875406742095947
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5889428958072649,
|
|
"learning_rate": 3.695664553868873e-05,
|
|
"loss_nats": 1.1013712882995605,
|
|
"step": 7220,
|
|
"unclipped_grad_norm": 1.2469474077224731
|
|
},
|
|
{
|
|
"bits_per_byte": 1.512987102019007,
|
|
"learning_rate": 3.678091246420277e-05,
|
|
"loss_nats": 1.048722743988037,
|
|
"step": 7230,
|
|
"unclipped_grad_norm": 1.2010514736175537
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5378745394427134,
|
|
"learning_rate": 3.660737030015427e-05,
|
|
"loss_nats": 1.0659734010696411,
|
|
"step": 7240,
|
|
"unclipped_grad_norm": 1.201562762260437
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5587365509403246,
|
|
"learning_rate": 3.6436022011900056e-05,
|
|
"loss_nats": 1.0804338455200195,
|
|
"step": 7250,
|
|
"unclipped_grad_norm": 1.305864691734314
|
|
},
|
|
{
|
|
"bits_per_byte": 1.457526824699372,
|
|
"learning_rate": 3.626687052730954e-05,
|
|
"loss_nats": 1.0102806091308594,
|
|
"step": 7260,
|
|
"unclipped_grad_norm": 1.1730221509933472
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4923226985872373,
|
|
"learning_rate": 3.6099918736714955e-05,
|
|
"loss_nats": 1.0343992710113525,
|
|
"step": 7270,
|
|
"unclipped_grad_norm": 1.222469687461853
|
|
},
|
|
{
|
|
"bits_per_byte": 1.56038208094378,
|
|
"learning_rate": 3.5935169492861716e-05,
|
|
"loss_nats": 1.0815744400024414,
|
|
"step": 7280,
|
|
"unclipped_grad_norm": 1.2304381132125854
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4970130094415677,
|
|
"learning_rate": 3.5772625610859886e-05,
|
|
"loss_nats": 1.0376503467559814,
|
|
"step": 7290,
|
|
"unclipped_grad_norm": 1.1906161308288574
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5193676583660174,
|
|
"learning_rate": 3.5612289868135887e-05,
|
|
"loss_nats": 1.053145408630371,
|
|
"step": 7300,
|
|
"unclipped_grad_norm": 1.2887614965438843
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5227811700203424,
|
|
"learning_rate": 3.545416500438522e-05,
|
|
"loss_nats": 1.055511474609375,
|
|
"step": 7310,
|
|
"unclipped_grad_norm": 1.1584672927856445
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4581731355013146,
|
|
"learning_rate": 3.52982537215255e-05,
|
|
"loss_nats": 1.0107285976409912,
|
|
"step": 7320,
|
|
"unclipped_grad_norm": 1.2041478157043457
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4976147767369368,
|
|
"learning_rate": 3.5144558683650396e-05,
|
|
"loss_nats": 1.0380674600601196,
|
|
"step": 7330,
|
|
"unclipped_grad_norm": 1.2121145725250244
|
|
},
|
|
{
|
|
"bits_per_byte": 1.456979231939025,
|
|
"learning_rate": 3.499308251698404e-05,
|
|
"loss_nats": 1.0099010467529297,
|
|
"step": 7340,
|
|
"unclipped_grad_norm": 1.236539363861084
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5469846324415673,
|
|
"learning_rate": 3.484382780983616e-05,
|
|
"loss_nats": 1.0722880363464355,
|
|
"step": 7350,
|
|
"unclipped_grad_norm": 1.240252137184143
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4978240796230369,
|
|
"learning_rate": 3.469679711255795e-05,
|
|
"loss_nats": 1.038212537765503,
|
|
"step": 7360,
|
|
"unclipped_grad_norm": 1.2412164211273193
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5397654886889498,
|
|
"learning_rate": 3.4551992937498284e-05,
|
|
"loss_nats": 1.067284107208252,
|
|
"step": 7370,
|
|
"unclipped_grad_norm": 1.2414754629135132
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4664532402271044,
|
|
"learning_rate": 3.440941775896104e-05,
|
|
"loss_nats": 1.0164679288864136,
|
|
"step": 7380,
|
|
"unclipped_grad_norm": 1.196345567703247
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5561741814251195,
|
|
"learning_rate": 3.4269074013162596e-05,
|
|
"loss_nats": 1.0786577463150024,
|
|
"step": 7390,
|
|
"unclipped_grad_norm": 1.2357971668243408
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4878109057245024,
|
|
"learning_rate": 3.41309640981904e-05,
|
|
"loss_nats": 1.0312719345092773,
|
|
"step": 7400,
|
|
"unclipped_grad_norm": 1.2395251989364624
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4828184213526059,
|
|
"learning_rate": 3.3995090373961795e-05,
|
|
"loss_nats": 1.0278114080429077,
|
|
"step": 7410,
|
|
"unclipped_grad_norm": 1.1855907440185547
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5432890692398271,
|
|
"learning_rate": 3.3861455162183876e-05,
|
|
"loss_nats": 1.0697264671325684,
|
|
"step": 7420,
|
|
"unclipped_grad_norm": 1.2337415218353271
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5564147851536767,
|
|
"learning_rate": 3.3730060746313664e-05,
|
|
"loss_nats": 1.078824520111084,
|
|
"step": 7430,
|
|
"unclipped_grad_norm": 1.271388053894043
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5798298791033463,
|
|
"learning_rate": 3.3600909371519225e-05,
|
|
"loss_nats": 1.0950546264648438,
|
|
"step": 7440,
|
|
"unclipped_grad_norm": 1.3756663799285889
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5198117175705452,
|
|
"learning_rate": 3.3474003244641194e-05,
|
|
"loss_nats": 1.0534532070159912,
|
|
"step": 7450,
|
|
"unclipped_grad_norm": 1.2638351917266846
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5153886677556554,
|
|
"learning_rate": 3.334934453415514e-05,
|
|
"loss_nats": 1.0503873825073242,
|
|
"step": 7460,
|
|
"unclipped_grad_norm": 1.2390333414077759
|
|
},
|
|
{
|
|
"bits_per_byte": 1.500396252149341,
|
|
"learning_rate": 3.3226935370134486e-05,
|
|
"loss_nats": 1.0399954319000244,
|
|
"step": 7470,
|
|
"unclipped_grad_norm": 1.1914620399475098
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5462138062004,
|
|
"learning_rate": 3.31067778442141e-05,
|
|
"loss_nats": 1.071753740310669,
|
|
"step": 7480,
|
|
"unclipped_grad_norm": 1.2698674201965332
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5915580639962847,
|
|
"learning_rate": 3.298887400955457e-05,
|
|
"loss_nats": 1.1031839847564697,
|
|
"step": 7490,
|
|
"unclipped_grad_norm": 1.3026750087738037
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5533068866699349,
|
|
"learning_rate": 3.287322588080714e-05,
|
|
"loss_nats": 1.0766702890396118,
|
|
"step": 7500,
|
|
"unclipped_grad_norm": 1.1910096406936646
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5004994417398587,
|
|
"learning_rate": 3.275983543407924e-05,
|
|
"loss_nats": 1.0400669574737549,
|
|
"step": 7510,
|
|
"unclipped_grad_norm": 1.2149840593338013
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5185469571561,
|
|
"learning_rate": 3.2648704606900735e-05,
|
|
"loss_nats": 1.0525765419006348,
|
|
"step": 7520,
|
|
"unclipped_grad_norm": 1.25714111328125
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5776331447038756,
|
|
"learning_rate": 3.253983529819086e-05,
|
|
"loss_nats": 1.0935319662094116,
|
|
"step": 7530,
|
|
"unclipped_grad_norm": 1.2656311988830566
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5129728274589855,
|
|
"learning_rate": 3.243322936822571e-05,
|
|
"loss_nats": 1.0487128496170044,
|
|
"step": 7540,
|
|
"unclipped_grad_norm": 1.2008068561553955
|
|
},
|
|
{
|
|
"bits_per_byte": 1.542592195538531,
|
|
"learning_rate": 3.232888863860648e-05,
|
|
"loss_nats": 1.0692434310913086,
|
|
"step": 7550,
|
|
"unclipped_grad_norm": 1.3036167621612549
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5036082001368547,
|
|
"learning_rate": 3.222681489222838e-05,
|
|
"loss_nats": 1.0422217845916748,
|
|
"step": 7560,
|
|
"unclipped_grad_norm": 1.2302430868148804
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4793622580008667,
|
|
"learning_rate": 3.212700987325009e-05,
|
|
"loss_nats": 1.0254157781600952,
|
|
"step": 7570,
|
|
"unclipped_grad_norm": 1.2421420812606812
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6626436530774094,
|
|
"learning_rate": 3.202947528706403e-05,
|
|
"loss_nats": 1.1524567604064941,
|
|
"step": 7580,
|
|
"unclipped_grad_norm": 1.2984310388565063
|
|
},
|
|
{
|
|
"bits_per_byte": 1.510733441362101,
|
|
"learning_rate": 3.1934212800267116e-05,
|
|
"loss_nats": 1.0471606254577637,
|
|
"step": 7590,
|
|
"unclipped_grad_norm": 1.2202924489974976
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5062574208907455,
|
|
"learning_rate": 3.1841224040632484e-05,
|
|
"loss_nats": 1.044058084487915,
|
|
"step": 7600,
|
|
"unclipped_grad_norm": 1.2830675840377808
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5167008953817387,
|
|
"learning_rate": 3.175051059708143e-05,
|
|
"loss_nats": 1.0512969493865967,
|
|
"step": 7610,
|
|
"unclipped_grad_norm": 1.2556214332580566
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4693692060724834,
|
|
"learning_rate": 3.1662074019656414e-05,
|
|
"loss_nats": 1.018489122390747,
|
|
"step": 7620,
|
|
"unclipped_grad_norm": 1.2434303760528564
|
|
},
|
|
{
|
|
"bits_per_byte": 1.482038652013594,
|
|
"learning_rate": 3.157591581949455e-05,
|
|
"loss_nats": 1.0272709131240845,
|
|
"step": 7630,
|
|
"unclipped_grad_norm": 1.2516939640045166
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4886369383965965,
|
|
"learning_rate": 3.149203746880175e-05,
|
|
"loss_nats": 1.0318444967269897,
|
|
"step": 7640,
|
|
"unclipped_grad_norm": 1.2500675916671753
|
|
},
|
|
{
|
|
"bits_per_byte": 1.61461749782397,
|
|
"learning_rate": 3.141044040082763e-05,
|
|
"loss_nats": 1.1191675662994385,
|
|
"step": 7650,
|
|
"unclipped_grad_norm": 1.2970722913742065
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5327549598918293,
|
|
"learning_rate": 3.133112600984091e-05,
|
|
"loss_nats": 1.0624247789382935,
|
|
"step": 7660,
|
|
"unclipped_grad_norm": 1.313043236732483
|
|
},
|
|
{
|
|
"bits_per_byte": 1.560077327686451,
|
|
"learning_rate": 3.125409565110571e-05,
|
|
"loss_nats": 1.0813632011413574,
|
|
"step": 7670,
|
|
"unclipped_grad_norm": 1.2718157768249512
|
|
},
|
|
{
|
|
"bits_per_byte": 1.506822211916179,
|
|
"learning_rate": 3.117935064085835e-05,
|
|
"loss_nats": 1.0444495677947998,
|
|
"step": 7680,
|
|
"unclipped_grad_norm": 1.2939503192901611
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5160353225228995,
|
|
"learning_rate": 3.110689225628479e-05,
|
|
"loss_nats": 1.0508356094360352,
|
|
"step": 7690,
|
|
"unclipped_grad_norm": 1.2968342304229736
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4732078588397413,
|
|
"learning_rate": 3.1036721735498926e-05,
|
|
"loss_nats": 1.0211498737335205,
|
|
"step": 7700,
|
|
"unclipped_grad_norm": 1.2705638408660889
|
|
},
|
|
{
|
|
"bits_per_byte": 1.539139815805111,
|
|
"learning_rate": 3.096884027752132e-05,
|
|
"loss_nats": 1.0668504238128662,
|
|
"step": 7710,
|
|
"unclipped_grad_norm": 1.2731654644012451
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5645254869683667,
|
|
"learning_rate": 3.090324904225885e-05,
|
|
"loss_nats": 1.0844464302062988,
|
|
"step": 7720,
|
|
"unclipped_grad_norm": 1.281749963760376
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5594863952980864,
|
|
"learning_rate": 3.083994915048472e-05,
|
|
"loss_nats": 1.080953598022461,
|
|
"step": 7730,
|
|
"unclipped_grad_norm": 1.2457833290100098
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5044648457208023,
|
|
"learning_rate": 3.077894168381942e-05,
|
|
"loss_nats": 1.0428155660629272,
|
|
"step": 7740,
|
|
"unclipped_grad_norm": 1.313395619392395
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5058419108062608,
|
|
"learning_rate": 3.0720227684712254e-05,
|
|
"loss_nats": 1.0437700748443604,
|
|
"step": 7750,
|
|
"unclipped_grad_norm": 1.252753496170044
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5365542286320395,
|
|
"learning_rate": 3.0663808156423456e-05,
|
|
"loss_nats": 1.0650582313537598,
|
|
"step": 7760,
|
|
"unclipped_grad_norm": 1.208566665649414
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4627389309164203,
|
|
"learning_rate": 3.060968406300711e-05,
|
|
"loss_nats": 1.0138933658599854,
|
|
"step": 7770,
|
|
"unclipped_grad_norm": 1.2469086647033691
|
|
},
|
|
{
|
|
"bits_per_byte": 1.539320397588517,
|
|
"learning_rate": 3.055785632929463e-05,
|
|
"loss_nats": 1.0669755935668945,
|
|
"step": 7780,
|
|
"unclipped_grad_norm": 1.2808659076690674
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4840185162903266,
|
|
"learning_rate": 3.0508325840878984e-05,
|
|
"loss_nats": 1.028643250465393,
|
|
"step": 7790,
|
|
"unclipped_grad_norm": 1.1823458671569824
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5017442521668036,
|
|
"learning_rate": 3.046109344409957e-05,
|
|
"loss_nats": 1.0409297943115234,
|
|
"step": 7800,
|
|
"unclipped_grad_norm": 1.2036527395248413
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4928728710873473,
|
|
"learning_rate": 3.0416159946027727e-05,
|
|
"loss_nats": 1.0347806215286255,
|
|
"step": 7810,
|
|
"unclipped_grad_norm": 1.2650182247161865
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4653472197994057,
|
|
"learning_rate": 3.0373526114452946e-05,
|
|
"loss_nats": 1.0157012939453125,
|
|
"step": 7820,
|
|
"unclipped_grad_norm": 1.2730145454406738
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5153749091435864,
|
|
"learning_rate": 3.0333192677869803e-05,
|
|
"loss_nats": 1.0503778457641602,
|
|
"step": 7830,
|
|
"unclipped_grad_norm": 1.3102401494979858
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5122207473267624,
|
|
"learning_rate": 3.029516032546544e-05,
|
|
"loss_nats": 1.0481915473937988,
|
|
"step": 7840,
|
|
"unclipped_grad_norm": 1.2572261095046997
|
|
},
|
|
{
|
|
"bits_per_byte": 1.458128591994741,
|
|
"learning_rate": 3.02594297071078e-05,
|
|
"loss_nats": 1.0106977224349976,
|
|
"step": 7850,
|
|
"unclipped_grad_norm": 1.253402829170227
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5241797329371587,
|
|
"learning_rate": 3.0226001433334603e-05,
|
|
"loss_nats": 1.056480884552002,
|
|
"step": 7860,
|
|
"unclipped_grad_norm": 1.2006174325942993
|
|
},
|
|
{
|
|
"bits_per_byte": 1.539945898489705,
|
|
"learning_rate": 3.019487607534278e-05,
|
|
"loss_nats": 1.0674091577529907,
|
|
"step": 7870,
|
|
"unclipped_grad_norm": 1.2778089046478271
|
|
},
|
|
{
|
|
"bits_per_byte": 1.453407496245906,
|
|
"learning_rate": 3.016605416497886e-05,
|
|
"loss_nats": 1.007425308227539,
|
|
"step": 7880,
|
|
"unclipped_grad_norm": 1.2744390964508057
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5524588402185304,
|
|
"learning_rate": 3.0139536194729734e-05,
|
|
"loss_nats": 1.076082468032837,
|
|
"step": 7890,
|
|
"unclipped_grad_norm": 1.2452044486999512
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5187442212566398,
|
|
"learning_rate": 3.0115322617714362e-05,
|
|
"loss_nats": 1.0527132749557495,
|
|
"step": 7900,
|
|
"unclipped_grad_norm": 1.270271897315979
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5143311464355,
|
|
"learning_rate": 3.0093413847675965e-05,
|
|
"loss_nats": 1.0496543645858765,
|
|
"step": 7910,
|
|
"unclipped_grad_norm": 1.2356430292129517
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5350039770171622,
|
|
"learning_rate": 3.0073810258974985e-05,
|
|
"loss_nats": 1.063983678817749,
|
|
"step": 7920,
|
|
"unclipped_grad_norm": 1.2489451169967651
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4882054339255817,
|
|
"learning_rate": 3.0056512186582658e-05,
|
|
"loss_nats": 1.0315454006195068,
|
|
"step": 7930,
|
|
"unclipped_grad_norm": 1.2472636699676514
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4964919020094534,
|
|
"learning_rate": 3.0041519926075316e-05,
|
|
"loss_nats": 1.0372891426086426,
|
|
"step": 7940,
|
|
"unclipped_grad_norm": 1.3003731966018677
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5605986071012163,
|
|
"learning_rate": 3.0028833733629313e-05,
|
|
"loss_nats": 1.0817245244979858,
|
|
"step": 7950,
|
|
"unclipped_grad_norm": 1.2693594694137573
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4811777068633747,
|
|
"learning_rate": 3.0018453826016686e-05,
|
|
"loss_nats": 1.0266741514205933,
|
|
"step": 7960,
|
|
"unclipped_grad_norm": 1.3084068298339844
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5064964767754447,
|
|
"learning_rate": 3.001038038060143e-05,
|
|
"loss_nats": 1.0442237854003906,
|
|
"step": 7970,
|
|
"unclipped_grad_norm": 1.2679471969604492
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5642893547887322,
|
|
"learning_rate": 3.0004613535336433e-05,
|
|
"loss_nats": 1.0842827558517456,
|
|
"step": 7980,
|
|
"unclipped_grad_norm": 1.3388662338256836
|
|
},
|
|
{
|
|
"bits_per_byte": 1.598760697414419,
|
|
"learning_rate": 3.000115338876117e-05,
|
|
"loss_nats": 1.1081764698028564,
|
|
"step": 7990,
|
|
"unclipped_grad_norm": 1.3723206520080566
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4810289418703784,
|
|
"learning_rate": 3e-05,
|
|
"loss_nats": 1.0265710353851318,
|
|
"step": 8000,
|
|
"unclipped_grad_norm": 1.1791528463363647
|
|
}
|
|
]
|
|
}
|