wuyang
9617 lines
272 KiB
JSON
9617 lines
272 KiB
JSON
{
|
|
"architecture": "block",
|
|
"batch_size": 32,
|
|
"canonical_sha256_without_self": "6db370af4c7fc867e59305adb946f528981bb02fea76beb10dfc1fa42360d383",
|
|
"depth": 16,
|
|
"diagnostics": [
|
|
{
|
|
"activation_grad_rms_by_block": [
|
|
0.00026696090935729444,
|
|
0.00023596938990522176,
|
|
0.00020828393462579697,
|
|
0.00018593396816868335,
|
|
0.00016940363275352865,
|
|
0.00015857137623243034,
|
|
0.00015041495498735458,
|
|
0.00014485203428193927,
|
|
0.00013822126493323594,
|
|
0.00013424422650132328,
|
|
0.00013012201816309243,
|
|
0.00012678767961915582,
|
|
0.00012444572348613292,
|
|
0.00012238229101058096,
|
|
0.00012059078289894387,
|
|
0.00011903457198059186
|
|
],
|
|
"activation_grad_statistics": {
|
|
"first_quartile_mean": 0.00022428705051424913,
|
|
"first_to_last_ratio": 1.84426351740015,
|
|
"imbalance_abs_log_ratio": 0.6120800202390253,
|
|
"last_quartile_mean": 0.0001216133423440624,
|
|
"mean": 0.00015851367243158165,
|
|
"normalized": [
|
|
1.6841506808980231,
|
|
1.488637454961949,
|
|
1.3139808789408836,
|
|
1.1729837894515815,
|
|
1.0687004480742655,
|
|
1.0003640304332333,
|
|
0.9489083981211609,
|
|
0.9138141338846396,
|
|
0.8719832353445448,
|
|
0.8468936744826624,
|
|
0.8208882941580717,
|
|
0.7998532724291052,
|
|
0.785078798422557,
|
|
0.772061419896786,
|
|
0.7607595045215659,
|
|
0.7509419859789702
|
|
],
|
|
"population_cv": 0.27150931413441437
|
|
},
|
|
"activation_output_rms_by_block": [
|
|
0.0048858714289963245,
|
|
0.006843581795692444,
|
|
0.004983869381248951,
|
|
0.007334618363529444,
|
|
0.005787155125290155,
|
|
0.007929109036922455,
|
|
0.005876626819372177,
|
|
0.008892860263586044,
|
|
0.006918553728610277,
|
|
0.009880779311060905,
|
|
0.007473294623196125,
|
|
0.010543949902057648,
|
|
0.008116732351481915,
|
|
0.011967957951128483,
|
|
0.008228716440498829,
|
|
0.012017112225294113
|
|
],
|
|
"activation_output_statistics": {
|
|
"first_quartile_mean": 0.006011985242366791,
|
|
"first_to_last_ratio": 0.5962715478148781,
|
|
"imbalance_abs_log_ratio": 0.5170590985397028,
|
|
"last_quartile_mean": 0.010082629742100835,
|
|
"mean": 0.007980049296747893,
|
|
"normalized": [
|
|
0.612260807835794,
|
|
0.8575864059488212,
|
|
0.6245411771177937,
|
|
0.9191194303171183,
|
|
0.725202929216063,
|
|
0.9936165482278163,
|
|
0.736414851693595,
|
|
1.114386632575083,
|
|
0.866981326973731,
|
|
1.2381852471873345,
|
|
0.9364973003665171,
|
|
1.321288817896729,
|
|
1.017128096538166,
|
|
1.499734839483483,
|
|
1.031161103710509,
|
|
1.5058944849114457
|
|
],
|
|
"population_cv": 0.2698221312597705
|
|
},
|
|
"bits_per_byte": 8.029409055280222,
|
|
"branch_output_rms_by_sublayer": [
|
|
0.0029694242402911186,
|
|
0.0038970725145190954,
|
|
0.003062367672100663,
|
|
0.0038208949845284224,
|
|
0.003184171626344323,
|
|
0.0038178148679435253,
|
|
0.003822639584541321,
|
|
0.0037665588315576315,
|
|
0.004263176582753658,
|
|
0.0038892142474651337,
|
|
0.004122398793697357,
|
|
0.003805827582255006,
|
|
0.004464023746550083,
|
|
0.0037649995647370815,
|
|
0.005492920521646738,
|
|
0.0037115272134542465,
|
|
0.005817505065351725,
|
|
0.003727257251739502,
|
|
0.006086217239499092,
|
|
0.00368275074288249,
|
|
0.0065266890451312065,
|
|
0.0036771725863218307,
|
|
0.0064827753230929375,
|
|
0.003755199955776334,
|
|
0.007209060713648796,
|
|
0.0037714343052357435,
|
|
0.007561341859400272,
|
|
0.0037067290395498276,
|
|
0.007512317039072514,
|
|
0.0035454558674246073,
|
|
0.007687372621148825,
|
|
0.0036056456156075
|
|
],
|
|
"capture": {
|
|
"all_gradients_finite": true,
|
|
"all_gradients_present": true,
|
|
"count": 16,
|
|
"dtypes": [
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32"
|
|
],
|
|
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
|
"shape": [
|
|
16,
|
|
256,
|
|
192
|
|
],
|
|
"storage_unique": true
|
|
},
|
|
"core_parameter_grad_rms_by_block": [
|
|
0.006887647168374071,
|
|
0.005982369825085796,
|
|
0.005174924477158116,
|
|
0.004946754810206586,
|
|
0.004967935165757302,
|
|
0.004123664448940147,
|
|
0.00409438714185196,
|
|
0.004247982317789705,
|
|
0.0044010147653324285,
|
|
0.004095379628913903,
|
|
0.00386485833459634,
|
|
0.0038097011691932423,
|
|
0.0037333099925226194,
|
|
0.004058250822051008,
|
|
0.003851730608686184,
|
|
0.00381805442027398
|
|
],
|
|
"core_parameter_grad_statistics": {
|
|
"first_quartile_mean": 0.005747924070206143,
|
|
"first_to_last_ratio": 1.487043658003427,
|
|
"imbalance_abs_log_ratio": 0.39679002683413417,
|
|
"last_quartile_mean": 0.003865336460883448,
|
|
"mean": 0.004503622818545837,
|
|
"normalized": [
|
|
1.5293570189783356,
|
|
1.3283461040410636,
|
|
1.1490581440008412,
|
|
1.0983945613375836,
|
|
1.10309752080024,
|
|
0.9156327283801325,
|
|
0.909131893770355,
|
|
0.9432366983079914,
|
|
0.9772165526849029,
|
|
0.9093522690331003,
|
|
0.858166522889289,
|
|
0.8459192349556811,
|
|
0.8289570736583262,
|
|
0.9011080602352411,
|
|
0.8552515971863425,
|
|
0.8477740197405745
|
|
],
|
|
"population_cv": 0.191198476656264
|
|
},
|
|
"depth_weights": [
|
|
{
|
|
"entropy_mean": 0.0,
|
|
"mean_weights": [
|
|
1.0
|
|
],
|
|
"sources": 1
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6931471824645996,
|
|
"mean_weights": [
|
|
0.5,
|
|
0.5
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0986123085021973,
|
|
"mean_weights": [
|
|
0.3333333432674408,
|
|
0.3333333432674408,
|
|
0.3333333432674408
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862943649291992,
|
|
"mean_weights": [
|
|
0.25,
|
|
0.25,
|
|
0.25,
|
|
0.25
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6094379425048828,
|
|
"mean_weights": [
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343,
|
|
0.20000001788139343
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791759729385376,
|
|
"mean_weights": [
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592,
|
|
0.1666666567325592
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9459102153778076,
|
|
"mean_weights": [
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548,
|
|
0.1428571492433548
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.079441547393799,
|
|
"mean_weights": [
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125,
|
|
0.125
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
}
|
|
],
|
|
"layer_input_rms_by_sublayer": [
|
|
0.02822229079902172,
|
|
0.014168336987495422,
|
|
0.01429277565330267,
|
|
0.014353902079164982,
|
|
0.014476437121629715,
|
|
0.009725152514874935,
|
|
0.00981744285672903,
|
|
0.009922475554049015,
|
|
0.010004729963839054,
|
|
0.0075368694961071014,
|
|
0.007599710486829281,
|
|
0.007675632368773222,
|
|
0.007730280980467796,
|
|
0.006251458544284105,
|
|
0.0062932223081588745,
|
|
0.006393968593329191,
|
|
0.006436701864004135,
|
|
0.005461040884256363,
|
|
0.005505258683115244,
|
|
0.005599391181021929,
|
|
0.005641042720526457,
|
|
0.004922671243548393,
|
|
0.004954405128955841,
|
|
0.005046301055699587,
|
|
0.005088083446025848,
|
|
0.00459245452657342,
|
|
0.0046256110072135925,
|
|
0.0047518787905573845,
|
|
0.004772109445184469,
|
|
0.004340849816799164,
|
|
0.004358765669167042,
|
|
0.004422077443450689
|
|
],
|
|
"loss_nats": 5.5655622482299805,
|
|
"loss_scale": 1.0,
|
|
"output_weights": {
|
|
"entropy_mean": 2.1972246170043945,
|
|
"mean_weights": [
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519,
|
|
0.11111113429069519
|
|
],
|
|
"sources": 9
|
|
},
|
|
"step": 0,
|
|
"stream_state_rms_by_sublayer": [
|
|
0.0029694242402911186,
|
|
0.0048858714289963245,
|
|
0.005668045952916145,
|
|
0.006843581795692444,
|
|
0.003184171626344323,
|
|
0.004983869381248951,
|
|
0.006258165929466486,
|
|
0.007334618363529444,
|
|
0.004263176582753658,
|
|
0.005787155125290155,
|
|
0.006935684010386467,
|
|
0.007929109036922455,
|
|
0.004464023746550083,
|
|
0.005876626819372177,
|
|
0.008052977733314037,
|
|
0.008892860263586044,
|
|
0.005817505065351725,
|
|
0.006918553728610277,
|
|
0.009139667265117168,
|
|
0.009880779311060905,
|
|
0.0065266890451312065,
|
|
0.007473294623196125,
|
|
0.00979399960488081,
|
|
0.010543949902057648,
|
|
0.007209060713648796,
|
|
0.008116732351481915,
|
|
0.0113961361348629,
|
|
0.011967957951128483,
|
|
0.007512317039072514,
|
|
0.008228716440498829,
|
|
0.011409307830035686,
|
|
0.012017112225294113
|
|
]
|
|
},
|
|
{
|
|
"activation_grad_rms_by_block": [
|
|
8.836213237373158e-05,
|
|
8.102934953058138e-05,
|
|
6.889242649776861e-05,
|
|
6.090942770242691e-05,
|
|
5.060216426500119e-05,
|
|
4.3459844164317474e-05,
|
|
3.709086377057247e-05,
|
|
3.2571624615229666e-05,
|
|
2.8896074582007714e-05,
|
|
2.6415440515847877e-05,
|
|
2.491714622010477e-05,
|
|
2.3614533347426914e-05,
|
|
2.2646127035841346e-05,
|
|
2.2026059014024213e-05,
|
|
2.118862721545156e-05,
|
|
2.081490310956724e-05
|
|
],
|
|
"activation_grad_statistics": {
|
|
"first_quartile_mean": 7.479833402612712e-05,
|
|
"first_to_last_ratio": 3.4518703579034327,
|
|
"imbalance_abs_log_ratio": 1.2389162168680456,
|
|
"last_quartile_mean": 2.166892909372109e-05,
|
|
"mean": 4.083979649749381e-05,
|
|
"normalized": [
|
|
2.1636281262849595,
|
|
1.984078190388482,
|
|
1.6868944609456193,
|
|
1.4914233890995197,
|
|
1.2390405585910906,
|
|
1.0641542782169457,
|
|
0.9082039322318514,
|
|
0.7975462026905171,
|
|
0.7075469777079071,
|
|
0.6468063697983631,
|
|
0.6101192551641105,
|
|
0.578223580249134,
|
|
0.5545112605355675,
|
|
0.5393283244047475,
|
|
0.518823036171393,
|
|
0.5096720575197914
|
|
],
|
|
"population_cv": 0.5343944992100478
|
|
},
|
|
"activation_output_rms_by_block": [
|
|
0.009077239781618118,
|
|
0.016407418996095657,
|
|
0.010880433954298496,
|
|
0.01978805474936962,
|
|
0.012077964842319489,
|
|
0.02216547541320324,
|
|
0.013358108699321747,
|
|
0.02644582837820053,
|
|
0.017588114365935326,
|
|
0.03464006260037422,
|
|
0.023338599130511284,
|
|
0.0432722382247448,
|
|
0.023765092715620995,
|
|
0.04745635390281677,
|
|
0.02874108962714672,
|
|
0.05685042589902878
|
|
],
|
|
"activation_output_statistics": {
|
|
"first_quartile_mean": 0.014038286870345473,
|
|
"first_to_last_ratio": 0.3580899608898232,
|
|
"imbalance_abs_log_ratio": 1.0269710367470868,
|
|
"last_quartile_mean": 0.039203240536153316,
|
|
"mean": 0.025365781330037862,
|
|
"normalized": [
|
|
0.35785374254838964,
|
|
0.6468327855789793,
|
|
0.4289414078254319,
|
|
0.7801082289523973,
|
|
0.4761518947581916,
|
|
0.8738337339112493,
|
|
0.5266192484086122,
|
|
1.042578899270242,
|
|
0.6933795626909267,
|
|
1.3656217464649458,
|
|
0.9200820123318648,
|
|
1.7059296404760187,
|
|
0.9368957496877358,
|
|
1.8708808250514843,
|
|
1.133065418060348,
|
|
2.2412251039831825
|
|
],
|
|
"population_cv": 0.5292067698975893
|
|
},
|
|
"bits_per_byte": 5.216988460540972,
|
|
"branch_output_rms_by_sublayer": [
|
|
0.004302801564335823,
|
|
0.007946929894387722,
|
|
0.004027083050459623,
|
|
0.008643880486488342,
|
|
0.004623814020305872,
|
|
0.0097366226837039,
|
|
0.005764479748904705,
|
|
0.009610678069293499,
|
|
0.0056845624931156635,
|
|
0.010435172356665134,
|
|
0.006739616859704256,
|
|
0.010610987432301044,
|
|
0.006770467385649681,
|
|
0.010895426385104656,
|
|
0.009306530468165874,
|
|
0.011636263690888882,
|
|
0.010160733014345169,
|
|
0.012227990664541721,
|
|
0.012430583126842976,
|
|
0.012838989496231079,
|
|
0.015689926221966743,
|
|
0.013102410361170769,
|
|
0.015432302840054035,
|
|
0.013170000165700912,
|
|
0.015551909804344177,
|
|
0.012761367484927177,
|
|
0.018755123019218445,
|
|
0.013125414028763771,
|
|
0.019000543281435966,
|
|
0.01451444998383522,
|
|
0.022471774369478226,
|
|
0.013329307548701763
|
|
],
|
|
"capture": {
|
|
"all_gradients_finite": true,
|
|
"all_gradients_present": true,
|
|
"count": 16,
|
|
"dtypes": [
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32"
|
|
],
|
|
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
|
"shape": [
|
|
16,
|
|
256,
|
|
192
|
|
],
|
|
"storage_unique": true
|
|
},
|
|
"core_parameter_grad_rms_by_block": [
|
|
0.00035471167919676877,
|
|
0.00030778414793465906,
|
|
0.0002905596100869004,
|
|
0.0002669019974562734,
|
|
0.0002519467909729747,
|
|
0.0002484952213280207,
|
|
0.00022573236044592232,
|
|
0.00021694664140763627,
|
|
0.00020872399951088957,
|
|
0.00018641204799172754,
|
|
0.0001823569017298164,
|
|
0.0001572320679108608,
|
|
0.00014731276753985554,
|
|
0.00014544881518036755,
|
|
0.00014011786536140868,
|
|
0.0001387938590477502
|
|
],
|
|
"core_parameter_grad_statistics": {
|
|
"first_quartile_mean": 0.0003049893586686504,
|
|
"first_to_last_ratio": 2.1340115402633257,
|
|
"imbalance_abs_log_ratio": 0.7580035606734332,
|
|
"last_quartile_mean": 0.0001429183267823455,
|
|
"mean": 0.00021684229831886448,
|
|
"normalized": [
|
|
1.6358048311919693,
|
|
1.4193916515405378,
|
|
1.339958173933553,
|
|
1.230857630294052,
|
|
1.1618895064582344,
|
|
1.1459720878009276,
|
|
1.0409978228232257,
|
|
1.0004811934275772,
|
|
0.9625612766931799,
|
|
0.8596664462466195,
|
|
0.840965545668873,
|
|
0.7250986967480513,
|
|
0.6793543911033149,
|
|
0.6707585019528177,
|
|
0.6461740465200508,
|
|
0.6400681975970168
|
|
],
|
|
"population_cv": 0.294333589312772
|
|
},
|
|
"depth_weights": [
|
|
{
|
|
"entropy_mean": 0.0,
|
|
"mean_weights": [
|
|
1.0
|
|
],
|
|
"sources": 1
|
|
},
|
|
{
|
|
"entropy_mean": 0.6924017667770386,
|
|
"mean_weights": [
|
|
0.4816230535507202,
|
|
0.5183769464492798
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6925060749053955,
|
|
"mean_weights": [
|
|
0.5170555114746094,
|
|
0.4829444885253906
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6922789812088013,
|
|
"mean_weights": [
|
|
0.5104873776435852,
|
|
0.4895126223564148
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6927056312561035,
|
|
"mean_weights": [
|
|
0.5133966207504272,
|
|
0.48660340905189514
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 1.097900390625,
|
|
"mean_weights": [
|
|
0.3491937518119812,
|
|
0.32761698961257935,
|
|
0.32318928837776184
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0982853174209595,
|
|
"mean_weights": [
|
|
0.3437661826610565,
|
|
0.3295633792877197,
|
|
0.32667046785354614
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.097551703453064,
|
|
"mean_weights": [
|
|
0.3531244695186615,
|
|
0.3277324438095093,
|
|
0.3191430866718292
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.098252296447754,
|
|
"mean_weights": [
|
|
0.3423447012901306,
|
|
0.3336608111858368,
|
|
0.323994517326355
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.3856292963027954,
|
|
"mean_weights": [
|
|
0.2598890960216522,
|
|
0.25528842210769653,
|
|
0.2465861737728119,
|
|
0.23823627829551697
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3862417936325073,
|
|
"mean_weights": [
|
|
0.2525826096534729,
|
|
0.2512264847755432,
|
|
0.24808365106582642,
|
|
0.24810725450515747
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3854787349700928,
|
|
"mean_weights": [
|
|
0.2609134912490845,
|
|
0.2559056282043457,
|
|
0.24643474817276,
|
|
0.23674613237380981
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.386183738708496,
|
|
"mean_weights": [
|
|
0.25091099739074707,
|
|
0.2541857361793518,
|
|
0.24937650561332703,
|
|
0.24552680552005768
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.6086668968200684,
|
|
"mean_weights": [
|
|
0.20840778946876526,
|
|
0.2072581648826599,
|
|
0.20017966628074646,
|
|
0.19401825964450836,
|
|
0.1901361048221588
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6093640327453613,
|
|
"mean_weights": [
|
|
0.20082800090312958,
|
|
0.20345230400562286,
|
|
0.19957157969474792,
|
|
0.1978151500225067,
|
|
0.19833296537399292
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6085686683654785,
|
|
"mean_weights": [
|
|
0.2064795196056366,
|
|
0.20873931050300598,
|
|
0.20255593955516815,
|
|
0.1941128820180893,
|
|
0.18811233341693878
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6093556880950928,
|
|
"mean_weights": [
|
|
0.2007957398891449,
|
|
0.20384852588176727,
|
|
0.2003384232521057,
|
|
0.19736894965171814,
|
|
0.19764836132526398
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.7904106378555298,
|
|
"mean_weights": [
|
|
0.17261356115341187,
|
|
0.1785532385110855,
|
|
0.16990220546722412,
|
|
0.16298523545265198,
|
|
0.1574374884366989,
|
|
0.1585082709789276
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.791741132736206,
|
|
"mean_weights": [
|
|
0.16599833965301514,
|
|
0.16723501682281494,
|
|
0.16713106632232666,
|
|
0.1660757064819336,
|
|
0.16713276505470276,
|
|
0.1664271056652069
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7893474102020264,
|
|
"mean_weights": [
|
|
0.17279052734375,
|
|
0.18296070396900177,
|
|
0.1714734584093094,
|
|
0.16615013778209686,
|
|
0.1573297083377838,
|
|
0.14929547905921936
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7916802167892456,
|
|
"mean_weights": [
|
|
0.1656593233346939,
|
|
0.16598697006702423,
|
|
0.1664597988128662,
|
|
0.16510574519634247,
|
|
0.16595518589019775,
|
|
0.17083296179771423
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.9424470663070679,
|
|
"mean_weights": [
|
|
0.14923666417598724,
|
|
0.16057875752449036,
|
|
0.15159368515014648,
|
|
0.14453238248825073,
|
|
0.13740569353103638,
|
|
0.1286274939775467,
|
|
0.12802532315254211
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9457793235778809,
|
|
"mean_weights": [
|
|
0.1414237767457962,
|
|
0.14120322465896606,
|
|
0.14217954874038696,
|
|
0.14177344739437103,
|
|
0.14084486663341522,
|
|
0.145671084523201,
|
|
0.14690405130386353
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.940324306488037,
|
|
"mean_weights": [
|
|
0.15125462412834167,
|
|
0.16445259749889374,
|
|
0.15269356966018677,
|
|
0.1458570957183838,
|
|
0.1393839716911316,
|
|
0.12596309185028076,
|
|
0.12039506435394287
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9458446502685547,
|
|
"mean_weights": [
|
|
0.1414341926574707,
|
|
0.14160503447055817,
|
|
0.14392146468162537,
|
|
0.1416645050048828,
|
|
0.14250624179840088,
|
|
0.1432597041130066,
|
|
0.1456088423728943
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 2.073972225189209,
|
|
"mean_weights": [
|
|
0.13393926620483398,
|
|
0.1460031270980835,
|
|
0.13552871346473694,
|
|
0.12954440712928772,
|
|
0.12087934464216232,
|
|
0.11472145467996597,
|
|
0.10819212347269058,
|
|
0.11119155585765839
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.0792579650878906,
|
|
"mean_weights": [
|
|
0.12344250082969666,
|
|
0.1223982647061348,
|
|
0.12281741201877594,
|
|
0.12404577434062958,
|
|
0.1245848536491394,
|
|
0.1274431347846985,
|
|
0.1261020004749298,
|
|
0.12916606664657593
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.070988178253174,
|
|
"mean_weights": [
|
|
0.13450230658054352,
|
|
0.1483522206544876,
|
|
0.13837148249149323,
|
|
0.13302284479141235,
|
|
0.12319637835025787,
|
|
0.11490452289581299,
|
|
0.10592599213123322,
|
|
0.10172426700592041
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.0791521072387695,
|
|
"mean_weights": [
|
|
0.12388300150632858,
|
|
0.12212474644184113,
|
|
0.12333755195140839,
|
|
0.12223256379365921,
|
|
0.12343418598175049,
|
|
0.12649230659008026,
|
|
0.12949144840240479,
|
|
0.12900421023368835
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.18794584274292,
|
|
"mean_weights": [
|
|
0.12259161472320557,
|
|
0.13473162055015564,
|
|
0.1250339150428772,
|
|
0.11940491199493408,
|
|
0.11259052157402039,
|
|
0.10314525663852692,
|
|
0.09605535864830017,
|
|
0.09219000488519669,
|
|
0.09425680339336395
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.197138786315918,
|
|
"mean_weights": [
|
|
0.11026664823293686,
|
|
0.11047966778278351,
|
|
0.11075460910797119,
|
|
0.10965213924646378,
|
|
0.10944131761789322,
|
|
0.111859992146492,
|
|
0.11317223310470581,
|
|
0.1125045120716095,
|
|
0.11186887323856354
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.189896583557129,
|
|
"mean_weights": [
|
|
0.12157853692770004,
|
|
0.13215823471546173,
|
|
0.1242145225405693,
|
|
0.11826580762863159,
|
|
0.11163709312677383,
|
|
0.1045699343085289,
|
|
0.09787097573280334,
|
|
0.09515326470136642,
|
|
0.09455163776874542
|
|
],
|
|
"sources": 9
|
|
}
|
|
],
|
|
"layer_input_rms_by_sublayer": [
|
|
0.028616169467568398,
|
|
0.013977974653244019,
|
|
0.015653911978006363,
|
|
0.015701081603765488,
|
|
0.01704590953886509,
|
|
0.011780167929828167,
|
|
0.012960037216544151,
|
|
0.013451050966978073,
|
|
0.014845823869109154,
|
|
0.011557077057659626,
|
|
0.012834185734391212,
|
|
0.013193685561418533,
|
|
0.014761172235012054,
|
|
0.012218841351568699,
|
|
0.013451457023620605,
|
|
0.01399444043636322,
|
|
0.015514986589550972,
|
|
0.013508559204638004,
|
|
0.014893320389091969,
|
|
0.015537911094725132,
|
|
0.017302624881267548,
|
|
0.015652775764465332,
|
|
0.017217284068465233,
|
|
0.017610477283596992,
|
|
0.019684365019202232,
|
|
0.017625300213694572,
|
|
0.019488012418150902,
|
|
0.019592516124248505,
|
|
0.022200310602784157,
|
|
0.019619757309556007,
|
|
0.022100279107689857,
|
|
0.021992051973938942
|
|
],
|
|
"loss_nats": 3.616140842437744,
|
|
"loss_scale": 1.0,
|
|
"output_weights": {
|
|
"entropy_mean": 2.1926791667938232,
|
|
"mean_weights": [
|
|
0.09771968424320221,
|
|
0.09518586099147797,
|
|
0.10263510048389435,
|
|
0.10697779804468155,
|
|
0.11385782063007355,
|
|
0.11843808740377426,
|
|
0.1225852519273758,
|
|
0.12233239412307739,
|
|
0.12026800215244293
|
|
],
|
|
"sources": 9
|
|
},
|
|
"step": 100,
|
|
"stream_state_rms_by_sublayer": [
|
|
0.004302801564335823,
|
|
0.009077239781618118,
|
|
0.010217261500656605,
|
|
0.016407418996095657,
|
|
0.004623814020305872,
|
|
0.010880433954298496,
|
|
0.012616241350769997,
|
|
0.01978805474936962,
|
|
0.0056845624931156635,
|
|
0.012077964842319489,
|
|
0.014198417775332928,
|
|
0.02216547541320324,
|
|
0.006770467385649681,
|
|
0.013358108699321747,
|
|
0.017934804782271385,
|
|
0.02644582837820053,
|
|
0.010160733014345169,
|
|
0.017588114365935326,
|
|
0.02522732876241207,
|
|
0.03464006260037422,
|
|
0.015689926221966743,
|
|
0.023338599130511284,
|
|
0.03383738547563553,
|
|
0.0432722382247448,
|
|
0.015551909804344177,
|
|
0.023765092715620995,
|
|
0.03784027695655823,
|
|
0.04745635390281677,
|
|
0.019000543281435966,
|
|
0.02874108962714672,
|
|
0.04672664403915405,
|
|
0.05685042589902878
|
|
]
|
|
},
|
|
{
|
|
"activation_grad_rms_by_block": [
|
|
0.00015875036478973925,
|
|
0.00016262078133877367,
|
|
0.00014673685654997826,
|
|
0.00013248690811451524,
|
|
0.00010898245091084391,
|
|
9.717086504679173e-05,
|
|
8.911665645428002e-05,
|
|
7.969579746713862e-05,
|
|
7.858985918574035e-05,
|
|
6.784927973058075e-05,
|
|
6.598868640139699e-05,
|
|
5.512806092156097e-05,
|
|
4.8474703362444416e-05,
|
|
4.3001487938454375e-05,
|
|
3.610895510064438e-05,
|
|
3.5184959415346384e-05
|
|
],
|
|
"activation_grad_statistics": {
|
|
"first_quartile_mean": 0.0001501487276982516,
|
|
"first_to_last_ratio": 3.689835475493601,
|
|
"imbalance_abs_log_ratio": 1.3055818704713245,
|
|
"last_quartile_mean": 4.069252645422239e-05,
|
|
"mean": 8.786791704551433e-05,
|
|
"normalized": [
|
|
1.806693160912291,
|
|
1.8507412808538344,
|
|
1.6699708094135275,
|
|
1.5077961623454543,
|
|
1.2402985591930278,
|
|
1.1058742293442396,
|
|
1.0142115512777987,
|
|
0.9069954102344013,
|
|
0.8944090383413996,
|
|
0.7721735305895072,
|
|
0.7509986422827776,
|
|
0.6273969245567232,
|
|
0.5516769372982315,
|
|
0.489387815079083,
|
|
0.4109458413807676,
|
|
0.4004301068969357
|
|
],
|
|
"population_cv": 0.4737558240032668
|
|
},
|
|
"activation_output_rms_by_block": [
|
|
0.02468283288180828,
|
|
0.04329179599881172,
|
|
0.023283371701836586,
|
|
0.03508218377828598,
|
|
0.025414658710360527,
|
|
0.03976430371403694,
|
|
0.02767801284790039,
|
|
0.03781749680638313,
|
|
0.03550800308585167,
|
|
0.04360951855778694,
|
|
0.040389809757471085,
|
|
0.05188606306910515,
|
|
0.03616104647517204,
|
|
0.06982707977294922,
|
|
0.04418032616376877,
|
|
0.09354322403669357
|
|
],
|
|
"activation_output_statistics": {
|
|
"first_quartile_mean": 0.03158504609018564,
|
|
"first_to_last_ratio": 0.5184002104527677,
|
|
"imbalance_abs_log_ratio": 0.6570077279781673,
|
|
"last_quartile_mean": 0.0609279191121459,
|
|
"mean": 0.042007482959888875,
|
|
"normalized": [
|
|
0.5875818102545408,
|
|
1.0305734347413573,
|
|
0.5542672414833535,
|
|
0.8351412964157943,
|
|
0.6050031308618963,
|
|
0.9466004842995747,
|
|
0.6588829155588524,
|
|
0.9002561958423794,
|
|
0.8452780453367492,
|
|
1.038136910022145,
|
|
0.9614908323842579,
|
|
1.2351623904400295,
|
|
0.8608239277202269,
|
|
1.662253361850413,
|
|
1.0517251463496313,
|
|
2.2268228764387987
|
|
],
|
|
"population_cv": 0.4130647858768022
|
|
},
|
|
"bits_per_byte": 3.137961051112634,
|
|
"branch_output_rms_by_sublayer": [
|
|
0.013650949113070965,
|
|
0.01843169331550598,
|
|
0.019044609740376472,
|
|
0.014011893421411514,
|
|
0.018698271363973618,
|
|
0.012105143629014492,
|
|
0.021805183961987495,
|
|
0.010238726623356342,
|
|
0.020464032888412476,
|
|
0.011298555880784988,
|
|
0.021483227610588074,
|
|
0.011180058121681213,
|
|
0.022897671908140182,
|
|
0.01260110642760992,
|
|
0.023534495383501053,
|
|
0.012234007939696312,
|
|
0.028167055919766426,
|
|
0.01504397764801979,
|
|
0.03861295431852341,
|
|
0.015801267698407173,
|
|
0.035523753613233566,
|
|
0.0176701582968235,
|
|
0.020830485969781876,
|
|
0.024693600833415985,
|
|
0.022914253175258636,
|
|
0.028566768392920494,
|
|
0.02199499122798443,
|
|
0.03558572381734848,
|
|
0.013380669057369232,
|
|
0.04152822494506836,
|
|
0.013844908215105534,
|
|
0.04695267602801323
|
|
],
|
|
"capture": {
|
|
"all_gradients_finite": true,
|
|
"all_gradients_present": true,
|
|
"count": 16,
|
|
"dtypes": [
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32"
|
|
],
|
|
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
|
"shape": [
|
|
16,
|
|
256,
|
|
192
|
|
],
|
|
"storage_unique": true
|
|
},
|
|
"core_parameter_grad_rms_by_block": [
|
|
0.0008395536207241178,
|
|
0.0006692968987216388,
|
|
0.0005333404576068953,
|
|
0.0004419910350210091,
|
|
0.0004333787114417237,
|
|
0.0004609433225170186,
|
|
0.0004705681902150354,
|
|
0.0005107414594492524,
|
|
0.0005339389183678795,
|
|
0.0004946080122888242,
|
|
0.000448918054550765,
|
|
0.00035668617352665593,
|
|
0.00032572588316331664,
|
|
0.000267052972482006,
|
|
0.0002153120435976381,
|
|
0.0002101978579413297
|
|
],
|
|
"core_parameter_grad_statistics": {
|
|
"first_quartile_mean": 0.0006210455030184152,
|
|
"first_to_last_ratio": 2.439565392966498,
|
|
"imbalance_abs_log_ratio": 0.8918199058035992,
|
|
"last_quartile_mean": 0.00025457218929607263,
|
|
"mean": 0.0004507658507259441,
|
|
"normalized": [
|
|
1.8625049332642285,
|
|
1.484799475478804,
|
|
1.183187361571351,
|
|
0.980533539329114,
|
|
0.9614275587730993,
|
|
1.022578178392804,
|
|
1.04393043407614,
|
|
1.1330526893878927,
|
|
1.1845150148530281,
|
|
1.0972614972768537,
|
|
0.9959007627303548,
|
|
0.7912892535053935,
|
|
0.7226055004804499,
|
|
0.5924427772244185,
|
|
0.47765828589473863,
|
|
0.46631273776132937
|
|
],
|
|
"population_cv": 0.34661834917809303
|
|
},
|
|
"depth_weights": [
|
|
{
|
|
"entropy_mean": 0.0,
|
|
"mean_weights": [
|
|
1.0
|
|
],
|
|
"sources": 1
|
|
},
|
|
{
|
|
"entropy_mean": 0.6546831130981445,
|
|
"mean_weights": [
|
|
0.3866303563117981,
|
|
0.6133696436882019
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6832369565963745,
|
|
"mean_weights": [
|
|
0.48925086855888367,
|
|
0.510749101638794
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6867785453796387,
|
|
"mean_weights": [
|
|
0.5347544550895691,
|
|
0.4652456045150757
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6725893020629883,
|
|
"mean_weights": [
|
|
0.4844614565372467,
|
|
0.5155385732650757
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 1.0954453945159912,
|
|
"mean_weights": [
|
|
0.332688570022583,
|
|
0.3308390974998474,
|
|
0.3364723324775696
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0907697677612305,
|
|
"mean_weights": [
|
|
0.3209903836250305,
|
|
0.33586689829826355,
|
|
0.34314271807670593
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0933208465576172,
|
|
"mean_weights": [
|
|
0.33219069242477417,
|
|
0.312181293964386,
|
|
0.35562801361083984
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0861705541610718,
|
|
"mean_weights": [
|
|
0.319164514541626,
|
|
0.32808834314346313,
|
|
0.3527471721172333
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.3827753067016602,
|
|
"mean_weights": [
|
|
0.24703660607337952,
|
|
0.23925676941871643,
|
|
0.2577625513076782,
|
|
0.25594407320022583
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3791232109069824,
|
|
"mean_weights": [
|
|
0.24712958931922913,
|
|
0.2454403042793274,
|
|
0.24896711111068726,
|
|
0.25846296548843384
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3816200494766235,
|
|
"mean_weights": [
|
|
0.2438572347164154,
|
|
0.2418915331363678,
|
|
0.2618292272090912,
|
|
0.252422034740448
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3765439987182617,
|
|
"mean_weights": [
|
|
0.24315571784973145,
|
|
0.25071609020233154,
|
|
0.23800383508205414,
|
|
0.26812440156936646
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.6069204807281494,
|
|
"mean_weights": [
|
|
0.1977769434452057,
|
|
0.19419988989830017,
|
|
0.19346478581428528,
|
|
0.20617546141147614,
|
|
0.20838290452957153
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6016991138458252,
|
|
"mean_weights": [
|
|
0.19928976893424988,
|
|
0.19587931036949158,
|
|
0.19414740800857544,
|
|
0.18995173275470734,
|
|
0.22073179483413696
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6063685417175293,
|
|
"mean_weights": [
|
|
0.19720253348350525,
|
|
0.19060829281806946,
|
|
0.18910765647888184,
|
|
0.20065464079380035,
|
|
0.2224268764257431
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6036955118179321,
|
|
"mean_weights": [
|
|
0.19818633794784546,
|
|
0.20014071464538574,
|
|
0.19487890601158142,
|
|
0.18811310827732086,
|
|
0.2186809629201889
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.7846078872680664,
|
|
"mean_weights": [
|
|
0.15616348385810852,
|
|
0.1693512499332428,
|
|
0.1684132218360901,
|
|
0.14385344088077545,
|
|
0.16338428854942322,
|
|
0.19883427023887634
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7881443500518799,
|
|
"mean_weights": [
|
|
0.16932255029678345,
|
|
0.1670200079679489,
|
|
0.16782812774181366,
|
|
0.15219610929489136,
|
|
0.1653819978237152,
|
|
0.1782512068748474
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7859947681427002,
|
|
"mean_weights": [
|
|
0.15620459616184235,
|
|
0.18192027509212494,
|
|
0.16651901602745056,
|
|
0.14659695327281952,
|
|
0.16306468844413757,
|
|
0.18569447100162506
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7886135578155518,
|
|
"mean_weights": [
|
|
0.17137612402439117,
|
|
0.16813044250011444,
|
|
0.16593416035175323,
|
|
0.15192663669586182,
|
|
0.16282179951667786,
|
|
0.17981085181236267
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.937737226486206,
|
|
"mean_weights": [
|
|
0.12751945853233337,
|
|
0.15645764768123627,
|
|
0.14123380184173584,
|
|
0.1275305449962616,
|
|
0.13446006178855896,
|
|
0.15007954835891724,
|
|
0.16271892189979553
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.943496823310852,
|
|
"mean_weights": [
|
|
0.14288359880447388,
|
|
0.1535882204771042,
|
|
0.14250940084457397,
|
|
0.1380811333656311,
|
|
0.1316976547241211,
|
|
0.13936954736709595,
|
|
0.15187042951583862
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9273982048034668,
|
|
"mean_weights": [
|
|
0.12311284244060516,
|
|
0.16783711314201355,
|
|
0.141703262925148,
|
|
0.11705678701400757,
|
|
0.12322613596916199,
|
|
0.14240297675132751,
|
|
0.184660866856575
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.937564730644226,
|
|
"mean_weights": [
|
|
0.13963916897773743,
|
|
0.16193091869354248,
|
|
0.14834126830101013,
|
|
0.12877905368804932,
|
|
0.1304946392774582,
|
|
0.13670659065246582,
|
|
0.15410837531089783
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 2.0582637786865234,
|
|
"mean_weights": [
|
|
0.10422320663928986,
|
|
0.14683584868907928,
|
|
0.12450192868709564,
|
|
0.10290674865245819,
|
|
0.1043052226305008,
|
|
0.1136581152677536,
|
|
0.13712583482265472,
|
|
0.1664431095123291
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.073244333267212,
|
|
"mean_weights": [
|
|
0.11922207474708557,
|
|
0.13852310180664062,
|
|
0.1310172826051712,
|
|
0.1193084791302681,
|
|
0.1166592389345169,
|
|
0.11741948127746582,
|
|
0.12622521817684174,
|
|
0.13162514567375183
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.05503249168396,
|
|
"mean_weights": [
|
|
0.10400840640068054,
|
|
0.1620255410671234,
|
|
0.13802370429039001,
|
|
0.10328710824251175,
|
|
0.10247690975666046,
|
|
0.11085900664329529,
|
|
0.13948379456996918,
|
|
0.13983553647994995
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.0654377937316895,
|
|
"mean_weights": [
|
|
0.12162524461746216,
|
|
0.14755912125110626,
|
|
0.1359378546476364,
|
|
0.11907999217510223,
|
|
0.10864339768886566,
|
|
0.11126887798309326,
|
|
0.12606029212474823,
|
|
0.12982521951198578
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.158114194869995,
|
|
"mean_weights": [
|
|
0.08706305176019669,
|
|
0.1523345708847046,
|
|
0.1240597814321518,
|
|
0.09008736908435822,
|
|
0.08798909187316895,
|
|
0.09299354255199432,
|
|
0.11455917358398438,
|
|
0.09811963140964508,
|
|
0.15279380977153778
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1800241470336914,
|
|
"mean_weights": [
|
|
0.10917439311742783,
|
|
0.13041174411773682,
|
|
0.1256810426712036,
|
|
0.10147396475076675,
|
|
0.10038554668426514,
|
|
0.10393467545509338,
|
|
0.11099927127361298,
|
|
0.11076852679252625,
|
|
0.10717083513736725
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.1717891693115234,
|
|
"mean_weights": [
|
|
0.09730195999145508,
|
|
0.15644855797290802,
|
|
0.1323733627796173,
|
|
0.10013202577829361,
|
|
0.09525948762893677,
|
|
0.09674295783042908,
|
|
0.12173393368721008,
|
|
0.10994285345077515,
|
|
0.09006483852863312
|
|
],
|
|
"sources": 9
|
|
}
|
|
],
|
|
"layer_input_rms_by_sublayer": [
|
|
0.04337252303957939,
|
|
0.018801197409629822,
|
|
0.025790294632315636,
|
|
0.029389970004558563,
|
|
0.031753405928611755,
|
|
0.022941799834370613,
|
|
0.02458134852349758,
|
|
0.02529502473771572,
|
|
0.026913003996014595,
|
|
0.020798955112695694,
|
|
0.022398613393306732,
|
|
0.022611256688833237,
|
|
0.02451939694583416,
|
|
0.01929222047328949,
|
|
0.020593678578734398,
|
|
0.020593887194991112,
|
|
0.021762654185295105,
|
|
0.01771371066570282,
|
|
0.019139636307954788,
|
|
0.0189701896160841,
|
|
0.02020873688161373,
|
|
0.016272971406579018,
|
|
0.017277248203754425,
|
|
0.01622883230447769,
|
|
0.018368225544691086,
|
|
0.014602466486394405,
|
|
0.017397841438651085,
|
|
0.017420755699276924,
|
|
0.02048002928495407,
|
|
0.016635844483971596,
|
|
0.021542873233556747,
|
|
0.021214226260781288
|
|
],
|
|
"loss_nats": 2.1750688552856445,
|
|
"loss_scale": 1.0,
|
|
"output_weights": {
|
|
"entropy_mean": 2.173111915588379,
|
|
"mean_weights": [
|
|
0.1217261254787445,
|
|
0.09291160106658936,
|
|
0.09640607982873917,
|
|
0.10597392916679382,
|
|
0.09402626007795334,
|
|
0.09312883019447327,
|
|
0.11353092640638351,
|
|
0.147413432598114,
|
|
0.13488279283046722
|
|
],
|
|
"sources": 9
|
|
},
|
|
"step": 500,
|
|
"stream_state_rms_by_sublayer": [
|
|
0.013650949113070965,
|
|
0.02468283288180828,
|
|
0.0356261320412159,
|
|
0.04329179599881172,
|
|
0.018698271363973618,
|
|
0.023283371701836586,
|
|
0.03009101189672947,
|
|
0.03508218377828598,
|
|
0.020464032888412476,
|
|
0.025414658710360527,
|
|
0.034230802208185196,
|
|
0.03976430371403694,
|
|
0.022897671908140182,
|
|
0.02767801284790039,
|
|
0.031892068684101105,
|
|
0.03781749680638313,
|
|
0.028167055919766426,
|
|
0.03550800308585167,
|
|
0.035372670739889145,
|
|
0.04360951855778694,
|
|
0.035523753613233566,
|
|
0.040389809757471085,
|
|
0.03893519565463066,
|
|
0.05188606306910515,
|
|
0.022914253175258636,
|
|
0.03616104647517204,
|
|
0.04390736669301987,
|
|
0.06982707977294922,
|
|
0.013380669057369232,
|
|
0.04418032616376877,
|
|
0.051131755113601685,
|
|
0.09354322403669357
|
|
]
|
|
},
|
|
{
|
|
"activation_grad_rms_by_block": [
|
|
7.532809104304761e-05,
|
|
7.798708247719333e-05,
|
|
7.679135887883604e-05,
|
|
7.304617611225694e-05,
|
|
7.868888496886939e-05,
|
|
7.320931035792455e-05,
|
|
8.902617992134765e-05,
|
|
7.941434887470677e-05,
|
|
8.246234938269481e-05,
|
|
7.884583465056494e-05,
|
|
0.0001634695363463834,
|
|
0.00012072100798832253,
|
|
0.000141287746373564,
|
|
8.843849354889244e-05,
|
|
5.323962977854535e-05,
|
|
4.3788353650597855e-05
|
|
],
|
|
"activation_grad_statistics": {
|
|
"first_quartile_mean": 7.578817712783348e-05,
|
|
"first_to_last_ratio": 0.9277698246768501,
|
|
"imbalance_abs_log_ratio": 0.07497161071478954,
|
|
"last_quartile_mean": 8.168855583789991e-05,
|
|
"mean": 8.723402402210922e-05,
|
|
"normalized": [
|
|
0.8635173246617158,
|
|
0.8939984524550617,
|
|
0.8802913741474712,
|
|
0.8373587820933674,
|
|
0.9020435071174282,
|
|
0.8392288579897432,
|
|
1.0205442305262016,
|
|
0.9103598024387758,
|
|
0.9453003034893238,
|
|
0.9038426867776004,
|
|
1.8739194732659872,
|
|
1.383875263598136,
|
|
1.6196403634636298,
|
|
1.0138073365327953,
|
|
0.6103080807673381,
|
|
0.5019641606754243
|
|
],
|
|
"population_cv": 0.3374467083690701
|
|
},
|
|
"activation_output_rms_by_block": [
|
|
0.09266375005245209,
|
|
0.20113667845726013,
|
|
0.14358533918857574,
|
|
0.17041823267936707,
|
|
0.05199930816888809,
|
|
0.07249397784471512,
|
|
0.03721517324447632,
|
|
0.07351206243038177,
|
|
0.05156216770410538,
|
|
0.08937478810548782,
|
|
0.062482453882694244,
|
|
0.09533937275409698,
|
|
0.05472978577017784,
|
|
0.08922836184501648,
|
|
0.08201298862695694,
|
|
0.20069743692874908
|
|
],
|
|
"activation_output_statistics": {
|
|
"first_quartile_mean": 0.15195100009441376,
|
|
"first_to_last_ratio": 1.4245342605399758,
|
|
"imbalance_abs_log_ratio": 0.353844925589721,
|
|
"last_quartile_mean": 0.10666714329272509,
|
|
"mean": 0.09802824235521257,
|
|
"normalized": [
|
|
0.9452760533712127,
|
|
2.0518237767482,
|
|
1.4647344044819621,
|
|
1.7384605557023458,
|
|
0.5304523157771692,
|
|
0.739521347144304,
|
|
0.37963725912400215,
|
|
0.7499069723601224,
|
|
0.5259929839123919,
|
|
0.9117248861979151,
|
|
0.6373923716420872,
|
|
0.9725704599356962,
|
|
0.5583063049509795,
|
|
0.9102311711526044,
|
|
0.8366261258645934,
|
|
2.0473430116344136
|
|
],
|
|
"population_cv": 0.5182803671925184
|
|
},
|
|
"bits_per_byte": 2.0687232408845437,
|
|
"branch_output_rms_by_sublayer": [
|
|
0.04482239484786987,
|
|
0.07626598328351974,
|
|
0.10980217158794403,
|
|
0.05728650838136673,
|
|
0.14111557602882385,
|
|
0.014705370180308819,
|
|
0.04088159278035164,
|
|
0.014925715513527393,
|
|
0.04827577620744705,
|
|
0.015019712969660759,
|
|
0.04055904969573021,
|
|
0.02146836742758751,
|
|
0.028491884469985962,
|
|
0.0222648773342371,
|
|
0.0389801524579525,
|
|
0.027360456064343452,
|
|
0.036527831107378006,
|
|
0.03173159062862396,
|
|
0.03612164035439491,
|
|
0.0401885062456131,
|
|
0.04509544372558594,
|
|
0.040587812662124634,
|
|
0.02791302651166916,
|
|
0.05573032796382904,
|
|
0.02739257924258709,
|
|
0.044091735035181046,
|
|
0.017023129388689995,
|
|
0.05625532940030098,
|
|
0.02216578833758831,
|
|
0.07780560851097107,
|
|
0.0160520039498806,
|
|
0.13296744227409363
|
|
],
|
|
"capture": {
|
|
"all_gradients_finite": true,
|
|
"all_gradients_present": true,
|
|
"count": 16,
|
|
"dtypes": [
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32"
|
|
],
|
|
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
|
"shape": [
|
|
16,
|
|
256,
|
|
192
|
|
],
|
|
"storage_unique": true
|
|
},
|
|
"core_parameter_grad_rms_by_block": [
|
|
0.0003765937729119843,
|
|
0.0003149609092301355,
|
|
0.00027693435492303714,
|
|
0.0001635333627570268,
|
|
0.000180917699642147,
|
|
0.00017837059454152616,
|
|
0.000200147742296221,
|
|
0.00024611271555124715,
|
|
0.0002957900398296376,
|
|
0.0002986151161329657,
|
|
0.0009593337211615513,
|
|
0.0007552262449127961,
|
|
0.0007605792150238727,
|
|
0.0005098371211048817,
|
|
0.00029447700210242914,
|
|
0.0002817503499702034
|
|
],
|
|
"core_parameter_grad_statistics": {
|
|
"first_quartile_mean": 0.00028300559995554593,
|
|
"first_to_last_ratio": 0.6130161476493403,
|
|
"imbalance_abs_log_ratio": 0.48936400138742103,
|
|
"last_quartile_mean": 0.00046166092205034674,
|
|
"mean": 0.00038082374763072894,
|
|
"normalized": [
|
|
0.9888925657996352,
|
|
0.827051651031862,
|
|
0.727198229222749,
|
|
0.4294200762805349,
|
|
0.47506937465878935,
|
|
0.46838096534485474,
|
|
0.5255652872002538,
|
|
0.6462640974530133,
|
|
0.7767111207477916,
|
|
0.7841294509357175,
|
|
2.5191016241239836,
|
|
1.9831385243472572,
|
|
1.997194817171378,
|
|
1.3387744968027897,
|
|
0.7732632324914067,
|
|
0.7398444863879827
|
|
],
|
|
"population_cv": 0.6091669692594756
|
|
},
|
|
"depth_weights": [
|
|
{
|
|
"entropy_mean": 0.0,
|
|
"mean_weights": [
|
|
1.0
|
|
],
|
|
"sources": 1
|
|
},
|
|
{
|
|
"entropy_mean": 0.573702335357666,
|
|
"mean_weights": [
|
|
0.7102901935577393,
|
|
0.28970980644226074
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6441590189933777,
|
|
"mean_weights": [
|
|
0.564466118812561,
|
|
0.43553388118743896
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.3999374806880951,
|
|
"mean_weights": [
|
|
0.8575316667556763,
|
|
0.14246836304664612
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.676960825920105,
|
|
"mean_weights": [
|
|
0.46661365032196045,
|
|
0.5333863496780396
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.9920796155929565,
|
|
"mean_weights": [
|
|
0.5238763093948364,
|
|
0.2048875242471695,
|
|
0.2712361812591553
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.092700719833374,
|
|
"mean_weights": [
|
|
0.2899855673313141,
|
|
0.34371256828308105,
|
|
0.36630189418792725
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 0.9979273080825806,
|
|
"mean_weights": [
|
|
0.49421679973602295,
|
|
0.18891796469688416,
|
|
0.3168652057647705
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.091429591178894,
|
|
"mean_weights": [
|
|
0.3115330636501312,
|
|
0.3673863708972931,
|
|
0.3210805654525757
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.2927134037017822,
|
|
"mean_weights": [
|
|
0.35746461153030396,
|
|
0.11402665078639984,
|
|
0.22846516966819763,
|
|
0.3000435531139374
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.378767490386963,
|
|
"mean_weights": [
|
|
0.2549898624420166,
|
|
0.23654252290725708,
|
|
0.21668663620948792,
|
|
0.2917810082435608
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.2672696113586426,
|
|
"mean_weights": [
|
|
0.3839718699455261,
|
|
0.12064628303050995,
|
|
0.1715085655450821,
|
|
0.32387328147888184
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.375225305557251,
|
|
"mean_weights": [
|
|
0.23853997886180878,
|
|
0.24968335032463074,
|
|
0.2089165896177292,
|
|
0.3028600811958313
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.5798927545547485,
|
|
"mean_weights": [
|
|
0.22726407647132874,
|
|
0.12221817672252655,
|
|
0.18549029529094696,
|
|
0.23680329322814941,
|
|
0.22822415828704834
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.594498634338379,
|
|
"mean_weights": [
|
|
0.21032124757766724,
|
|
0.19536492228507996,
|
|
0.1600019335746765,
|
|
0.1794220358133316,
|
|
0.2548898458480835
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.5424928665161133,
|
|
"mean_weights": [
|
|
0.23003876209259033,
|
|
0.11985675245523453,
|
|
0.1379462629556656,
|
|
0.19240085780620575,
|
|
0.319757342338562
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.595255732536316,
|
|
"mean_weights": [
|
|
0.2046268880367279,
|
|
0.19130583107471466,
|
|
0.17092418670654297,
|
|
0.17953911423683167,
|
|
0.253603994846344
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.7457678318023682,
|
|
"mean_weights": [
|
|
0.17887650430202484,
|
|
0.11389653384685516,
|
|
0.11488981544971466,
|
|
0.1474636197090149,
|
|
0.20503896474838257,
|
|
0.23983454704284668
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7712774276733398,
|
|
"mean_weights": [
|
|
0.16810789704322815,
|
|
0.16247668862342834,
|
|
0.1383969932794571,
|
|
0.13383737206459045,
|
|
0.1661752164363861,
|
|
0.23100584745407104
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.675686001777649,
|
|
"mean_weights": [
|
|
0.16882894933223724,
|
|
0.11588791012763977,
|
|
0.08043210208415985,
|
|
0.1250714659690857,
|
|
0.19613179564476013,
|
|
0.3136477470397949
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7837879657745361,
|
|
"mean_weights": [
|
|
0.17464125156402588,
|
|
0.18035069108009338,
|
|
0.1619444340467453,
|
|
0.15330034494400024,
|
|
0.15676669776439667,
|
|
0.17299658060073853
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.6526988744735718,
|
|
"mean_weights": [
|
|
0.09642067551612854,
|
|
0.07173497974872589,
|
|
0.05561239272356033,
|
|
0.09171071648597717,
|
|
0.11456114053726196,
|
|
0.12951162457466125,
|
|
0.44044846296310425
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.8849880695343018,
|
|
"mean_weights": [
|
|
0.13606244325637817,
|
|
0.13500544428825378,
|
|
0.12342768162488937,
|
|
0.11247340589761734,
|
|
0.11233137547969818,
|
|
0.11334770917892456,
|
|
0.2673519253730774
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.644601583480835,
|
|
"mean_weights": [
|
|
0.10291393101215363,
|
|
0.08116044104099274,
|
|
0.05550362914800644,
|
|
0.08702410012483597,
|
|
0.11089185625314713,
|
|
0.11528079956769943,
|
|
0.4472252428531647
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.9121408462524414,
|
|
"mean_weights": [
|
|
0.14125728607177734,
|
|
0.1494411677122116,
|
|
0.12786200642585754,
|
|
0.11922307312488556,
|
|
0.1195257231593132,
|
|
0.11599892377853394,
|
|
0.2266918122768402
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.7500674724578857,
|
|
"mean_weights": [
|
|
0.07664570212364197,
|
|
0.0604618638753891,
|
|
0.041497331112623215,
|
|
0.06405743956565857,
|
|
0.0716431736946106,
|
|
0.07514288276433945,
|
|
0.270142525434494,
|
|
0.3404091000556946
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.952926754951477,
|
|
"mean_weights": [
|
|
0.10562807321548462,
|
|
0.10351885855197906,
|
|
0.07600137591362,
|
|
0.08402621746063232,
|
|
0.08850346505641937,
|
|
0.09127135574817657,
|
|
0.15880608558654785,
|
|
0.2922445833683014
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.7527103424072266,
|
|
"mean_weights": [
|
|
0.08054207265377045,
|
|
0.06648321449756622,
|
|
0.04810335487127304,
|
|
0.06408451497554779,
|
|
0.07014857977628708,
|
|
0.07217346131801605,
|
|
0.20459653437137604,
|
|
0.39386826753616333
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.9524379968643188,
|
|
"mean_weights": [
|
|
0.10777557641267776,
|
|
0.09555133432149887,
|
|
0.06806209683418274,
|
|
0.08129536360502243,
|
|
0.0918373167514801,
|
|
0.10075562447309494,
|
|
0.2000727653503418,
|
|
0.25464993715286255
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 2.000408887863159,
|
|
"mean_weights": [
|
|
0.07838036119937897,
|
|
0.07984089851379395,
|
|
0.04941036179661751,
|
|
0.06521518528461456,
|
|
0.06765615195035934,
|
|
0.06713232398033142,
|
|
0.18892306089401245,
|
|
0.23950761556625366,
|
|
0.16393405199050903
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.096604347229004,
|
|
"mean_weights": [
|
|
0.09755367040634155,
|
|
0.08867941796779633,
|
|
0.060870569199323654,
|
|
0.06980546563863754,
|
|
0.08883253484964371,
|
|
0.09541568160057068,
|
|
0.17317476868629456,
|
|
0.19020704925060272,
|
|
0.13546085357666016
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.027402877807617,
|
|
"mean_weights": [
|
|
0.08629745244979858,
|
|
0.08307671546936035,
|
|
0.05121873319149017,
|
|
0.0663638487458229,
|
|
0.07480055093765259,
|
|
0.08018200099468231,
|
|
0.15043827891349792,
|
|
0.25634968280792236,
|
|
0.1512727439403534
|
|
],
|
|
"sources": 9
|
|
}
|
|
],
|
|
"layer_input_rms_by_sublayer": [
|
|
0.055510263890028,
|
|
0.04334520921111107,
|
|
0.05630175396800041,
|
|
0.058260008692741394,
|
|
0.11489111185073853,
|
|
0.08273965120315552,
|
|
0.11550594866275787,
|
|
0.09142354130744934,
|
|
0.12066955119371414,
|
|
0.0711931437253952,
|
|
0.08880246430635452,
|
|
0.06823744624853134,
|
|
0.09300445020198822,
|
|
0.06510575115680695,
|
|
0.07136023044586182,
|
|
0.05894701927900314,
|
|
0.07342072576284409,
|
|
0.05002941936254501,
|
|
0.059311795979738235,
|
|
0.04801420122385025,
|
|
0.06730704009532928,
|
|
0.02817549556493759,
|
|
0.046817973256111145,
|
|
0.03746112436056137,
|
|
0.05246857553720474,
|
|
0.03106791339814663,
|
|
0.03552677109837532,
|
|
0.03383694216609001,
|
|
0.03949129208922386,
|
|
0.03412646800279617,
|
|
0.039247769862413406,
|
|
0.04047730192542076
|
|
],
|
|
"loss_nats": 1.433929681777954,
|
|
"loss_scale": 1.0,
|
|
"output_weights": {
|
|
"entropy_mean": 1.913235068321228,
|
|
"mean_weights": [
|
|
0.09498223662376404,
|
|
0.06303347647190094,
|
|
0.06038350611925125,
|
|
0.06479769200086594,
|
|
0.05589054524898529,
|
|
0.06028280779719353,
|
|
0.07675591856241226,
|
|
0.2476816177368164,
|
|
0.27619218826293945
|
|
],
|
|
"sources": 9
|
|
},
|
|
"step": 2000,
|
|
"stream_state_rms_by_sublayer": [
|
|
0.04482239484786987,
|
|
0.09266375005245209,
|
|
0.17245197296142578,
|
|
0.20113667845726013,
|
|
0.14111557602882385,
|
|
0.14358533918857574,
|
|
0.16786204278469086,
|
|
0.17041823267936707,
|
|
0.04827577620744705,
|
|
0.05199930816888809,
|
|
0.06626670062541962,
|
|
0.07249397784471512,
|
|
0.028491884469985962,
|
|
0.03721517324447632,
|
|
0.05938867852091789,
|
|
0.07351206243038177,
|
|
0.036527831107378006,
|
|
0.05156216770410538,
|
|
0.066385917365551,
|
|
0.08937478810548782,
|
|
0.04509544372558594,
|
|
0.062482453882694244,
|
|
0.07032289355993271,
|
|
0.09533937275409698,
|
|
0.02739257924258709,
|
|
0.05472978577017784,
|
|
0.058526892215013504,
|
|
0.08922836184501648,
|
|
0.02216578833758831,
|
|
0.08201298862695694,
|
|
0.0856289193034172,
|
|
0.20069743692874908
|
|
]
|
|
},
|
|
{
|
|
"activation_grad_rms_by_block": [
|
|
5.492530908668414e-05,
|
|
5.6656335800653324e-05,
|
|
6.325786671368405e-05,
|
|
5.9555983170866966e-05,
|
|
6.814395601395518e-05,
|
|
6.255660991882905e-05,
|
|
7.046976679703221e-05,
|
|
6.272071914281696e-05,
|
|
7.296914554899558e-05,
|
|
6.464503530878574e-05,
|
|
0.00014864247350487858,
|
|
0.00010719253623392433,
|
|
0.0001445752423023805,
|
|
9.124213829636574e-05,
|
|
6.016199040459469e-05,
|
|
4.548503056867048e-05
|
|
],
|
|
"activation_grad_statistics": {
|
|
"first_quartile_mean": 5.859887369297212e-05,
|
|
"first_to_last_ratio": 0.6864419649392262,
|
|
"imbalance_abs_log_ratio": 0.37623359494524666,
|
|
"last_quartile_mean": 8.536610039300285e-05,
|
|
"mean": 7.707500867581984e-05,
|
|
"normalized": [
|
|
0.7126215102706234,
|
|
0.7350804985173837,
|
|
0.8207312305308824,
|
|
0.772701608395031,
|
|
0.8841251812318441,
|
|
0.8116328625007921,
|
|
0.9143011205283218,
|
|
0.8137620769739057,
|
|
0.9467289955931933,
|
|
0.8387288748897194,
|
|
1.9285430654970663,
|
|
1.3907560709435642,
|
|
1.8757732861305145,
|
|
1.18380964029642,
|
|
0.7805641729816484,
|
|
0.5901398047190899
|
|
],
|
|
"population_cv": 0.3863401678553132
|
|
},
|
|
"activation_output_rms_by_block": [
|
|
0.14486487209796906,
|
|
0.30995267629623413,
|
|
0.2574450671672821,
|
|
0.28713259100914,
|
|
0.05978552997112274,
|
|
0.08792455494403839,
|
|
0.04697949066758156,
|
|
0.09852420538663864,
|
|
0.07244286686182022,
|
|
0.13140366971492767,
|
|
0.08212646842002869,
|
|
0.12985831499099731,
|
|
0.06840891391038895,
|
|
0.11060241609811783,
|
|
0.0924442708492279,
|
|
0.23933660984039307
|
|
],
|
|
"activation_output_statistics": {
|
|
"first_quartile_mean": 0.24984880164265633,
|
|
"first_to_last_ratio": 1.9565592145673032,
|
|
"imbalance_abs_log_ratio": 0.6711874277699096,
|
|
"last_quartile_mean": 0.12769805267453194,
|
|
"mean": 0.13870203238911927,
|
|
"normalized": [
|
|
1.0444322235420485,
|
|
2.234665714390418,
|
|
1.8561016210998063,
|
|
2.0701397525568246,
|
|
0.43103571693454673,
|
|
0.6339096365752733,
|
|
0.3387080193301259,
|
|
0.7103299330916477,
|
|
0.5222913147990983,
|
|
0.9473809968860692,
|
|
0.5921071739571081,
|
|
0.9362394534110972,
|
|
0.4932077254533107,
|
|
0.7974102051210767,
|
|
0.6664954309384717,
|
|
1.725545081913077
|
|
],
|
|
"population_cv": 0.5979328649415788
|
|
},
|
|
"bits_per_byte": 1.9415087059372296,
|
|
"branch_output_rms_by_sublayer": [
|
|
0.06868160516023636,
|
|
0.12181960046291351,
|
|
0.16538235545158386,
|
|
0.0929274708032608,
|
|
0.2537953853607178,
|
|
0.01320737786591053,
|
|
0.04932497814297676,
|
|
0.013867277652025223,
|
|
0.055594079196453094,
|
|
0.0172379482537508,
|
|
0.04591496288776398,
|
|
0.035032741725444794,
|
|
0.03727728873491287,
|
|
0.025992874056100845,
|
|
0.0513935461640358,
|
|
0.04214895889163017,
|
|
0.05297093465924263,
|
|
0.04459983855485916,
|
|
0.05040198564529419,
|
|
0.06986947357654572,
|
|
0.054249413311481476,
|
|
0.05820845440030098,
|
|
0.03775457665324211,
|
|
0.08249213546514511,
|
|
0.03362919017672539,
|
|
0.05606101080775261,
|
|
0.02498369663953781,
|
|
0.07096737623214722,
|
|
0.02770768664777279,
|
|
0.08668208867311478,
|
|
0.022235019132494926,
|
|
0.16932900249958038
|
|
],
|
|
"capture": {
|
|
"all_gradients_finite": true,
|
|
"all_gradients_present": true,
|
|
"count": 16,
|
|
"dtypes": [
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32"
|
|
],
|
|
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
|
"shape": [
|
|
16,
|
|
256,
|
|
192
|
|
],
|
|
"storage_unique": true
|
|
},
|
|
"core_parameter_grad_rms_by_block": [
|
|
0.0003274625120219605,
|
|
0.0002950920627246982,
|
|
0.0002675536202763891,
|
|
0.00011636112532418193,
|
|
0.00013949143839672265,
|
|
0.0001705676155207084,
|
|
0.00017763939173179324,
|
|
0.00023071980568273914,
|
|
0.00033685005198124334,
|
|
0.0003711494248343089,
|
|
0.0010020275600187679,
|
|
0.0008489747430264115,
|
|
0.0009447094037588545,
|
|
0.0006663788253529681,
|
|
0.0003910019531118643,
|
|
0.00034755869426738895
|
|
],
|
|
"core_parameter_grad_statistics": {
|
|
"first_quartile_mean": 0.00025161733008680743,
|
|
"first_to_last_ratio": 0.4283488185904071,
|
|
"imbalance_abs_log_ratio": 0.8478174186232621,
|
|
"last_quartile_mean": 0.000587412219122769,
|
|
"mean": 0.00041459613925193755,
|
|
"normalized": [
|
|
0.789834928547107,
|
|
0.7117578645501561,
|
|
0.6453355324512677,
|
|
0.2806613817826046,
|
|
0.3364513684290676,
|
|
0.4114066663246461,
|
|
0.4284636901161473,
|
|
0.5564928947457894,
|
|
0.8124775416120065,
|
|
0.895207141832,
|
|
2.4168762444984226,
|
|
2.047715023488232,
|
|
2.278625665601732,
|
|
1.607296263190791,
|
|
0.9430911581023291,
|
|
0.8383066347277007
|
|
],
|
|
"population_cv": 0.6738437321539833
|
|
},
|
|
"depth_weights": [
|
|
{
|
|
"entropy_mean": 0.0,
|
|
"mean_weights": [
|
|
1.0
|
|
],
|
|
"sources": 1
|
|
},
|
|
{
|
|
"entropy_mean": 0.4194117784500122,
|
|
"mean_weights": [
|
|
0.8254866600036621,
|
|
0.1745133101940155
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.5809611082077026,
|
|
"mean_weights": [
|
|
0.6678193211555481,
|
|
0.3321806788444519
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.31704896688461304,
|
|
"mean_weights": [
|
|
0.8999109268188477,
|
|
0.10008908808231354
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6732906103134155,
|
|
"mean_weights": [
|
|
0.4609490633010864,
|
|
0.5390509366989136
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.9927794337272644,
|
|
"mean_weights": [
|
|
0.4832823872566223,
|
|
0.17606672644615173,
|
|
0.34065088629722595
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0832945108413696,
|
|
"mean_weights": [
|
|
0.2762538492679596,
|
|
0.3156396150588989,
|
|
0.40810656547546387
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 0.9905548095703125,
|
|
"mean_weights": [
|
|
0.4605642259120941,
|
|
0.16346418857574463,
|
|
0.37597161531448364
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.09372079372406,
|
|
"mean_weights": [
|
|
0.32242417335510254,
|
|
0.34579548239707947,
|
|
0.3317803740501404
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.2746227979660034,
|
|
"mean_weights": [
|
|
0.37224799394607544,
|
|
0.09596331417560577,
|
|
0.2427763193845749,
|
|
0.2890123426914215
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3707606792449951,
|
|
"mean_weights": [
|
|
0.27432072162628174,
|
|
0.2153838574886322,
|
|
0.20746882259845734,
|
|
0.30282658338546753
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.198214054107666,
|
|
"mean_weights": [
|
|
0.44253695011138916,
|
|
0.10026887059211731,
|
|
0.13153521716594696,
|
|
0.325658917427063
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.372360110282898,
|
|
"mean_weights": [
|
|
0.24982473254203796,
|
|
0.2343549281358719,
|
|
0.2051662802696228,
|
|
0.31065404415130615
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.5764262676239014,
|
|
"mean_weights": [
|
|
0.2228686511516571,
|
|
0.11472160369157791,
|
|
0.20744341611862183,
|
|
0.22938773036003113,
|
|
0.22557863593101501
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.5863620042800903,
|
|
"mean_weights": [
|
|
0.2179623544216156,
|
|
0.17305074632167816,
|
|
0.15634828805923462,
|
|
0.18247520923614502,
|
|
0.2701634168624878
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.4903430938720703,
|
|
"mean_weights": [
|
|
0.2547823488712311,
|
|
0.09995707869529724,
|
|
0.10266982018947601,
|
|
0.1966245472431183,
|
|
0.3459661602973938
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.5923073291778564,
|
|
"mean_weights": [
|
|
0.22257764637470245,
|
|
0.17832019925117493,
|
|
0.1619993895292282,
|
|
0.1890055239200592,
|
|
0.2480972558259964
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.691253662109375,
|
|
"mean_weights": [
|
|
0.18423792719841003,
|
|
0.09657466411590576,
|
|
0.08509969711303711,
|
|
0.15257084369659424,
|
|
0.18859976530075073,
|
|
0.29291707277297974
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7595478296279907,
|
|
"mean_weights": [
|
|
0.1724337935447693,
|
|
0.14752474427223206,
|
|
0.13264453411102295,
|
|
0.13973915576934814,
|
|
0.15476800501346588,
|
|
0.2528897523880005
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.5512144565582275,
|
|
"mean_weights": [
|
|
0.16704371571540833,
|
|
0.08849328011274338,
|
|
0.04574323818087578,
|
|
0.12566779553890228,
|
|
0.1718127727508545,
|
|
0.40123921632766724
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7799327373504639,
|
|
"mean_weights": [
|
|
0.18437626957893372,
|
|
0.17012780904769897,
|
|
0.17119532823562622,
|
|
0.15378496050834656,
|
|
0.1488233208656311,
|
|
0.17169234156608582
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.5764116048812866,
|
|
"mean_weights": [
|
|
0.09582588821649551,
|
|
0.05901939421892166,
|
|
0.040849827229976654,
|
|
0.09081389009952545,
|
|
0.10021352022886276,
|
|
0.13485172390937805,
|
|
0.4784257411956787
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.8431057929992676,
|
|
"mean_weights": [
|
|
0.13672879338264465,
|
|
0.12035658955574036,
|
|
0.11983893811702728,
|
|
0.10482897609472275,
|
|
0.10095320641994476,
|
|
0.10664592683315277,
|
|
0.3106476068496704
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.5526533126831055,
|
|
"mean_weights": [
|
|
0.10306400060653687,
|
|
0.06465981900691986,
|
|
0.04082629829645157,
|
|
0.08607284724712372,
|
|
0.09878605604171753,
|
|
0.11178258061408997,
|
|
0.4948084056377411
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.8949611186981201,
|
|
"mean_weights": [
|
|
0.1451701521873474,
|
|
0.13929708302021027,
|
|
0.1330569088459015,
|
|
0.11840799450874329,
|
|
0.10893036425113678,
|
|
0.10698798298835754,
|
|
0.24814949929714203
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.676889419555664,
|
|
"mean_weights": [
|
|
0.07736113667488098,
|
|
0.04877929389476776,
|
|
0.032506294548511505,
|
|
0.06437233090400696,
|
|
0.06250663101673126,
|
|
0.06869478523731232,
|
|
0.2701185345649719,
|
|
0.3756609857082367
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.8798964023590088,
|
|
"mean_weights": [
|
|
0.10042576491832733,
|
|
0.08721707761287689,
|
|
0.06748545169830322,
|
|
0.0845440924167633,
|
|
0.07678259164094925,
|
|
0.08073122054338455,
|
|
0.1554347574710846,
|
|
0.34737905859947205
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.663856029510498,
|
|
"mean_weights": [
|
|
0.07920834422111511,
|
|
0.05299394577741623,
|
|
0.03748705983161926,
|
|
0.06209969520568848,
|
|
0.060945168137550354,
|
|
0.06926221400499344,
|
|
0.19488413631916046,
|
|
0.44311946630477905
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.9090958833694458,
|
|
"mean_weights": [
|
|
0.1065661758184433,
|
|
0.0803120881319046,
|
|
0.06028372794389725,
|
|
0.08741366118192673,
|
|
0.08287163078784943,
|
|
0.09473372995853424,
|
|
0.1887454390525818,
|
|
0.29907354712486267
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.8632590770721436,
|
|
"mean_weights": [
|
|
0.0691547840833664,
|
|
0.05765335261821747,
|
|
0.03405343368649483,
|
|
0.05519368499517441,
|
|
0.05223262310028076,
|
|
0.057386673986911774,
|
|
0.1615200936794281,
|
|
0.25146448612213135,
|
|
0.2613408863544464
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.0701708793640137,
|
|
"mean_weights": [
|
|
0.09429951012134552,
|
|
0.07856893539428711,
|
|
0.04940951615571976,
|
|
0.076503224670887,
|
|
0.08159374445676804,
|
|
0.0895766168832779,
|
|
0.16711056232452393,
|
|
0.2026878148317337,
|
|
0.16025006771087646
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 1.9422186613082886,
|
|
"mean_weights": [
|
|
0.07884211093187332,
|
|
0.06442099809646606,
|
|
0.037228696048259735,
|
|
0.06615173816680908,
|
|
0.06501315534114838,
|
|
0.0695233941078186,
|
|
0.13319489359855652,
|
|
0.2805612087249756,
|
|
0.20506379008293152
|
|
],
|
|
"sources": 9
|
|
}
|
|
],
|
|
"layer_input_rms_by_sublayer": [
|
|
0.05865398049354553,
|
|
0.05170911177992821,
|
|
0.06799201667308807,
|
|
0.06424286961555481,
|
|
0.17415167391300201,
|
|
0.13562268018722534,
|
|
0.1871119737625122,
|
|
0.1505064219236374,
|
|
0.18585707247257233,
|
|
0.1044568344950676,
|
|
0.12573961913585663,
|
|
0.08286763727664948,
|
|
0.13412274420261383,
|
|
0.0987003743648529,
|
|
0.09964247792959213,
|
|
0.07276491075754166,
|
|
0.1041010394692421,
|
|
0.06199917569756508,
|
|
0.0827898308634758,
|
|
0.06073001027107239,
|
|
0.10040723532438278,
|
|
0.036688607186079025,
|
|
0.0660671666264534,
|
|
0.05293381214141846,
|
|
0.07866564393043518,
|
|
0.04279600828886032,
|
|
0.04821427911520004,
|
|
0.04622510448098183,
|
|
0.053889211267232895,
|
|
0.04172775521874428,
|
|
0.05102568119764328,
|
|
0.053032174706459045
|
|
],
|
|
"loss_nats": 1.3457512855529785,
|
|
"loss_scale": 1.0,
|
|
"output_weights": {
|
|
"entropy_mean": 1.7949464321136475,
|
|
"mean_weights": [
|
|
0.0834353119134903,
|
|
0.05303390324115753,
|
|
0.048570070415735245,
|
|
0.05988207459449768,
|
|
0.04784438759088516,
|
|
0.05165069177746773,
|
|
0.06509792059659958,
|
|
0.2334408313035965,
|
|
0.3570447862148285
|
|
],
|
|
"sources": 9
|
|
},
|
|
"step": 4000,
|
|
"stream_state_rms_by_sublayer": [
|
|
0.06868160516023636,
|
|
0.14486487209796906,
|
|
0.2633455693721771,
|
|
0.30995267629623413,
|
|
0.2537953853607178,
|
|
0.2574450671672821,
|
|
0.28461524844169617,
|
|
0.28713259100914,
|
|
0.055594079196453094,
|
|
0.05978552997112274,
|
|
0.07737205922603607,
|
|
0.08792455494403839,
|
|
0.03727728873491287,
|
|
0.04697949066758156,
|
|
0.07717122882604599,
|
|
0.09852420538663864,
|
|
0.05297093465924263,
|
|
0.07244286686182022,
|
|
0.09201814234256744,
|
|
0.13140366971492767,
|
|
0.054249413311481476,
|
|
0.08212646842002869,
|
|
0.09215818345546722,
|
|
0.12985831499099731,
|
|
0.03362919017672539,
|
|
0.06840891391038895,
|
|
0.07437368482351303,
|
|
0.11060241609811783,
|
|
0.02770768664777279,
|
|
0.0924442708492279,
|
|
0.09714528173208237,
|
|
0.23933660984039307
|
|
]
|
|
},
|
|
{
|
|
"activation_grad_rms_by_block": [
|
|
6.899879372213036e-05,
|
|
7.050983549561352e-05,
|
|
8.400688238907605e-05,
|
|
8.067955786827952e-05,
|
|
9.6201452834066e-05,
|
|
8.90634983079508e-05,
|
|
0.00010480848868610337,
|
|
9.241609222954139e-05,
|
|
0.00012084992340533063,
|
|
0.00010282631410518661,
|
|
0.00023664520995225757,
|
|
0.0001637248496990651,
|
|
0.00023947370937094092,
|
|
0.0001411983830621466,
|
|
0.00010364947956986725,
|
|
6.421437865355983e-05
|
|
],
|
|
"activation_grad_statistics": {
|
|
"first_quartile_mean": 7.604876736877486e-05,
|
|
"first_to_last_ratio": 0.5545581271583456,
|
|
"imbalance_abs_log_ratio": 0.5895836496274593,
|
|
"last_quartile_mean": 0.00013713398766412865,
|
|
"mean": 0.00011620417808444472,
|
|
"normalized": [
|
|
0.5937720558721171,
|
|
0.6067753901617422,
|
|
0.7229248016196876,
|
|
0.6942913688494942,
|
|
0.8278656965686476,
|
|
0.7664397251123708,
|
|
0.9019339098972828,
|
|
0.7952906147865294,
|
|
1.0399791590755874,
|
|
0.8848762221825061,
|
|
2.0364604255476015,
|
|
1.4089411619958057,
|
|
2.060801197672231,
|
|
1.2150886946555293,
|
|
0.891960007621636,
|
|
0.5525995683812308
|
|
],
|
|
"population_cv": 0.4517546138899352
|
|
},
|
|
"activation_output_rms_by_block": [
|
|
0.15090593695640564,
|
|
0.3367030918598175,
|
|
0.29793402552604675,
|
|
0.3255155086517334,
|
|
0.05378299206495285,
|
|
0.08782409131526947,
|
|
0.044081781059503555,
|
|
0.09709836542606354,
|
|
0.0669482871890068,
|
|
0.12247904390096664,
|
|
0.07360157370567322,
|
|
0.12167256325483322,
|
|
0.06349270045757294,
|
|
0.10058031976222992,
|
|
0.07551935315132141,
|
|
0.19356463849544525
|
|
],
|
|
"activation_output_statistics": {
|
|
"first_quartile_mean": 0.2777646407485008,
|
|
"first_to_last_ratio": 2.565024996839381,
|
|
"imbalance_abs_log_ratio": 0.941968223931731,
|
|
"last_quartile_mean": 0.10828925296664238,
|
|
"mean": 0.13823151704855263,
|
|
"normalized": [
|
|
1.0916897982346618,
|
|
2.4357910485895444,
|
|
2.155326309711265,
|
|
2.3548573842056504,
|
|
0.38907908422984344,
|
|
0.6353405734845695,
|
|
0.3188981934128684,
|
|
0.7024329002477675,
|
|
0.4843199916954669,
|
|
0.8860428252259355,
|
|
0.5324514645948746,
|
|
0.8802085505008006,
|
|
0.45932144718683565,
|
|
0.7276221943430017,
|
|
0.5463251417894509,
|
|
1.400293092547464
|
|
],
|
|
"population_cv": 0.6866316887805146
|
|
},
|
|
"bits_per_byte": 1.8676321823979032,
|
|
"branch_output_rms_by_sublayer": [
|
|
0.07169820368289948,
|
|
0.12625248730182648,
|
|
0.19025787711143494,
|
|
0.09543261677026749,
|
|
0.2934682369232178,
|
|
0.013759235851466656,
|
|
0.04595690965652466,
|
|
0.014787186868488789,
|
|
0.048878561705350876,
|
|
0.017967192456126213,
|
|
0.039782870560884476,
|
|
0.03611106425523758,
|
|
0.03420666605234146,
|
|
0.025706099346280098,
|
|
0.046541497111320496,
|
|
0.04672740399837494,
|
|
0.04840917885303497,
|
|
0.04226908087730408,
|
|
0.0452793650329113,
|
|
0.07270561158657074,
|
|
0.04624715819954872,
|
|
0.05458534136414528,
|
|
0.0338163822889328,
|
|
0.0825076773762703,
|
|
0.03031807579100132,
|
|
0.0527048334479332,
|
|
0.022609181702136993,
|
|
0.06718777120113373,
|
|
0.02396680787205696,
|
|
0.07009968906641006,
|
|
0.018549330532550812,
|
|
0.1417386531829834
|
|
],
|
|
"capture": {
|
|
"all_gradients_finite": true,
|
|
"all_gradients_present": true,
|
|
"count": 16,
|
|
"dtypes": [
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32",
|
|
"torch.float32"
|
|
],
|
|
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
|
|
"shape": [
|
|
16,
|
|
256,
|
|
192
|
|
],
|
|
"storage_unique": true
|
|
},
|
|
"core_parameter_grad_rms_by_block": [
|
|
0.00046247293295990515,
|
|
0.0003824975682884986,
|
|
0.00041690386457328876,
|
|
0.00021732383526524047,
|
|
0.00022655054548791778,
|
|
0.0002917516444309179,
|
|
0.000280629308661355,
|
|
0.0003847878626551635,
|
|
0.0005123546145807317,
|
|
0.0005899763687075028,
|
|
0.0015400377759773427,
|
|
0.0012824907104132966,
|
|
0.001537981149104749,
|
|
0.001057571910144345,
|
|
0.0006563117026664665,
|
|
0.0005021677227557722
|
|
],
|
|
"core_parameter_grad_statistics": {
|
|
"first_quartile_mean": 0.00036979955027173326,
|
|
"first_to_last_ratio": 0.394029142562531,
|
|
"imbalance_abs_log_ratio": 0.9313304065247964,
|
|
"last_quartile_mean": 0.0009385081211678332,
|
|
"mean": 0.0006463630947920308,
|
|
"normalized": [
|
|
0.7155002144865761,
|
|
0.5917688855852271,
|
|
0.6449994870258314,
|
|
0.3362256246006202,
|
|
0.3505004343739815,
|
|
0.45137423033842894,
|
|
0.434166664097133,
|
|
0.595312241301416,
|
|
0.792673063652533,
|
|
0.9127630792369563,
|
|
2.3826202151483513,
|
|
1.9841645055956383,
|
|
2.3794383706260316,
|
|
1.6361885736756394,
|
|
1.0153916706485797,
|
|
0.7769127396070564
|
|
],
|
|
"population_cv": 0.675897465620864
|
|
},
|
|
"depth_weights": [
|
|
{
|
|
"entropy_mean": 0.0,
|
|
"mean_weights": [
|
|
1.0
|
|
],
|
|
"sources": 1
|
|
},
|
|
{
|
|
"entropy_mean": 0.43482720851898193,
|
|
"mean_weights": [
|
|
0.8150075674057007,
|
|
0.18499243259429932
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.5611247420310974,
|
|
"mean_weights": [
|
|
0.6986296772956848,
|
|
0.3013703525066376
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.31192272901535034,
|
|
"mean_weights": [
|
|
0.9031083583831787,
|
|
0.09689167141914368
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.6764674186706543,
|
|
"mean_weights": [
|
|
0.4619978070259094,
|
|
0.5380021929740906
|
|
],
|
|
"sources": 2
|
|
},
|
|
{
|
|
"entropy_mean": 0.9888706207275391,
|
|
"mean_weights": [
|
|
0.48646074533462524,
|
|
0.1758481115102768,
|
|
0.33769115805625916
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0781972408294678,
|
|
"mean_weights": [
|
|
0.276906281709671,
|
|
0.2993447184562683,
|
|
0.42374899983406067
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 0.9839773178100586,
|
|
"mean_weights": [
|
|
0.46118396520614624,
|
|
0.15739500522613525,
|
|
0.3814210295677185
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.0957449674606323,
|
|
"mean_weights": [
|
|
0.3292894959449768,
|
|
0.32970887422561646,
|
|
0.3410016596317291
|
|
],
|
|
"sources": 3
|
|
},
|
|
{
|
|
"entropy_mean": 1.2506439685821533,
|
|
"mean_weights": [
|
|
0.3648650050163269,
|
|
0.0835169106721878,
|
|
0.212466761469841,
|
|
0.3391513228416443
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.3655824661254883,
|
|
"mean_weights": [
|
|
0.2861315608024597,
|
|
0.20811524987220764,
|
|
0.20031729340553284,
|
|
0.3054359257221222
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.1652674674987793,
|
|
"mean_weights": [
|
|
0.43905186653137207,
|
|
0.08880466222763062,
|
|
0.10791443288326263,
|
|
0.3642290234565735
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.363007664680481,
|
|
"mean_weights": [
|
|
0.2524264454841614,
|
|
0.21885012090206146,
|
|
0.19597390294075012,
|
|
0.33274954557418823
|
|
],
|
|
"sources": 4
|
|
},
|
|
{
|
|
"entropy_mean": 1.5649902820587158,
|
|
"mean_weights": [
|
|
0.22229184210300446,
|
|
0.10384112596511841,
|
|
0.1934116780757904,
|
|
0.2539443373680115,
|
|
0.22651101648807526
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.5797945261001587,
|
|
"mean_weights": [
|
|
0.21991753578186035,
|
|
0.1605391800403595,
|
|
0.15625253319740295,
|
|
0.18265283107757568,
|
|
0.2806379497051239
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.4139833450317383,
|
|
"mean_weights": [
|
|
0.24990326166152954,
|
|
0.08050692081451416,
|
|
0.06835947930812836,
|
|
0.20686665177345276,
|
|
0.394363671541214
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.5895863771438599,
|
|
"mean_weights": [
|
|
0.2244223803281784,
|
|
0.16505929827690125,
|
|
0.1643591672182083,
|
|
0.19251328706741333,
|
|
0.2536458969116211
|
|
],
|
|
"sources": 5
|
|
},
|
|
{
|
|
"entropy_mean": 1.6177464723587036,
|
|
"mean_weights": [
|
|
0.1738494634628296,
|
|
0.07661910355091095,
|
|
0.06195354461669922,
|
|
0.14467598497867584,
|
|
0.19003543257713318,
|
|
0.35286644101142883
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.7444759607315063,
|
|
"mean_weights": [
|
|
0.172578826546669,
|
|
0.1303727626800537,
|
|
0.13036483526229858,
|
|
0.13626831769943237,
|
|
0.1541597843170166,
|
|
0.27625542879104614
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.470452904701233,
|
|
"mean_weights": [
|
|
0.15901926159858704,
|
|
0.07292591035366058,
|
|
0.03418249264359474,
|
|
0.11740352213382721,
|
|
0.16174395382404327,
|
|
0.45472484827041626
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.778969645500183,
|
|
"mean_weights": [
|
|
0.18775978684425354,
|
|
0.15501554310321808,
|
|
0.17432880401611328,
|
|
0.15116485953330994,
|
|
0.14906319975852966,
|
|
0.1826678216457367
|
|
],
|
|
"sources": 6
|
|
},
|
|
{
|
|
"entropy_mean": 1.531482219696045,
|
|
"mean_weights": [
|
|
0.09255115687847137,
|
|
0.04949861764907837,
|
|
0.03270748257637024,
|
|
0.08605319261550903,
|
|
0.09672325849533081,
|
|
0.1471216380596161,
|
|
0.4953446388244629
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.8260389566421509,
|
|
"mean_weights": [
|
|
0.13673919439315796,
|
|
0.10493747889995575,
|
|
0.11646746844053268,
|
|
0.09983028471469879,
|
|
0.10169579833745956,
|
|
0.1137751042842865,
|
|
0.32655465602874756
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.4906136989593506,
|
|
"mean_weights": [
|
|
0.10088150203227997,
|
|
0.05257141590118408,
|
|
0.03239753842353821,
|
|
0.08061188459396362,
|
|
0.09571854770183563,
|
|
0.11411523073911667,
|
|
0.523703932762146
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.8863525390625,
|
|
"mean_weights": [
|
|
0.14810651540756226,
|
|
0.12629631161689758,
|
|
0.13384857773780823,
|
|
0.11124427616596222,
|
|
0.1082591712474823,
|
|
0.11263860762119293,
|
|
0.2596065402030945
|
|
],
|
|
"sources": 7
|
|
},
|
|
{
|
|
"entropy_mean": 1.5897703170776367,
|
|
"mean_weights": [
|
|
0.07248121500015259,
|
|
0.03783262521028519,
|
|
0.02471475675702095,
|
|
0.053565122187137604,
|
|
0.058412034064531326,
|
|
0.06658737361431122,
|
|
0.26026394963264465,
|
|
0.42614293098449707
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.814478874206543,
|
|
"mean_weights": [
|
|
0.09743618965148926,
|
|
0.07339455187320709,
|
|
0.06428516656160355,
|
|
0.07672879844903946,
|
|
0.07112542539834976,
|
|
0.07841956615447998,
|
|
0.14423531293869019,
|
|
0.3943749666213989
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.5432119369506836,
|
|
"mean_weights": [
|
|
0.07432855665683746,
|
|
0.04178062081336975,
|
|
0.029931148514151573,
|
|
0.0517733134329319,
|
|
0.053720228374004364,
|
|
0.06251072138547897,
|
|
0.17899465560913086,
|
|
0.5069607496261597
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.8694422245025635,
|
|
"mean_weights": [
|
|
0.10448876023292542,
|
|
0.07235300540924072,
|
|
0.05752243846654892,
|
|
0.08492527902126312,
|
|
0.07930260896682739,
|
|
0.08949960768222809,
|
|
0.1757010817527771,
|
|
0.33620721101760864
|
|
],
|
|
"sources": 8
|
|
},
|
|
{
|
|
"entropy_mean": 1.746185302734375,
|
|
"mean_weights": [
|
|
0.06162667274475098,
|
|
0.04293470457196236,
|
|
0.02593991532921791,
|
|
0.04546971619129181,
|
|
0.044149380177259445,
|
|
0.052234895527362823,
|
|
0.14164786040782928,
|
|
0.2587880492210388,
|
|
0.3272087574005127
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 2.061832904815674,
|
|
"mean_weights": [
|
|
0.09089995175600052,
|
|
0.06807379424571991,
|
|
0.04443599283695221,
|
|
0.08077649772167206,
|
|
0.08381937444210052,
|
|
0.09205536544322968,
|
|
0.15836367011070251,
|
|
0.20683875679969788,
|
|
0.17473658919334412
|
|
],
|
|
"sources": 9
|
|
},
|
|
{
|
|
"entropy_mean": 1.849208116531372,
|
|
"mean_weights": [
|
|
0.07359056174755096,
|
|
0.050991274416446686,
|
|
0.029770921915769577,
|
|
0.0592363178730011,
|
|
0.055861424654722214,
|
|
0.06031768023967743,
|
|
0.11290363222360611,
|
|
0.2822529673576355,
|
|
0.275075227022171
|
|
],
|
|
"sources": 9
|
|
}
|
|
],
|
|
"layer_input_rms_by_sublayer": [
|
|
0.06120532751083374,
|
|
0.05362993851304054,
|
|
0.06824863702058792,
|
|
0.06727377325296402,
|
|
0.1878596842288971,
|
|
0.1534913182258606,
|
|
0.21103329956531525,
|
|
0.1696806252002716,
|
|
0.20562554895877838,
|
|
0.10344218462705612,
|
|
0.13388557732105255,
|
|
0.08117230981588364,
|
|
0.14070458710193634,
|
|
0.10246162861585617,
|
|
0.10507974773645401,
|
|
0.06554406136274338,
|
|
0.11050253361463547,
|
|
0.05469200015068054,
|
|
0.08388946205377579,
|
|
0.05699421092867851,
|
|
0.10618287324905396,
|
|
0.03451571613550186,
|
|
0.06762637197971344,
|
|
0.050157804042100906,
|
|
0.08278355002403259,
|
|
0.03850048407912254,
|
|
0.04651740565896034,
|
|
0.04368532821536064,
|
|
0.05266447737812996,
|
|
0.036846842616796494,
|
|
0.04717044159770012,
|
|
0.048093993216753006
|
|
],
|
|
"loss_nats": 1.294543981552124,
|
|
"loss_scale": 1.0,
|
|
"output_weights": {
|
|
"entropy_mean": 1.636631727218628,
|
|
"mean_weights": [
|
|
0.07672396302223206,
|
|
0.03940216451883316,
|
|
0.03699969872832298,
|
|
0.05207572132349014,
|
|
0.04068697616457939,
|
|
0.04208337888121605,
|
|
0.051315970718860626,
|
|
0.2188674658536911,
|
|
0.4418446719646454
|
|
],
|
|
"sources": 9
|
|
},
|
|
"step": 8000,
|
|
"stream_state_rms_by_sublayer": [
|
|
0.07169820368289948,
|
|
0.15090593695640564,
|
|
0.29125523567199707,
|
|
0.3367030918598175,
|
|
0.2934682369232178,
|
|
0.29793402552604675,
|
|
0.3226769268512726,
|
|
0.3255155086517334,
|
|
0.048878561705350876,
|
|
0.05378299206495285,
|
|
0.0762643963098526,
|
|
0.08782409131526947,
|
|
0.03420666605234146,
|
|
0.044081781059503555,
|
|
0.07245275378227234,
|
|
0.09709836542606354,
|
|
0.04840917885303497,
|
|
0.0669482871890068,
|
|
0.08269722014665604,
|
|
0.12247904390096664,
|
|
0.04624715819954872,
|
|
0.07360157370567322,
|
|
0.08215674012899399,
|
|
0.12167256325483322,
|
|
0.03031807579100132,
|
|
0.06349270045757294,
|
|
0.06805067509412766,
|
|
0.10058031976222992,
|
|
0.02396680787205696,
|
|
0.07551935315132141,
|
|
0.07906875014305115,
|
|
0.19356463849544525
|
|
]
|
|
}
|
|
],
|
|
"environment": {
|
|
"autocast": "cuda-bfloat16-forward-fp32-cross-entropy",
|
|
"compile": false,
|
|
"compute_capability": [
|
|
12,
|
|
0
|
|
],
|
|
"cublas_workspace_config": ":4096:8",
|
|
"cuda": "12.8",
|
|
"deterministic_algorithms": true,
|
|
"gpu": "NVIDIA GeForce RTX 5090",
|
|
"python": "3.10.14",
|
|
"torch": "2.11.0+cu128"
|
|
},
|
|
"evaluations": [
|
|
{
|
|
"bits_per_byte": 8.030247040746552,
|
|
"cross_entropy_nats": 5.566143095493317,
|
|
"step": 0
|
|
},
|
|
{
|
|
"bits_per_byte": 5.230785295754151,
|
|
"cross_entropy_nats": 3.6257040798664093,
|
|
"step": 100
|
|
},
|
|
{
|
|
"bits_per_byte": 3.16099769723908,
|
|
"cross_entropy_nats": 2.191036641597748,
|
|
"step": 500
|
|
},
|
|
{
|
|
"bits_per_byte": 1.983385019569791,
|
|
"cross_entropy_nats": 1.3747777342796326,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"bits_per_byte": 1.818564586202164,
|
|
"cross_entropy_nats": 1.2605329155921936,
|
|
"step": 4000
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7311772990915377,
|
|
"cross_entropy_nats": 1.1999606639146805,
|
|
"step": 8000
|
|
}
|
|
],
|
|
"gradient_gate": null,
|
|
"hashes": {
|
|
"final_mixer_parameters": "7517e8197003ce4ba5267d82e82d6a41d49e7ce8744b4095b5a5160f1d4d01c2",
|
|
"final_model_state": "9867cf3c0d2372bb731808e8fd98ce5f38db0a40bd3cc08683bc5f66c9472428",
|
|
"final_optimizer_state": "fa42da28c86335ac8bab241bdce833e1a77d2508025d738b471764f2dd16cf93",
|
|
"final_public_parameters": "8c73d93d872090c7e7d7cdebca989ac8c083cf6d1cb5fb16509c6e6993d3c9a0",
|
|
"initial_mixer_parameters": "64fe0b90f5ea84ed88b6d93838558443c700c2b89ade2455ed526bd8db5d506f",
|
|
"initial_public_parameter_elements": 9541824,
|
|
"initial_public_parameter_structure": "e732db766f25e01f6ce1772cc182ced9de2c56c4a2384130117242b9444f0abe",
|
|
"initial_public_parameter_tensors": 115,
|
|
"initial_public_parameters": "9fd4f04212ab5cea822f469902d8e80ecc368da329f3c20abacfe6b7a50ed523"
|
|
},
|
|
"manifest": {
|
|
"diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716",
|
|
"file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371",
|
|
"formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
|
|
"input_gate_tensor_hashes": {
|
|
"0": "add6249175ac3bb3fa911069fbe8f25d17d9ab05df34b66ee89996983522ce83",
|
|
"1": "619b5ee4f70e975312ead9adf5df4f9d8584ce2839369c749442e429923405cf",
|
|
"7999": "fe22994f7750c2f8323fdb5ac51c9be1e208733abaf6a337f5b1ae1df1cd2730"
|
|
},
|
|
"path": "experiments/k3/attnres_gradient/manifest.json",
|
|
"validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338"
|
|
},
|
|
"model": {
|
|
"attnres_aggregation_groups": 8,
|
|
"context": 256,
|
|
"d_ff": 768,
|
|
"d_head": 32,
|
|
"d_model": 192,
|
|
"heads": 6,
|
|
"layers": 16,
|
|
"parameters": {
|
|
"core": 9541824,
|
|
"embedding": 98304,
|
|
"mixer": 12672,
|
|
"total": 9554496
|
|
},
|
|
"sublayers": 32,
|
|
"sublayers_per_attnres_group": 4,
|
|
"transformer_blocks_per_attnres_group": 2,
|
|
"vocabulary": 256
|
|
},
|
|
"optimizer": {
|
|
"betas": [
|
|
0.9,
|
|
0.95
|
|
],
|
|
"epsilon": 1e-08,
|
|
"grad_clip": 1.0,
|
|
"min_lr": 3e-05,
|
|
"name": "AdamW",
|
|
"peak_lr": 0.0003,
|
|
"warmup_steps": 400,
|
|
"weight_decay_ndim_ge_2": 0.1
|
|
},
|
|
"protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
|
|
"run_kind": "formal",
|
|
"schema_version": 1,
|
|
"seed": 2026073002,
|
|
"steps": 8000,
|
|
"target_bytes_seen": 65536000,
|
|
"timing": {
|
|
"mean_ms": 54.937352295237,
|
|
"measured_steps": 7980,
|
|
"median_ms": 54.89947448950261,
|
|
"p95_ms": 55.28610254696105,
|
|
"peak_allocated_bytes": 6536921088,
|
|
"peak_reserved_bytes": 6889144320,
|
|
"warmup_steps_excluded": 20
|
|
},
|
|
"training_history": [
|
|
{
|
|
"bits_per_byte": 8.033595800932828,
|
|
"learning_rate": 7.499999999999999e-07,
|
|
"loss_nats": 5.568464279174805,
|
|
"step": 1,
|
|
"unclipped_grad_norm": 13.626673698425293
|
|
},
|
|
{
|
|
"bits_per_byte": 7.5882983213188435,
|
|
"learning_rate": 7.499999999999999e-06,
|
|
"loss_nats": 5.259807586669922,
|
|
"step": 10,
|
|
"unclipped_grad_norm": 10.73694896697998
|
|
},
|
|
{
|
|
"bits_per_byte": 6.986202883052029,
|
|
"learning_rate": 1.4999999999999999e-05,
|
|
"loss_nats": 4.842466831207275,
|
|
"step": 20,
|
|
"unclipped_grad_norm": 4.4299492835998535
|
|
},
|
|
{
|
|
"bits_per_byte": 6.739012218966915,
|
|
"learning_rate": 2.2499999999999998e-05,
|
|
"loss_nats": 4.6711273193359375,
|
|
"step": 30,
|
|
"unclipped_grad_norm": 3.205341339111328
|
|
},
|
|
{
|
|
"bits_per_byte": 6.623832685961678,
|
|
"learning_rate": 2.9999999999999997e-05,
|
|
"loss_nats": 4.5912909507751465,
|
|
"step": 40,
|
|
"unclipped_grad_norm": 3.10516619682312
|
|
},
|
|
{
|
|
"bits_per_byte": 6.3865585415253125,
|
|
"learning_rate": 3.75e-05,
|
|
"loss_nats": 4.426825046539307,
|
|
"step": 50,
|
|
"unclipped_grad_norm": 3.1714048385620117
|
|
},
|
|
{
|
|
"bits_per_byte": 6.103358837933146,
|
|
"learning_rate": 4.4999999999999996e-05,
|
|
"loss_nats": 4.230525970458984,
|
|
"step": 60,
|
|
"unclipped_grad_norm": 2.932384490966797
|
|
},
|
|
{
|
|
"bits_per_byte": 5.8558104499342365,
|
|
"learning_rate": 5.2499999999999995e-05,
|
|
"loss_nats": 4.058938503265381,
|
|
"step": 70,
|
|
"unclipped_grad_norm": 2.6460657119750977
|
|
},
|
|
{
|
|
"bits_per_byte": 5.640669096802708,
|
|
"learning_rate": 5.9999999999999995e-05,
|
|
"loss_nats": 3.90981388092041,
|
|
"step": 80,
|
|
"unclipped_grad_norm": 2.5319507122039795
|
|
},
|
|
{
|
|
"bits_per_byte": 5.451491620506636,
|
|
"learning_rate": 6.749999999999999e-05,
|
|
"loss_nats": 3.778686046600342,
|
|
"step": 90,
|
|
"unclipped_grad_norm": 2.4583022594451904
|
|
},
|
|
{
|
|
"bits_per_byte": 5.270375658987107,
|
|
"learning_rate": 7.5e-05,
|
|
"loss_nats": 3.6531460285186768,
|
|
"step": 100,
|
|
"unclipped_grad_norm": 2.433866262435913
|
|
},
|
|
{
|
|
"bits_per_byte": 5.10413206918359,
|
|
"learning_rate": 8.249999999999999e-05,
|
|
"loss_nats": 3.537914752960205,
|
|
"step": 110,
|
|
"unclipped_grad_norm": 2.285804271697998
|
|
},
|
|
{
|
|
"bits_per_byte": 4.823540998439711,
|
|
"learning_rate": 8.999999999999999e-05,
|
|
"loss_nats": 3.343423843383789,
|
|
"step": 120,
|
|
"unclipped_grad_norm": 2.0681240558624268
|
|
},
|
|
{
|
|
"bits_per_byte": 4.663901886394628,
|
|
"learning_rate": 9.75e-05,
|
|
"loss_nats": 3.2327704429626465,
|
|
"step": 130,
|
|
"unclipped_grad_norm": 1.8398888111114502
|
|
},
|
|
{
|
|
"bits_per_byte": 4.463271741412299,
|
|
"learning_rate": 0.00010499999999999999,
|
|
"loss_nats": 3.0937042236328125,
|
|
"step": 140,
|
|
"unclipped_grad_norm": 1.681030035018921
|
|
},
|
|
{
|
|
"bits_per_byte": 4.327779681478958,
|
|
"learning_rate": 0.0001125,
|
|
"loss_nats": 2.999788284301758,
|
|
"step": 150,
|
|
"unclipped_grad_norm": 1.4562726020812988
|
|
},
|
|
{
|
|
"bits_per_byte": 4.208912840439223,
|
|
"learning_rate": 0.00011999999999999999,
|
|
"loss_nats": 2.917396068572998,
|
|
"step": 160,
|
|
"unclipped_grad_norm": 1.1446112394332886
|
|
},
|
|
{
|
|
"bits_per_byte": 4.0496410833363825,
|
|
"learning_rate": 0.00012749999999999998,
|
|
"loss_nats": 2.806997299194336,
|
|
"step": 170,
|
|
"unclipped_grad_norm": 1.193884253501892
|
|
},
|
|
{
|
|
"bits_per_byte": 3.9053046435997705,
|
|
"learning_rate": 0.00013499999999999997,
|
|
"loss_nats": 2.7069509029388428,
|
|
"step": 180,
|
|
"unclipped_grad_norm": 1.1651051044464111
|
|
},
|
|
{
|
|
"bits_per_byte": 3.9164302012840855,
|
|
"learning_rate": 0.0001425,
|
|
"loss_nats": 2.714662551879883,
|
|
"step": 190,
|
|
"unclipped_grad_norm": 1.1523211002349854
|
|
},
|
|
{
|
|
"bits_per_byte": 3.7885273917680182,
|
|
"learning_rate": 0.00015,
|
|
"loss_nats": 2.626007080078125,
|
|
"step": 200,
|
|
"unclipped_grad_norm": 0.7202922701835632
|
|
},
|
|
{
|
|
"bits_per_byte": 3.72972377171561,
|
|
"learning_rate": 0.0001575,
|
|
"loss_nats": 2.58524751663208,
|
|
"step": 210,
|
|
"unclipped_grad_norm": 0.606606662273407
|
|
},
|
|
{
|
|
"bits_per_byte": 3.7010143638417796,
|
|
"learning_rate": 0.00016499999999999997,
|
|
"loss_nats": 2.565347671508789,
|
|
"step": 220,
|
|
"unclipped_grad_norm": 0.6799013018608093
|
|
},
|
|
{
|
|
"bits_per_byte": 3.715506997864687,
|
|
"learning_rate": 0.0001725,
|
|
"loss_nats": 2.5753931999206543,
|
|
"step": 230,
|
|
"unclipped_grad_norm": 0.6519911289215088
|
|
},
|
|
{
|
|
"bits_per_byte": 3.7068112110717615,
|
|
"learning_rate": 0.00017999999999999998,
|
|
"loss_nats": 2.5693657398223877,
|
|
"step": 240,
|
|
"unclipped_grad_norm": 0.912699818611145
|
|
},
|
|
{
|
|
"bits_per_byte": 3.5753026132977044,
|
|
"learning_rate": 0.0001875,
|
|
"loss_nats": 2.478210926055908,
|
|
"step": 250,
|
|
"unclipped_grad_norm": 0.6829943060874939
|
|
},
|
|
{
|
|
"bits_per_byte": 3.5693214006659977,
|
|
"learning_rate": 0.000195,
|
|
"loss_nats": 2.474065065383911,
|
|
"step": 260,
|
|
"unclipped_grad_norm": 0.640023410320282
|
|
},
|
|
{
|
|
"bits_per_byte": 3.539509240034835,
|
|
"learning_rate": 0.00020249999999999996,
|
|
"loss_nats": 2.4534008502960205,
|
|
"step": 270,
|
|
"unclipped_grad_norm": 0.6751542687416077
|
|
},
|
|
{
|
|
"bits_per_byte": 3.5394008909647914,
|
|
"learning_rate": 0.00020999999999999998,
|
|
"loss_nats": 2.4533257484436035,
|
|
"step": 280,
|
|
"unclipped_grad_norm": 0.7841543555259705
|
|
},
|
|
{
|
|
"bits_per_byte": 3.4452169999768865,
|
|
"learning_rate": 0.00021749999999999997,
|
|
"loss_nats": 2.388042449951172,
|
|
"step": 290,
|
|
"unclipped_grad_norm": 0.8627198934555054
|
|
},
|
|
{
|
|
"bits_per_byte": 3.5355663657811545,
|
|
"learning_rate": 0.000225,
|
|
"loss_nats": 2.4506678581237793,
|
|
"step": 300,
|
|
"unclipped_grad_norm": 1.1167978048324585
|
|
},
|
|
{
|
|
"bits_per_byte": 3.4429038333227817,
|
|
"learning_rate": 0.00023249999999999996,
|
|
"loss_nats": 2.386439085006714,
|
|
"step": 310,
|
|
"unclipped_grad_norm": 0.8460172414779663
|
|
},
|
|
{
|
|
"bits_per_byte": 3.4221950583670893,
|
|
"learning_rate": 0.00023999999999999998,
|
|
"loss_nats": 2.372084856033325,
|
|
"step": 320,
|
|
"unclipped_grad_norm": 0.821152925491333
|
|
},
|
|
{
|
|
"bits_per_byte": 3.429671832130699,
|
|
"learning_rate": 0.0002475,
|
|
"loss_nats": 2.377267360687256,
|
|
"step": 330,
|
|
"unclipped_grad_norm": 0.9970561265945435
|
|
},
|
|
{
|
|
"bits_per_byte": 3.425132178078524,
|
|
"learning_rate": 0.00025499999999999996,
|
|
"loss_nats": 2.3741207122802734,
|
|
"step": 340,
|
|
"unclipped_grad_norm": 1.0991977453231812
|
|
},
|
|
{
|
|
"bits_per_byte": 3.393254505810499,
|
|
"learning_rate": 0.0002625,
|
|
"loss_nats": 2.352024793624878,
|
|
"step": 350,
|
|
"unclipped_grad_norm": 0.9725445508956909
|
|
},
|
|
{
|
|
"bits_per_byte": 3.382284076477262,
|
|
"learning_rate": 0.00026999999999999995,
|
|
"loss_nats": 2.3444206714630127,
|
|
"step": 360,
|
|
"unclipped_grad_norm": 1.0039488077163696
|
|
},
|
|
{
|
|
"bits_per_byte": 3.4119693139120884,
|
|
"learning_rate": 0.00027749999999999997,
|
|
"loss_nats": 2.364996910095215,
|
|
"step": 370,
|
|
"unclipped_grad_norm": 1.7546980381011963
|
|
},
|
|
{
|
|
"bits_per_byte": 3.446912748914394,
|
|
"learning_rate": 0.000285,
|
|
"loss_nats": 2.3892178535461426,
|
|
"step": 380,
|
|
"unclipped_grad_norm": 0.9759950041770935
|
|
},
|
|
{
|
|
"bits_per_byte": 3.3737516732026567,
|
|
"learning_rate": 0.0002925,
|
|
"loss_nats": 2.3385064601898193,
|
|
"step": 390,
|
|
"unclipped_grad_norm": 0.9621731638908386
|
|
},
|
|
{
|
|
"bits_per_byte": 3.3279967208365115,
|
|
"learning_rate": 0.0003,
|
|
"loss_nats": 2.3067915439605713,
|
|
"step": 400,
|
|
"unclipped_grad_norm": 0.9455732107162476
|
|
},
|
|
{
|
|
"bits_per_byte": 3.307004174506895,
|
|
"learning_rate": 0.0002999988466112388,
|
|
"loss_nats": 2.292240619659424,
|
|
"step": 410,
|
|
"unclipped_grad_norm": 1.4261541366577148
|
|
},
|
|
{
|
|
"bits_per_byte": 3.3082469211420302,
|
|
"learning_rate": 0.00029999538646466355,
|
|
"loss_nats": 2.2931020259857178,
|
|
"step": 420,
|
|
"unclipped_grad_norm": 0.9587261080741882
|
|
},
|
|
{
|
|
"bits_per_byte": 3.315426508884949,
|
|
"learning_rate": 0.0002999896196193985,
|
|
"loss_nats": 2.2980785369873047,
|
|
"step": 430,
|
|
"unclipped_grad_norm": 2.1751644611358643
|
|
},
|
|
{
|
|
"bits_per_byte": 3.299565580891777,
|
|
"learning_rate": 0.00029998154617398326,
|
|
"loss_nats": 2.2870845794677734,
|
|
"step": 440,
|
|
"unclipped_grad_norm": 1.11757493019104
|
|
},
|
|
{
|
|
"bits_per_byte": 3.305290883339,
|
|
"learning_rate": 0.0002999711662663707,
|
|
"loss_nats": 2.291053056716919,
|
|
"step": 450,
|
|
"unclipped_grad_norm": 1.8777321577072144
|
|
},
|
|
{
|
|
"bits_per_byte": 3.227380334741036,
|
|
"learning_rate": 0.0002999584800739247,
|
|
"loss_nats": 2.2370495796203613,
|
|
"step": 460,
|
|
"unclipped_grad_norm": 1.1160534620285034
|
|
},
|
|
{
|
|
"bits_per_byte": 3.2510836716135514,
|
|
"learning_rate": 0.0002999434878134173,
|
|
"loss_nats": 2.253479480743408,
|
|
"step": 470,
|
|
"unclipped_grad_norm": 1.6702216863632202
|
|
},
|
|
{
|
|
"bits_per_byte": 3.198324209843066,
|
|
"learning_rate": 0.000299926189741025,
|
|
"loss_nats": 2.216909408569336,
|
|
"step": 480,
|
|
"unclipped_grad_norm": 1.0277327299118042
|
|
},
|
|
{
|
|
"bits_per_byte": 3.1569424323230604,
|
|
"learning_rate": 0.000299906586152324,
|
|
"loss_nats": 2.188225746154785,
|
|
"step": 490,
|
|
"unclipped_grad_norm": 1.0115255117416382
|
|
},
|
|
{
|
|
"bits_per_byte": 3.1652801512368893,
|
|
"learning_rate": 0.0002998846773822856,
|
|
"loss_nats": 2.194005012512207,
|
|
"step": 500,
|
|
"unclipped_grad_norm": 1.3175867795944214
|
|
},
|
|
{
|
|
"bits_per_byte": 3.134954794375552,
|
|
"learning_rate": 0.0002998604638052702,
|
|
"loss_nats": 2.172985076904297,
|
|
"step": 510,
|
|
"unclipped_grad_norm": 0.9790136814117432
|
|
},
|
|
{
|
|
"bits_per_byte": 3.0134504274021845,
|
|
"learning_rate": 0.0002998339458350211,
|
|
"loss_nats": 2.0887646675109863,
|
|
"step": 520,
|
|
"unclipped_grad_norm": 1.0963760614395142
|
|
},
|
|
{
|
|
"bits_per_byte": 3.0906245462891544,
|
|
"learning_rate": 0.0002998051239246572,
|
|
"loss_nats": 2.1422576904296875,
|
|
"step": 530,
|
|
"unclipped_grad_norm": 1.0304492712020874
|
|
},
|
|
{
|
|
"bits_per_byte": 3.024995622754605,
|
|
"learning_rate": 0.00029977399856666537,
|
|
"loss_nats": 2.0967671871185303,
|
|
"step": 540,
|
|
"unclipped_grad_norm": 1.323768973350525
|
|
},
|
|
{
|
|
"bits_per_byte": 2.9805828229957934,
|
|
"learning_rate": 0.0002997405702928922,
|
|
"loss_nats": 2.0659825801849365,
|
|
"step": 550,
|
|
"unclipped_grad_norm": 1.1477609872817993
|
|
},
|
|
{
|
|
"bits_per_byte": 2.9872017472968992,
|
|
"learning_rate": 0.0002997048396745345,
|
|
"loss_nats": 2.070570468902588,
|
|
"step": 560,
|
|
"unclipped_grad_norm": 1.223684310913086
|
|
},
|
|
{
|
|
"bits_per_byte": 2.9579880862907393,
|
|
"learning_rate": 0.0002996668073221302,
|
|
"loss_nats": 2.050321102142334,
|
|
"step": 570,
|
|
"unclipped_grad_norm": 1.308420181274414
|
|
},
|
|
{
|
|
"bits_per_byte": 2.8546603937044113,
|
|
"learning_rate": 0.000299626473885547,
|
|
"loss_nats": 1.978699803352356,
|
|
"step": 580,
|
|
"unclipped_grad_norm": 1.0794485807418823
|
|
},
|
|
{
|
|
"bits_per_byte": 2.911534540396789,
|
|
"learning_rate": 0.00029958384005397226,
|
|
"loss_nats": 2.0181219577789307,
|
|
"step": 590,
|
|
"unclipped_grad_norm": 1.4487859010696411
|
|
},
|
|
{
|
|
"bits_per_byte": 2.825803768681442,
|
|
"learning_rate": 0.0002995389065559004,
|
|
"loss_nats": 1.9586979150772095,
|
|
"step": 600,
|
|
"unclipped_grad_norm": 1.2767362594604492
|
|
},
|
|
{
|
|
"bits_per_byte": 2.883143472409706,
|
|
"learning_rate": 0.000299491674159121,
|
|
"loss_nats": 1.9984427690505981,
|
|
"step": 610,
|
|
"unclipped_grad_norm": 1.1309611797332764
|
|
},
|
|
{
|
|
"bits_per_byte": 2.7721532647968363,
|
|
"learning_rate": 0.00029944214367070535,
|
|
"loss_nats": 1.9215102195739746,
|
|
"step": 620,
|
|
"unclipped_grad_norm": 1.533046007156372
|
|
},
|
|
{
|
|
"bits_per_byte": 2.7733690101557857,
|
|
"learning_rate": 0.00029939031593699284,
|
|
"loss_nats": 1.922352910041809,
|
|
"step": 630,
|
|
"unclipped_grad_norm": 0.9842169880867004
|
|
},
|
|
{
|
|
"bits_per_byte": 2.7997499449195846,
|
|
"learning_rate": 0.00029933619184357654,
|
|
"loss_nats": 1.940638780593872,
|
|
"step": 640,
|
|
"unclipped_grad_norm": 1.2905892133712769
|
|
},
|
|
{
|
|
"bits_per_byte": 2.715926288819655,
|
|
"learning_rate": 0.00029927977231528774,
|
|
"loss_nats": 1.8825366497039795,
|
|
"step": 650,
|
|
"unclipped_grad_norm": 1.1535414457321167
|
|
},
|
|
{
|
|
"bits_per_byte": 2.6764600540649592,
|
|
"learning_rate": 0.00029922105831618057,
|
|
"loss_nats": 1.8551807403564453,
|
|
"step": 660,
|
|
"unclipped_grad_norm": 1.1525585651397705
|
|
},
|
|
{
|
|
"bits_per_byte": 2.696502224265908,
|
|
"learning_rate": 0.0002991600508495153,
|
|
"loss_nats": 1.8690729141235352,
|
|
"step": 670,
|
|
"unclipped_grad_norm": 1.264070749282837
|
|
},
|
|
{
|
|
"bits_per_byte": 2.7320018511611033,
|
|
"learning_rate": 0.00029909675095774114,
|
|
"loss_nats": 1.8936793804168701,
|
|
"step": 680,
|
|
"unclipped_grad_norm": 1.6425000429153442
|
|
},
|
|
{
|
|
"bits_per_byte": 2.602245068668735,
|
|
"learning_rate": 0.0002990311597224787,
|
|
"loss_nats": 1.8037388324737549,
|
|
"step": 690,
|
|
"unclipped_grad_norm": 1.3727850914001465
|
|
},
|
|
{
|
|
"bits_per_byte": 2.5536510267021466,
|
|
"learning_rate": 0.00029896327826450103,
|
|
"loss_nats": 1.7700560092926025,
|
|
"step": 700,
|
|
"unclipped_grad_norm": 2.108612298965454
|
|
},
|
|
{
|
|
"bits_per_byte": 2.623769213372168,
|
|
"learning_rate": 0.0002988931077437152,
|
|
"loss_nats": 1.8186582326889038,
|
|
"step": 710,
|
|
"unclipped_grad_norm": 1.441254734992981
|
|
},
|
|
{
|
|
"bits_per_byte": 2.6835147824033516,
|
|
"learning_rate": 0.00029882064935914164,
|
|
"loss_nats": 1.8600707054138184,
|
|
"step": 720,
|
|
"unclipped_grad_norm": 1.4497488737106323
|
|
},
|
|
{
|
|
"bits_per_byte": 2.4942898388956416,
|
|
"learning_rate": 0.0002987459043488943,
|
|
"loss_nats": 1.728909969329834,
|
|
"step": 730,
|
|
"unclipped_grad_norm": 1.7889822721481323
|
|
},
|
|
{
|
|
"bits_per_byte": 2.518159998939494,
|
|
"learning_rate": 0.0002986688739901591,
|
|
"loss_nats": 1.7454555034637451,
|
|
"step": 740,
|
|
"unclipped_grad_norm": 1.1689488887786865
|
|
},
|
|
{
|
|
"bits_per_byte": 2.5159042744907776,
|
|
"learning_rate": 0.0002985895595991723,
|
|
"loss_nats": 1.743891954421997,
|
|
"step": 750,
|
|
"unclipped_grad_norm": 1.466252088546753
|
|
},
|
|
{
|
|
"bits_per_byte": 2.5144026939660944,
|
|
"learning_rate": 0.00029850796253119824,
|
|
"loss_nats": 1.7428511381149292,
|
|
"step": 760,
|
|
"unclipped_grad_norm": 1.454863429069519
|
|
},
|
|
{
|
|
"bits_per_byte": 2.485389736713491,
|
|
"learning_rate": 0.00029842408418050544,
|
|
"loss_nats": 1.722740888595581,
|
|
"step": 770,
|
|
"unclipped_grad_norm": 1.2171542644500732
|
|
},
|
|
{
|
|
"bits_per_byte": 2.4154798210336645,
|
|
"learning_rate": 0.0002983379259803436,
|
|
"loss_nats": 1.6742830276489258,
|
|
"step": 780,
|
|
"unclipped_grad_norm": 1.3978826999664307
|
|
},
|
|
{
|
|
"bits_per_byte": 2.492438617641754,
|
|
"learning_rate": 0.0002982494894029186,
|
|
"loss_nats": 1.7276268005371094,
|
|
"step": 790,
|
|
"unclipped_grad_norm": 1.4353379011154175
|
|
},
|
|
{
|
|
"bits_per_byte": 2.4424274386320564,
|
|
"learning_rate": 0.0002981587759593675,
|
|
"loss_nats": 1.6929616928100586,
|
|
"step": 800,
|
|
"unclipped_grad_norm": 1.3198370933532715
|
|
},
|
|
{
|
|
"bits_per_byte": 2.473405641636069,
|
|
"learning_rate": 0.00029806578719973283,
|
|
"loss_nats": 1.7144341468811035,
|
|
"step": 810,
|
|
"unclipped_grad_norm": 1.3822002410888672
|
|
},
|
|
{
|
|
"bits_per_byte": 2.441873482513627,
|
|
"learning_rate": 0.00029797052471293596,
|
|
"loss_nats": 1.6925777196884155,
|
|
"step": 820,
|
|
"unclipped_grad_norm": 1.3033113479614258
|
|
},
|
|
{
|
|
"bits_per_byte": 2.4253691674235776,
|
|
"learning_rate": 0.0002978729901267499,
|
|
"loss_nats": 1.6811378002166748,
|
|
"step": 830,
|
|
"unclipped_grad_norm": 1.450317621231079
|
|
},
|
|
{
|
|
"bits_per_byte": 2.4107564895103684,
|
|
"learning_rate": 0.0002977731851077716,
|
|
"loss_nats": 1.6710090637207031,
|
|
"step": 840,
|
|
"unclipped_grad_norm": 1.322706699371338
|
|
},
|
|
{
|
|
"bits_per_byte": 2.347782118191886,
|
|
"learning_rate": 0.0002976711113613935,
|
|
"loss_nats": 1.6273585557937622,
|
|
"step": 850,
|
|
"unclipped_grad_norm": 1.2753863334655762
|
|
},
|
|
{
|
|
"bits_per_byte": 2.3967822111971615,
|
|
"learning_rate": 0.0002975667706317743,
|
|
"loss_nats": 1.661322832107544,
|
|
"step": 860,
|
|
"unclipped_grad_norm": 2.4916832447052
|
|
},
|
|
{
|
|
"bits_per_byte": 2.3462526764777634,
|
|
"learning_rate": 0.00029746016470180915,
|
|
"loss_nats": 1.626298427581787,
|
|
"step": 870,
|
|
"unclipped_grad_norm": 1.3712787628173828
|
|
},
|
|
{
|
|
"bits_per_byte": 2.4080783756711326,
|
|
"learning_rate": 0.00029735129539309926,
|
|
"loss_nats": 1.6691527366638184,
|
|
"step": 880,
|
|
"unclipped_grad_norm": 1.4296884536743164
|
|
},
|
|
{
|
|
"bits_per_byte": 2.363901364126654,
|
|
"learning_rate": 0.0002972401645659207,
|
|
"loss_nats": 1.6385315656661987,
|
|
"step": 890,
|
|
"unclipped_grad_norm": 1.3781569004058838
|
|
},
|
|
{
|
|
"bits_per_byte": 2.2957977822288393,
|
|
"learning_rate": 0.00029712677411919283,
|
|
"loss_nats": 1.5913257598876953,
|
|
"step": 900,
|
|
"unclipped_grad_norm": 1.3345599174499512
|
|
},
|
|
{
|
|
"bits_per_byte": 2.3445476404771095,
|
|
"learning_rate": 0.0002970111259904454,
|
|
"loss_nats": 1.6251165866851807,
|
|
"step": 910,
|
|
"unclipped_grad_norm": 1.6203302145004272
|
|
},
|
|
{
|
|
"bits_per_byte": 2.297230053745225,
|
|
"learning_rate": 0.0002968932221557859,
|
|
"loss_nats": 1.5923185348510742,
|
|
"step": 920,
|
|
"unclipped_grad_norm": 1.685173511505127
|
|
},
|
|
{
|
|
"bits_per_byte": 2.297947909329926,
|
|
"learning_rate": 0.0002967730646298655,
|
|
"loss_nats": 1.5928161144256592,
|
|
"step": 930,
|
|
"unclipped_grad_norm": 1.4890943765640259
|
|
},
|
|
{
|
|
"bits_per_byte": 2.2609902135207154,
|
|
"learning_rate": 0.0002966506554658449,
|
|
"loss_nats": 1.5671989917755127,
|
|
"step": 940,
|
|
"unclipped_grad_norm": 1.3654035329818726
|
|
},
|
|
{
|
|
"bits_per_byte": 2.2430326450308753,
|
|
"learning_rate": 0.00029652599675535876,
|
|
"loss_nats": 1.5547517538070679,
|
|
"step": 950,
|
|
"unclipped_grad_norm": 1.1998120546340942
|
|
},
|
|
{
|
|
"bits_per_byte": 2.2479709548677502,
|
|
"learning_rate": 0.0002963990906284808,
|
|
"loss_nats": 1.558174729347229,
|
|
"step": 960,
|
|
"unclipped_grad_norm": 1.40827214717865
|
|
},
|
|
{
|
|
"bits_per_byte": 2.2631277858882894,
|
|
"learning_rate": 0.0002962699392536863,
|
|
"loss_nats": 1.5686806440353394,
|
|
"step": 970,
|
|
"unclipped_grad_norm": 1.4645981788635254
|
|
},
|
|
{
|
|
"bits_per_byte": 2.2528444272452495,
|
|
"learning_rate": 0.0002961385448378161,
|
|
"loss_nats": 1.5615527629852295,
|
|
"step": 980,
|
|
"unclipped_grad_norm": 1.594014048576355
|
|
},
|
|
{
|
|
"bits_per_byte": 2.177915369882643,
|
|
"learning_rate": 0.00029600490962603817,
|
|
"loss_nats": 1.5096158981323242,
|
|
"step": 990,
|
|
"unclipped_grad_norm": 1.2862826585769653
|
|
},
|
|
{
|
|
"bits_per_byte": 2.2127330855671676,
|
|
"learning_rate": 0.00029586903590180956,
|
|
"loss_nats": 1.5337496995925903,
|
|
"step": 1000,
|
|
"unclipped_grad_norm": 1.4609711170196533
|
|
},
|
|
{
|
|
"bits_per_byte": 2.208716086790965,
|
|
"learning_rate": 0.00029573092598683735,
|
|
"loss_nats": 1.5309653282165527,
|
|
"step": 1010,
|
|
"unclipped_grad_norm": 1.2298572063446045
|
|
},
|
|
{
|
|
"bits_per_byte": 2.233356385145381,
|
|
"learning_rate": 0.00029559058224103895,
|
|
"loss_nats": 1.5480446815490723,
|
|
"step": 1020,
|
|
"unclipped_grad_norm": 1.469051718711853
|
|
},
|
|
{
|
|
"bits_per_byte": 2.1593615375422615,
|
|
"learning_rate": 0.0002954480070625017,
|
|
"loss_nats": 1.4967553615570068,
|
|
"step": 1030,
|
|
"unclipped_grad_norm": 1.3269134759902954
|
|
},
|
|
{
|
|
"bits_per_byte": 2.236164345886018,
|
|
"learning_rate": 0.00029530320288744205,
|
|
"loss_nats": 1.5499910116195679,
|
|
"step": 1040,
|
|
"unclipped_grad_norm": 1.3589463233947754
|
|
},
|
|
{
|
|
"bits_per_byte": 2.2039167389359875,
|
|
"learning_rate": 0.0002951561721901638,
|
|
"loss_nats": 1.5276386737823486,
|
|
"step": 1050,
|
|
"unclipped_grad_norm": 1.4478572607040405
|
|
},
|
|
{
|
|
"bits_per_byte": 2.1642855728191144,
|
|
"learning_rate": 0.00029500691748301594,
|
|
"loss_nats": 1.5001684427261353,
|
|
"step": 1060,
|
|
"unclipped_grad_norm": 1.4022979736328125
|
|
},
|
|
{
|
|
"bits_per_byte": 2.2672321518511303,
|
|
"learning_rate": 0.00029485544131634963,
|
|
"loss_nats": 1.5715255737304688,
|
|
"step": 1070,
|
|
"unclipped_grad_norm": 1.4223567247390747
|
|
},
|
|
{
|
|
"bits_per_byte": 2.187999916581285,
|
|
"learning_rate": 0.0002947017462784745,
|
|
"loss_nats": 1.5166059732437134,
|
|
"step": 1080,
|
|
"unclipped_grad_norm": 1.188625454902649
|
|
},
|
|
{
|
|
"bits_per_byte": 2.1147346193831256,
|
|
"learning_rate": 0.00029454583499561476,
|
|
"loss_nats": 1.4658223390579224,
|
|
"step": 1090,
|
|
"unclipped_grad_norm": 1.1601507663726807
|
|
},
|
|
{
|
|
"bits_per_byte": 2.1969277079702247,
|
|
"learning_rate": 0.00029438771013186413,
|
|
"loss_nats": 1.522794246673584,
|
|
"step": 1100,
|
|
"unclipped_grad_norm": 1.1728529930114746
|
|
},
|
|
{
|
|
"bits_per_byte": 2.2198211785398274,
|
|
"learning_rate": 0.0002942273743891401,
|
|
"loss_nats": 1.5386627912521362,
|
|
"step": 1110,
|
|
"unclipped_grad_norm": 1.3392254114151
|
|
},
|
|
{
|
|
"bits_per_byte": 2.126032503683605,
|
|
"learning_rate": 0.00029406483050713824,
|
|
"loss_nats": 1.4736534357070923,
|
|
"step": 1120,
|
|
"unclipped_grad_norm": 1.3545775413513184
|
|
},
|
|
{
|
|
"bits_per_byte": 2.1170070261489755,
|
|
"learning_rate": 0.00029390008126328504,
|
|
"loss_nats": 1.4673974514007568,
|
|
"step": 1130,
|
|
"unclipped_grad_norm": 1.2578480243682861
|
|
},
|
|
{
|
|
"bits_per_byte": 2.127665650936198,
|
|
"learning_rate": 0.00029373312947269045,
|
|
"loss_nats": 1.4747854471206665,
|
|
"step": 1140,
|
|
"unclipped_grad_norm": 1.6450976133346558
|
|
},
|
|
{
|
|
"bits_per_byte": 2.1180404698980104,
|
|
"learning_rate": 0.00029356397798809995,
|
|
"loss_nats": 1.4681137800216675,
|
|
"step": 1150,
|
|
"unclipped_grad_norm": 1.3040683269500732
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0764574726764953,
|
|
"learning_rate": 0.0002933926296998457,
|
|
"loss_nats": 1.4392906427383423,
|
|
"step": 1160,
|
|
"unclipped_grad_norm": 1.3104010820388794
|
|
},
|
|
{
|
|
"bits_per_byte": 2.1839928927988326,
|
|
"learning_rate": 0.0002932190875357972,
|
|
"loss_nats": 1.5138285160064697,
|
|
"step": 1170,
|
|
"unclipped_grad_norm": 1.3359254598617554
|
|
},
|
|
{
|
|
"bits_per_byte": 2.091760488950268,
|
|
"learning_rate": 0.00029304335446131126,
|
|
"loss_nats": 1.4498978853225708,
|
|
"step": 1180,
|
|
"unclipped_grad_norm": 1.3903101682662964
|
|
},
|
|
{
|
|
"bits_per_byte": 2.1040587963308166,
|
|
"learning_rate": 0.00029286543347918116,
|
|
"loss_nats": 1.4584224224090576,
|
|
"step": 1190,
|
|
"unclipped_grad_norm": 1.4786474704742432
|
|
},
|
|
{
|
|
"bits_per_byte": 2.081396986391926,
|
|
"learning_rate": 0.0002926853276295856,
|
|
"loss_nats": 1.4427144527435303,
|
|
"step": 1200,
|
|
"unclipped_grad_norm": 1.1696821451187134
|
|
},
|
|
{
|
|
"bits_per_byte": 2.2277741722636764,
|
|
"learning_rate": 0.0002925030399900368,
|
|
"loss_nats": 1.544175386428833,
|
|
"step": 1210,
|
|
"unclipped_grad_norm": 1.6089588403701782
|
|
},
|
|
{
|
|
"bits_per_byte": 2.071474963298349,
|
|
"learning_rate": 0.00029231857367532726,
|
|
"loss_nats": 1.4358370304107666,
|
|
"step": 1220,
|
|
"unclipped_grad_norm": 1.219665765762329
|
|
},
|
|
{
|
|
"bits_per_byte": 2.020156544159443,
|
|
"learning_rate": 0.0002921319318374776,
|
|
"loss_nats": 1.4002658128738403,
|
|
"step": 1230,
|
|
"unclipped_grad_norm": 1.1330406665802002
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0202031514578267,
|
|
"learning_rate": 0.00029194311766568174,
|
|
"loss_nats": 1.4002981185913086,
|
|
"step": 1240,
|
|
"unclipped_grad_norm": 1.4843499660491943
|
|
},
|
|
{
|
|
"bits_per_byte": 2.1912978558942307,
|
|
"learning_rate": 0.000291752134386253,
|
|
"loss_nats": 1.5188919305801392,
|
|
"step": 1250,
|
|
"unclipped_grad_norm": 1.2303693294525146
|
|
},
|
|
{
|
|
"bits_per_byte": 2.124901717754182,
|
|
"learning_rate": 0.00029155898526256856,
|
|
"loss_nats": 1.472869634628296,
|
|
"step": 1260,
|
|
"unclipped_grad_norm": 1.2569689750671387
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0056606424661694,
|
|
"learning_rate": 0.0002913636735950141,
|
|
"loss_nats": 1.3902180194854736,
|
|
"step": 1270,
|
|
"unclipped_grad_norm": 1.3274110555648804
|
|
},
|
|
{
|
|
"bits_per_byte": 2.135876962601643,
|
|
"learning_rate": 0.0002911662027209272,
|
|
"loss_nats": 1.4804770946502686,
|
|
"step": 1280,
|
|
"unclipped_grad_norm": 1.203521966934204
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0353620462701763,
|
|
"learning_rate": 0.00029096657601454013,
|
|
"loss_nats": 1.4108054637908936,
|
|
"step": 1290,
|
|
"unclipped_grad_norm": 1.1280517578125
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0816712987200523,
|
|
"learning_rate": 0.00029076479688692266,
|
|
"loss_nats": 1.4429045915603638,
|
|
"step": 1300,
|
|
"unclipped_grad_norm": 1.2584275007247925
|
|
},
|
|
{
|
|
"bits_per_byte": 2.144767433755345,
|
|
"learning_rate": 0.00029056086878592323,
|
|
"loss_nats": 1.4866394996643066,
|
|
"step": 1310,
|
|
"unclipped_grad_norm": 1.2562772035598755
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0818640912716693,
|
|
"learning_rate": 0.00029035479519611045,
|
|
"loss_nats": 1.4430382251739502,
|
|
"step": 1320,
|
|
"unclipped_grad_norm": 1.1640011072158813
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9798470784675668,
|
|
"learning_rate": 0.0002901465796387134,
|
|
"loss_nats": 1.3723254203796387,
|
|
"step": 1330,
|
|
"unclipped_grad_norm": 1.20253324508667
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0020558861040847,
|
|
"learning_rate": 0.00028993622567156154,
|
|
"loss_nats": 1.3877193927764893,
|
|
"step": 1340,
|
|
"unclipped_grad_norm": 1.1452504396438599
|
|
},
|
|
{
|
|
"bits_per_byte": 2.176502532405805,
|
|
"learning_rate": 0.0002897237368890237,
|
|
"loss_nats": 1.5086365938186646,
|
|
"step": 1350,
|
|
"unclipped_grad_norm": 1.2560323476791382
|
|
},
|
|
{
|
|
"bits_per_byte": 2.009120589436227,
|
|
"learning_rate": 0.0002895091169219469,
|
|
"loss_nats": 1.3926162719726562,
|
|
"step": 1360,
|
|
"unclipped_grad_norm": 1.1015381813049316
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0924102394052277,
|
|
"learning_rate": 0.00028929236943759425,
|
|
"loss_nats": 1.4503482580184937,
|
|
"step": 1370,
|
|
"unclipped_grad_norm": 1.1648008823394775
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0633229856474515,
|
|
"learning_rate": 0.00028907349813958225,
|
|
"loss_nats": 1.4301865100860596,
|
|
"step": 1380,
|
|
"unclipped_grad_norm": 1.124997854232788
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0840274610368725,
|
|
"learning_rate": 0.0002888525067678175,
|
|
"loss_nats": 1.4445377588272095,
|
|
"step": 1390,
|
|
"unclipped_grad_norm": 1.2330824136734009
|
|
},
|
|
{
|
|
"bits_per_byte": 2.084845582507027,
|
|
"learning_rate": 0.00028862939909843273,
|
|
"loss_nats": 1.4451048374176025,
|
|
"step": 1400,
|
|
"unclipped_grad_norm": 1.2093770503997803
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0117663705371007,
|
|
"learning_rate": 0.0002884041789437225,
|
|
"loss_nats": 1.3944501876831055,
|
|
"step": 1410,
|
|
"unclipped_grad_norm": 1.1717233657836914
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0551273244032355,
|
|
"learning_rate": 0.0002881768501520778,
|
|
"loss_nats": 1.4245057106018066,
|
|
"step": 1420,
|
|
"unclipped_grad_norm": 1.118058204650879
|
|
},
|
|
{
|
|
"bits_per_byte": 2.04396186674392,
|
|
"learning_rate": 0.0002879474166079205,
|
|
"loss_nats": 1.4167664051055908,
|
|
"step": 1430,
|
|
"unclipped_grad_norm": 1.282050371170044
|
|
},
|
|
{
|
|
"bits_per_byte": 2.01288047414939,
|
|
"learning_rate": 0.00028771588223163694,
|
|
"loss_nats": 1.3952224254608154,
|
|
"step": 1440,
|
|
"unclipped_grad_norm": 1.2009295225143433
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0606357567277205,
|
|
"learning_rate": 0.00028748225097951054,
|
|
"loss_nats": 1.4283238649368286,
|
|
"step": 1450,
|
|
"unclipped_grad_norm": 1.1897740364074707
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0059322030718816,
|
|
"learning_rate": 0.000287246526843655,
|
|
"loss_nats": 1.3904062509536743,
|
|
"step": 1460,
|
|
"unclipped_grad_norm": 1.0764319896697998
|
|
},
|
|
{
|
|
"bits_per_byte": 2.00337052148728,
|
|
"learning_rate": 0.00028700871385194533,
|
|
"loss_nats": 1.3886306285858154,
|
|
"step": 1470,
|
|
"unclipped_grad_norm": 1.102418065071106
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9153828214446602,
|
|
"learning_rate": 0.00028676881606794945,
|
|
"loss_nats": 1.3276422023773193,
|
|
"step": 1480,
|
|
"unclipped_grad_norm": 1.2223329544067383
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0269326606034372,
|
|
"learning_rate": 0.00028652683759085854,
|
|
"loss_nats": 1.4049626588821411,
|
|
"step": 1490,
|
|
"unclipped_grad_norm": 1.1502348184585571
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9137665284918515,
|
|
"learning_rate": 0.0002862827825554171,
|
|
"loss_nats": 1.326521873474121,
|
|
"step": 1500,
|
|
"unclipped_grad_norm": 1.104032039642334
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9998878728073082,
|
|
"learning_rate": 0.00028603665513185245,
|
|
"loss_nats": 1.386216640472412,
|
|
"step": 1510,
|
|
"unclipped_grad_norm": 1.2259501218795776
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9900984483375461,
|
|
"learning_rate": 0.000285788459525803,
|
|
"loss_nats": 1.379431128501892,
|
|
"step": 1520,
|
|
"unclipped_grad_norm": 1.192402958869934
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9577958069392392,
|
|
"learning_rate": 0.00028553819997824707,
|
|
"loss_nats": 1.3570406436920166,
|
|
"step": 1530,
|
|
"unclipped_grad_norm": 1.1483442783355713
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0275844748502077,
|
|
"learning_rate": 0.00028528588076542964,
|
|
"loss_nats": 1.4054144620895386,
|
|
"step": 1540,
|
|
"unclipped_grad_norm": 1.132032871246338
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0123937632474482,
|
|
"learning_rate": 0.00028503150619878996,
|
|
"loss_nats": 1.3948850631713867,
|
|
"step": 1550,
|
|
"unclipped_grad_norm": 1.1288913488388062
|
|
},
|
|
{
|
|
"bits_per_byte": 2.071048790289511,
|
|
"learning_rate": 0.0002847750806248876,
|
|
"loss_nats": 1.4355416297912598,
|
|
"step": 1560,
|
|
"unclipped_grad_norm": 1.146075963973999
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0218080935556784,
|
|
"learning_rate": 0.000284516608425328,
|
|
"loss_nats": 1.4014105796813965,
|
|
"step": 1570,
|
|
"unclipped_grad_norm": 1.1695427894592285
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0099278759993773,
|
|
"learning_rate": 0.000284256094016688,
|
|
"loss_nats": 1.3931758403778076,
|
|
"step": 1580,
|
|
"unclipped_grad_norm": 1.1285966634750366
|
|
},
|
|
{
|
|
"bits_per_byte": 1.999800161655368,
|
|
"learning_rate": 0.0002839935418504399,
|
|
"loss_nats": 1.3861558437347412,
|
|
"step": 1590,
|
|
"unclipped_grad_norm": 1.136389136314392
|
|
},
|
|
{
|
|
"bits_per_byte": 2.019440580383901,
|
|
"learning_rate": 0.000283728956412876,
|
|
"loss_nats": 1.3997695446014404,
|
|
"step": 1600,
|
|
"unclipped_grad_norm": 1.4346710443496704
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0397723693689023,
|
|
"learning_rate": 0.00028346234222503136,
|
|
"loss_nats": 1.4138624668121338,
|
|
"step": 1610,
|
|
"unclipped_grad_norm": 1.3147592544555664
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0494484572717453,
|
|
"learning_rate": 0.00028319370384260675,
|
|
"loss_nats": 1.4205694198608398,
|
|
"step": 1620,
|
|
"unclipped_grad_norm": 1.2143282890319824
|
|
},
|
|
{
|
|
"bits_per_byte": 1.921803965697274,
|
|
"learning_rate": 0.00028292304585589105,
|
|
"loss_nats": 1.3320930004119873,
|
|
"step": 1630,
|
|
"unclipped_grad_norm": 1.3924999237060547
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0487386848716347,
|
|
"learning_rate": 0.0002826503728896824,
|
|
"loss_nats": 1.4200774431228638,
|
|
"step": 1640,
|
|
"unclipped_grad_norm": 1.1033058166503906
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9803865880433233,
|
|
"learning_rate": 0.0002823756896032095,
|
|
"loss_nats": 1.3726993799209595,
|
|
"step": 1650,
|
|
"unclipped_grad_norm": 1.1198365688323975
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9604881953384965,
|
|
"learning_rate": 0.0002820990006900518,
|
|
"loss_nats": 1.358906865119934,
|
|
"step": 1660,
|
|
"unclipped_grad_norm": 1.0561511516571045
|
|
},
|
|
{
|
|
"bits_per_byte": 2.1180337625746266,
|
|
"learning_rate": 0.00028182031087805947,
|
|
"loss_nats": 1.468109130859375,
|
|
"step": 1670,
|
|
"unclipped_grad_norm": 1.1442371606826782
|
|
},
|
|
{
|
|
"bits_per_byte": 2.035602821981384,
|
|
"learning_rate": 0.0002815396249292725,
|
|
"loss_nats": 1.4109723567962646,
|
|
"step": 1680,
|
|
"unclipped_grad_norm": 1.0968410968780518
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9074798746722124,
|
|
"learning_rate": 0.0002812569476398391,
|
|
"loss_nats": 1.3221642971038818,
|
|
"step": 1690,
|
|
"unclipped_grad_norm": 1.049187183380127
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9460256582794904,
|
|
"learning_rate": 0.00028097228383993446,
|
|
"loss_nats": 1.3488821983337402,
|
|
"step": 1700,
|
|
"unclipped_grad_norm": 1.1202492713928223
|
|
},
|
|
{
|
|
"bits_per_byte": 2.030805537918217,
|
|
"learning_rate": 0.0002806856383936774,
|
|
"loss_nats": 1.4076471328735352,
|
|
"step": 1710,
|
|
"unclipped_grad_norm": 1.272226095199585
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0120939974869945,
|
|
"learning_rate": 0.00028039701619904774,
|
|
"loss_nats": 1.3946772813796997,
|
|
"step": 1720,
|
|
"unclipped_grad_norm": 1.1422768831253052
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9049754633103482,
|
|
"learning_rate": 0.0002801064221878024,
|
|
"loss_nats": 1.3204283714294434,
|
|
"step": 1730,
|
|
"unclipped_grad_norm": 1.1759874820709229
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9628171843964806,
|
|
"learning_rate": 0.00027981386132539134,
|
|
"loss_nats": 1.3605211973190308,
|
|
"step": 1740,
|
|
"unclipped_grad_norm": 1.0964117050170898
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0712307479341234,
|
|
"learning_rate": 0.0002795193386108724,
|
|
"loss_nats": 1.4356677532196045,
|
|
"step": 1750,
|
|
"unclipped_grad_norm": 1.1146889925003052
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9217773083863903,
|
|
"learning_rate": 0.00027922285907682625,
|
|
"loss_nats": 1.332074522972107,
|
|
"step": 1760,
|
|
"unclipped_grad_norm": 1.044190764427185
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9951038314082574,
|
|
"learning_rate": 0.00027892442778927005,
|
|
"loss_nats": 1.382900595664978,
|
|
"step": 1770,
|
|
"unclipped_grad_norm": 1.0358567237854004
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9346029145744843,
|
|
"learning_rate": 0.00027862404984757116,
|
|
"loss_nats": 1.3409645557403564,
|
|
"step": 1780,
|
|
"unclipped_grad_norm": 1.1725130081176758
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9131714685198662,
|
|
"learning_rate": 0.0002783217303843597,
|
|
"loss_nats": 1.3261094093322754,
|
|
"step": 1790,
|
|
"unclipped_grad_norm": 1.2142103910446167
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9953318804033016,
|
|
"learning_rate": 0.00027801747456544134,
|
|
"loss_nats": 1.3830586671829224,
|
|
"step": 1800,
|
|
"unclipped_grad_norm": 1.2420480251312256
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9723242133535257,
|
|
"learning_rate": 0.00027771128758970843,
|
|
"loss_nats": 1.3671109676361084,
|
|
"step": 1810,
|
|
"unclipped_grad_norm": 1.1430673599243164
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0342909383206025,
|
|
"learning_rate": 0.00027740317468905165,
|
|
"loss_nats": 1.4100630283355713,
|
|
"step": 1820,
|
|
"unclipped_grad_norm": 1.1687225103378296
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8908906001526369,
|
|
"learning_rate": 0.0002770931411282704,
|
|
"loss_nats": 1.310665488243103,
|
|
"step": 1830,
|
|
"unclipped_grad_norm": 1.146242380142212
|
|
},
|
|
{
|
|
"bits_per_byte": 1.978836336428446,
|
|
"learning_rate": 0.0002767811922049827,
|
|
"loss_nats": 1.3716248273849487,
|
|
"step": 1840,
|
|
"unclipped_grad_norm": 1.0820521116256714
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8676517884201014,
|
|
"learning_rate": 0.00027646733324953503,
|
|
"loss_nats": 1.2945575714111328,
|
|
"step": 1850,
|
|
"unclipped_grad_norm": 1.0944465398788452
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0069171477133727,
|
|
"learning_rate": 0.00027615156962491097,
|
|
"loss_nats": 1.3910889625549316,
|
|
"step": 1860,
|
|
"unclipped_grad_norm": 1.2028480768203735
|
|
},
|
|
{
|
|
"bits_per_byte": 2.0278259384920188,
|
|
"learning_rate": 0.0002758339067266396,
|
|
"loss_nats": 1.4055818319320679,
|
|
"step": 1870,
|
|
"unclipped_grad_norm": 1.1793227195739746
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8922862393643884,
|
|
"learning_rate": 0.0002755143499827035,
|
|
"loss_nats": 1.3116328716278076,
|
|
"step": 1880,
|
|
"unclipped_grad_norm": 1.123720645904541
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9751340659033223,
|
|
"learning_rate": 0.0002751929048534456,
|
|
"loss_nats": 1.369058609008789,
|
|
"step": 1890,
|
|
"unclipped_grad_norm": 1.006156086921692
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8741764662285347,
|
|
"learning_rate": 0.0002748695768314763,
|
|
"loss_nats": 1.2990801334381104,
|
|
"step": 1900,
|
|
"unclipped_grad_norm": 1.0260387659072876
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8761183223394267,
|
|
"learning_rate": 0.00027454437144157947,
|
|
"loss_nats": 1.3004261255264282,
|
|
"step": 1910,
|
|
"unclipped_grad_norm": 1.11003839969635
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9401261374069434,
|
|
"learning_rate": 0.00027421729424061787,
|
|
"loss_nats": 1.3447929620742798,
|
|
"step": 1920,
|
|
"unclipped_grad_norm": 1.1032140254974365
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9031926911515042,
|
|
"learning_rate": 0.0002738883508174385,
|
|
"loss_nats": 1.31919264793396,
|
|
"step": 1930,
|
|
"unclipped_grad_norm": 1.1424028873443604
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8781860697507502,
|
|
"learning_rate": 0.0002735575467927768,
|
|
"loss_nats": 1.3018593788146973,
|
|
"step": 1940,
|
|
"unclipped_grad_norm": 1.0988123416900635
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9390512458390508,
|
|
"learning_rate": 0.0002732248878191609,
|
|
"loss_nats": 1.3440479040145874,
|
|
"step": 1950,
|
|
"unclipped_grad_norm": 1.28093421459198
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9095305958011006,
|
|
"learning_rate": 0.0002728903795808148,
|
|
"loss_nats": 1.3235857486724854,
|
|
"step": 1960,
|
|
"unclipped_grad_norm": 1.052490234375
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8582639434401043,
|
|
"learning_rate": 0.0002725540277935614,
|
|
"loss_nats": 1.2880504131317139,
|
|
"step": 1970,
|
|
"unclipped_grad_norm": 1.0047657489776611
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8448327863383223,
|
|
"learning_rate": 0.00027221583820472456,
|
|
"loss_nats": 1.278740644454956,
|
|
"step": 1980,
|
|
"unclipped_grad_norm": 1.101292610168457
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9462698736437154,
|
|
"learning_rate": 0.0002718758165930314,
|
|
"loss_nats": 1.3490514755249023,
|
|
"step": 1990,
|
|
"unclipped_grad_norm": 1.17987859249115
|
|
},
|
|
{
|
|
"bits_per_byte": 1.896080176642422,
|
|
"learning_rate": 0.0002715339687685131,
|
|
"loss_nats": 1.3142626285552979,
|
|
"step": 2000,
|
|
"unclipped_grad_norm": 1.1739213466644287
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9557370026257606,
|
|
"learning_rate": 0.00027119030057240576,
|
|
"loss_nats": 1.3556135892868042,
|
|
"step": 2010,
|
|
"unclipped_grad_norm": 1.0457618236541748
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9291909645171335,
|
|
"learning_rate": 0.0002708448178770506,
|
|
"loss_nats": 1.3372132778167725,
|
|
"step": 2020,
|
|
"unclipped_grad_norm": 1.1008185148239136
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9305355248815788,
|
|
"learning_rate": 0.00027049752658579376,
|
|
"loss_nats": 1.3381452560424805,
|
|
"step": 2030,
|
|
"unclipped_grad_norm": 1.1080964803695679
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9343679862734056,
|
|
"learning_rate": 0.00027014843263288514,
|
|
"loss_nats": 1.34080171585083,
|
|
"step": 2040,
|
|
"unclipped_grad_norm": 1.104992389678955
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8835350741578853,
|
|
"learning_rate": 0.00026979754198337724,
|
|
"loss_nats": 1.3055670261383057,
|
|
"step": 2050,
|
|
"unclipped_grad_norm": 1.1497511863708496
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7856179557677012,
|
|
"learning_rate": 0.00026944486063302305,
|
|
"loss_nats": 1.2376960515975952,
|
|
"step": 2060,
|
|
"unclipped_grad_norm": 1.0708798170089722
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8782170266279055,
|
|
"learning_rate": 0.0002690903946081738,
|
|
"loss_nats": 1.3018808364868164,
|
|
"step": 2070,
|
|
"unclipped_grad_norm": 1.0489474534988403
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9968320850667778,
|
|
"learning_rate": 0.00026873414996567596,
|
|
"loss_nats": 1.3840985298156738,
|
|
"step": 2080,
|
|
"unclipped_grad_norm": 1.0985883474349976
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8512350124993413,
|
|
"learning_rate": 0.00026837613279276736,
|
|
"loss_nats": 1.2831783294677734,
|
|
"step": 2090,
|
|
"unclipped_grad_norm": 0.9845348000526428
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9251153196569866,
|
|
"learning_rate": 0.0002680163492069739,
|
|
"loss_nats": 1.334388256072998,
|
|
"step": 2100,
|
|
"unclipped_grad_norm": 1.009676456451416
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8359577936231979,
|
|
"learning_rate": 0.00026765480535600414,
|
|
"loss_nats": 1.2725889682769775,
|
|
"step": 2110,
|
|
"unclipped_grad_norm": 1.0760974884033203
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8868116876221235,
|
|
"learning_rate": 0.0002672915074176452,
|
|
"loss_nats": 1.3078382015228271,
|
|
"step": 2120,
|
|
"unclipped_grad_norm": 0.9828676581382751
|
|
},
|
|
{
|
|
"bits_per_byte": 1.910013867050025,
|
|
"learning_rate": 0.00026692646159965624,
|
|
"loss_nats": 1.323920726776123,
|
|
"step": 2130,
|
|
"unclipped_grad_norm": 1.0712695121765137
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8022548694815659,
|
|
"learning_rate": 0.0002665596741396633,
|
|
"loss_nats": 1.2492278814315796,
|
|
"step": 2140,
|
|
"unclipped_grad_norm": 1.0451561212539673
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8551622363317934,
|
|
"learning_rate": 0.000266191151305052,
|
|
"loss_nats": 1.2859004735946655,
|
|
"step": 2150,
|
|
"unclipped_grad_norm": 1.0957838296890259
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9088151479735114,
|
|
"learning_rate": 0.00026582089939286075,
|
|
"loss_nats": 1.323089838027954,
|
|
"step": 2160,
|
|
"unclipped_grad_norm": 1.0826293230056763
|
|
},
|
|
{
|
|
"bits_per_byte": 1.871177432762789,
|
|
"learning_rate": 0.0002654489247296732,
|
|
"loss_nats": 1.2970013618469238,
|
|
"step": 2170,
|
|
"unclipped_grad_norm": 1.0311518907546997
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8055650195627224,
|
|
"learning_rate": 0.00026507523367151007,
|
|
"loss_nats": 1.2515223026275635,
|
|
"step": 2180,
|
|
"unclipped_grad_norm": 1.040400505065918
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9270800494604432,
|
|
"learning_rate": 0.00026469983260372037,
|
|
"loss_nats": 1.3357501029968262,
|
|
"step": 2190,
|
|
"unclipped_grad_norm": 1.10823392868042
|
|
},
|
|
{
|
|
"bits_per_byte": 1.980437322925328,
|
|
"learning_rate": 0.0002643227279408727,
|
|
"loss_nats": 1.372734546661377,
|
|
"step": 2200,
|
|
"unclipped_grad_norm": 1.11663019657135
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9995734885215308,
|
|
"learning_rate": 0.0002639439261266453,
|
|
"loss_nats": 1.3859987258911133,
|
|
"step": 2210,
|
|
"unclipped_grad_norm": 1.0498319864273071
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8802302555389057,
|
|
"learning_rate": 0.000263563433633716,
|
|
"loss_nats": 1.3032763004302979,
|
|
"step": 2220,
|
|
"unclipped_grad_norm": 1.0160832405090332
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8937652901618087,
|
|
"learning_rate": 0.00026318125696365156,
|
|
"loss_nats": 1.3126580715179443,
|
|
"step": 2230,
|
|
"unclipped_grad_norm": 1.0416548252105713
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8469913405893015,
|
|
"learning_rate": 0.000262797402646797,
|
|
"loss_nats": 1.280236840248108,
|
|
"step": 2240,
|
|
"unclipped_grad_norm": 1.0407441854476929
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8529231942002105,
|
|
"learning_rate": 0.00026241187724216334,
|
|
"loss_nats": 1.284348487854004,
|
|
"step": 2250,
|
|
"unclipped_grad_norm": 0.9867598414421082
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8825788506190881,
|
|
"learning_rate": 0.0002620246873373162,
|
|
"loss_nats": 1.3049042224884033,
|
|
"step": 2260,
|
|
"unclipped_grad_norm": 0.9744513034820557
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9268210435882438,
|
|
"learning_rate": 0.00026163583954826264,
|
|
"loss_nats": 1.3355705738067627,
|
|
"step": 2270,
|
|
"unclipped_grad_norm": 1.0475958585739136
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8280555347813536,
|
|
"learning_rate": 0.0002612453405193386,
|
|
"loss_nats": 1.2671115398406982,
|
|
"step": 2280,
|
|
"unclipped_grad_norm": 1.0294238328933716
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8388550133596329,
|
|
"learning_rate": 0.000260853196923095,
|
|
"loss_nats": 1.27459716796875,
|
|
"step": 2290,
|
|
"unclipped_grad_norm": 1.0671623945236206
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8867002428643644,
|
|
"learning_rate": 0.0002604594154601839,
|
|
"loss_nats": 1.3077609539031982,
|
|
"step": 2300,
|
|
"unclipped_grad_norm": 0.9974833726882935
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8601865374940996,
|
|
"learning_rate": 0.0002600640028592441,
|
|
"loss_nats": 1.289383053779602,
|
|
"step": 2310,
|
|
"unclipped_grad_norm": 1.0144871473312378
|
|
},
|
|
{
|
|
"bits_per_byte": 1.874428248829398,
|
|
"learning_rate": 0.0002596669658767859,
|
|
"loss_nats": 1.2992546558380127,
|
|
"step": 2320,
|
|
"unclipped_grad_norm": 1.0221670866012573
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8178058847378828,
|
|
"learning_rate": 0.000259268311297076,
|
|
"loss_nats": 1.2600070238113403,
|
|
"step": 2330,
|
|
"unclipped_grad_norm": 1.0877128839492798
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8362632348111303,
|
|
"learning_rate": 0.00025886804593202114,
|
|
"loss_nats": 1.2728006839752197,
|
|
"step": 2340,
|
|
"unclipped_grad_norm": 1.050228238105774
|
|
},
|
|
{
|
|
"bits_per_byte": 1.780467247357011,
|
|
"learning_rate": 0.00025846617662105193,
|
|
"loss_nats": 1.2341258525848389,
|
|
"step": 2350,
|
|
"unclipped_grad_norm": 0.9972993731498718
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8791837411084054,
|
|
"learning_rate": 0.00025806271023100637,
|
|
"loss_nats": 1.3025509119033813,
|
|
"step": 2360,
|
|
"unclipped_grad_norm": 1.0690455436706543
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8768631792003134,
|
|
"learning_rate": 0.00025765765365601165,
|
|
"loss_nats": 1.3009424209594727,
|
|
"step": 2370,
|
|
"unclipped_grad_norm": 1.0405195951461792
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8106722163627444,
|
|
"learning_rate": 0.00025725101381736707,
|
|
"loss_nats": 1.2550623416900635,
|
|
"step": 2380,
|
|
"unclipped_grad_norm": 1.000722050666809
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7959209204329396,
|
|
"learning_rate": 0.00025684279766342567,
|
|
"loss_nats": 1.2448375225067139,
|
|
"step": 2390,
|
|
"unclipped_grad_norm": 1.0424110889434814
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9177011475782908,
|
|
"learning_rate": 0.00025643301216947504,
|
|
"loss_nats": 1.3292491436004639,
|
|
"step": 2400,
|
|
"unclipped_grad_norm": 1.0584142208099365
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8792028311826512,
|
|
"learning_rate": 0.00025602166433761866,
|
|
"loss_nats": 1.3025641441345215,
|
|
"step": 2410,
|
|
"unclipped_grad_norm": 1.109039068222046
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9615544877738458,
|
|
"learning_rate": 0.0002556087611966561,
|
|
"loss_nats": 1.359645962715149,
|
|
"step": 2420,
|
|
"unclipped_grad_norm": 1.0733245611190796
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8745142401548294,
|
|
"learning_rate": 0.00025519430980196277,
|
|
"loss_nats": 1.299314260482788,
|
|
"step": 2430,
|
|
"unclipped_grad_norm": 1.0562880039215088
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7973332419618249,
|
|
"learning_rate": 0.0002547783172353695,
|
|
"loss_nats": 1.2458164691925049,
|
|
"step": 2440,
|
|
"unclipped_grad_norm": 1.0554269552230835
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8932479663480133,
|
|
"learning_rate": 0.0002543607906050415,
|
|
"loss_nats": 1.3122994899749756,
|
|
"step": 2450,
|
|
"unclipped_grad_norm": 1.1328672170639038
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9065160838967772,
|
|
"learning_rate": 0.0002539417370453568,
|
|
"loss_nats": 1.3214962482452393,
|
|
"step": 2460,
|
|
"unclipped_grad_norm": 1.1163077354431152
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8460499075584784,
|
|
"learning_rate": 0.00025352116371678464,
|
|
"loss_nats": 1.279584288597107,
|
|
"step": 2470,
|
|
"unclipped_grad_norm": 1.0860918760299683
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8485799443353212,
|
|
"learning_rate": 0.0002530990778057627,
|
|
"loss_nats": 1.2813379764556885,
|
|
"step": 2480,
|
|
"unclipped_grad_norm": 0.99190354347229
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7724624868552525,
|
|
"learning_rate": 0.00025267548652457475,
|
|
"loss_nats": 1.2285773754119873,
|
|
"step": 2490,
|
|
"unclipped_grad_norm": 1.025232195854187
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8512375922391042,
|
|
"learning_rate": 0.00025225039711122696,
|
|
"loss_nats": 1.2831801176071167,
|
|
"step": 2500,
|
|
"unclipped_grad_norm": 1.0759034156799316
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8193396260182773,
|
|
"learning_rate": 0.00025182381682932464,
|
|
"loss_nats": 1.2610701322555542,
|
|
"step": 2510,
|
|
"unclipped_grad_norm": 1.0067415237426758
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8146814759196581,
|
|
"learning_rate": 0.0002513957529679477,
|
|
"loss_nats": 1.2578413486480713,
|
|
"step": 2520,
|
|
"unclipped_grad_norm": 0.990001916885376
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8138671380678228,
|
|
"learning_rate": 0.00025096621284152656,
|
|
"loss_nats": 1.2572768926620483,
|
|
"step": 2530,
|
|
"unclipped_grad_norm": 0.9393356442451477
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8368911434694304,
|
|
"learning_rate": 0.0002505352037897168,
|
|
"loss_nats": 1.2732359170913696,
|
|
"step": 2540,
|
|
"unclipped_grad_norm": 1.1141552925109863
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8603834576293374,
|
|
"learning_rate": 0.0002501027331772739,
|
|
"loss_nats": 1.2895195484161377,
|
|
"step": 2550,
|
|
"unclipped_grad_norm": 1.0569089651107788
|
|
},
|
|
{
|
|
"bits_per_byte": 1.816235855118156,
|
|
"learning_rate": 0.00024966880839392746,
|
|
"loss_nats": 1.2589187622070312,
|
|
"step": 2560,
|
|
"unclipped_grad_norm": 0.9373295903205872
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8059492288047498,
|
|
"learning_rate": 0.0002492334368542548,
|
|
"loss_nats": 1.25178861618042,
|
|
"step": 2570,
|
|
"unclipped_grad_norm": 1.143198847770691
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8450266107858446,
|
|
"learning_rate": 0.00024879662599755416,
|
|
"loss_nats": 1.2788749933242798,
|
|
"step": 2580,
|
|
"unclipped_grad_norm": 1.057027816772461
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8154227211448768,
|
|
"learning_rate": 0.00024835838328771803,
|
|
"loss_nats": 1.2583551406860352,
|
|
"step": 2590,
|
|
"unclipped_grad_norm": 1.0370285511016846
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8561616275159571,
|
|
"learning_rate": 0.0002479187162131051,
|
|
"loss_nats": 1.2865931987762451,
|
|
"step": 2600,
|
|
"unclipped_grad_norm": 0.9653204083442688
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7605767658541238,
|
|
"learning_rate": 0.00024747763228641264,
|
|
"loss_nats": 1.2203388214111328,
|
|
"step": 2610,
|
|
"unclipped_grad_norm": 1.0564771890640259
|
|
},
|
|
{
|
|
"bits_per_byte": 1.812449141111459,
|
|
"learning_rate": 0.00024703513904454795,
|
|
"loss_nats": 1.2562940120697021,
|
|
"step": 2620,
|
|
"unclipped_grad_norm": 1.0267692804336548
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8349191903946374,
|
|
"learning_rate": 0.00024659124404849956,
|
|
"loss_nats": 1.2718690633773804,
|
|
"step": 2630,
|
|
"unclipped_grad_norm": 0.9602619409561157
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8363758834474455,
|
|
"learning_rate": 0.00024614595488320833,
|
|
"loss_nats": 1.2728787660598755,
|
|
"step": 2640,
|
|
"unclipped_grad_norm": 1.0029752254486084
|
|
},
|
|
{
|
|
"bits_per_byte": 1.771385015547597,
|
|
"learning_rate": 0.00024569927915743745,
|
|
"loss_nats": 1.2278305292129517,
|
|
"step": 2650,
|
|
"unclipped_grad_norm": 0.9146384000778198
|
|
},
|
|
{
|
|
"bits_per_byte": 1.779632271587072,
|
|
"learning_rate": 0.0002452512245036427,
|
|
"loss_nats": 1.2335470914840698,
|
|
"step": 2660,
|
|
"unclipped_grad_norm": 0.9666176438331604
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8136729696549987,
|
|
"learning_rate": 0.00024480179857784177,
|
|
"loss_nats": 1.2571423053741455,
|
|
"step": 2670,
|
|
"unclipped_grad_norm": 1.0540763139724731
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7953530337197905,
|
|
"learning_rate": 0.00024435100905948387,
|
|
"loss_nats": 1.2444438934326172,
|
|
"step": 2680,
|
|
"unclipped_grad_norm": 1.0946050882339478
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8769845989518226,
|
|
"learning_rate": 0.00024389886365131793,
|
|
"loss_nats": 1.3010265827178955,
|
|
"step": 2690,
|
|
"unclipped_grad_norm": 1.068537712097168
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8642107595416384,
|
|
"learning_rate": 0.00024344537007926163,
|
|
"loss_nats": 1.2921724319458008,
|
|
"step": 2700,
|
|
"unclipped_grad_norm": 1.1353117227554321
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7364708175959382,
|
|
"learning_rate": 0.00024299053609226878,
|
|
"loss_nats": 1.2036298513412476,
|
|
"step": 2710,
|
|
"unclipped_grad_norm": 0.9047647714614868
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8594279220211436,
|
|
"learning_rate": 0.00024253436946219733,
|
|
"loss_nats": 1.2888572216033936,
|
|
"step": 2720,
|
|
"unclipped_grad_norm": 1.0444105863571167
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7814004252205926,
|
|
"learning_rate": 0.0002420768779836764,
|
|
"loss_nats": 1.2347726821899414,
|
|
"step": 2730,
|
|
"unclipped_grad_norm": 0.9630917310714722
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7840532576101513,
|
|
"learning_rate": 0.00024161806947397305,
|
|
"loss_nats": 1.2366114854812622,
|
|
"step": 2740,
|
|
"unclipped_grad_norm": 1.0223243236541748
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8198583256932797,
|
|
"learning_rate": 0.00024115795177285886,
|
|
"loss_nats": 1.2614296674728394,
|
|
"step": 2750,
|
|
"unclipped_grad_norm": 1.0687127113342285
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9021738659277931,
|
|
"learning_rate": 0.0002406965327424758,
|
|
"loss_nats": 1.3184864521026611,
|
|
"step": 2760,
|
|
"unclipped_grad_norm": 1.0979900360107422
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7971648709466301,
|
|
"learning_rate": 0.00024023382026720197,
|
|
"loss_nats": 1.2456997632980347,
|
|
"step": 2770,
|
|
"unclipped_grad_norm": 0.965108335018158
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8086140999798692,
|
|
"learning_rate": 0.00023976982225351698,
|
|
"loss_nats": 1.2536357641220093,
|
|
"step": 2780,
|
|
"unclipped_grad_norm": 1.0402190685272217
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7938860217079309,
|
|
"learning_rate": 0.0002393045466298666,
|
|
"loss_nats": 1.243427038192749,
|
|
"step": 2790,
|
|
"unclipped_grad_norm": 1.0398236513137817
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7550451158717724,
|
|
"learning_rate": 0.00023883800134652763,
|
|
"loss_nats": 1.2165045738220215,
|
|
"step": 2800,
|
|
"unclipped_grad_norm": 1.0063706636428833
|
|
},
|
|
{
|
|
"bits_per_byte": 1.828329675126829,
|
|
"learning_rate": 0.0002383701943754715,
|
|
"loss_nats": 1.2673015594482422,
|
|
"step": 2810,
|
|
"unclipped_grad_norm": 0.9668177366256714
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8254919613875926,
|
|
"learning_rate": 0.00023790113371022884,
|
|
"loss_nats": 1.2653346061706543,
|
|
"step": 2820,
|
|
"unclipped_grad_norm": 0.9527478814125061
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7516809632382448,
|
|
"learning_rate": 0.0002374308273657521,
|
|
"loss_nats": 1.2141727209091187,
|
|
"step": 2830,
|
|
"unclipped_grad_norm": 0.9219958186149597
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8254823303591443,
|
|
"learning_rate": 0.00023695928337827911,
|
|
"loss_nats": 1.2653279304504395,
|
|
"step": 2840,
|
|
"unclipped_grad_norm": 1.0127615928649902
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7530225998976257,
|
|
"learning_rate": 0.00023648650980519564,
|
|
"loss_nats": 1.2151026725769043,
|
|
"step": 2850,
|
|
"unclipped_grad_norm": 1.0182650089263916
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7287747659524784,
|
|
"learning_rate": 0.00023601251472489757,
|
|
"loss_nats": 1.1982953548431396,
|
|
"step": 2860,
|
|
"unclipped_grad_norm": 0.978410542011261
|
|
},
|
|
{
|
|
"bits_per_byte": 1.855343334063152,
|
|
"learning_rate": 0.00023553730623665305,
|
|
"loss_nats": 1.2860260009765625,
|
|
"step": 2870,
|
|
"unclipped_grad_norm": 1.018862009048462
|
|
},
|
|
{
|
|
"bits_per_byte": 1.768852571013642,
|
|
"learning_rate": 0.00023506089246046403,
|
|
"loss_nats": 1.2260751724243164,
|
|
"step": 2880,
|
|
"unclipped_grad_norm": 0.9639438390731812
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8105554401428086,
|
|
"learning_rate": 0.00023458328153692736,
|
|
"loss_nats": 1.2549813985824585,
|
|
"step": 2890,
|
|
"unclipped_grad_norm": 0.9698073267936707
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7403103302764504,
|
|
"learning_rate": 0.0002341044816270961,
|
|
"loss_nats": 1.2062911987304688,
|
|
"step": 2900,
|
|
"unclipped_grad_norm": 0.9920547604560852
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7643693272709504,
|
|
"learning_rate": 0.00023362450091233953,
|
|
"loss_nats": 1.2229676246643066,
|
|
"step": 2910,
|
|
"unclipped_grad_norm": 1.0128531455993652
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8473500963990013,
|
|
"learning_rate": 0.00023314334759420388,
|
|
"loss_nats": 1.2804855108261108,
|
|
"step": 2920,
|
|
"unclipped_grad_norm": 1.0619620084762573
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8003603085996611,
|
|
"learning_rate": 0.0002326610298942717,
|
|
"loss_nats": 1.2479146718978882,
|
|
"step": 2930,
|
|
"unclipped_grad_norm": 1.0764249563217163
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8426398355571707,
|
|
"learning_rate": 0.00023217755605402175,
|
|
"loss_nats": 1.277220606803894,
|
|
"step": 2940,
|
|
"unclipped_grad_norm": 1.0425148010253906
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7862247105599451,
|
|
"learning_rate": 0.00023169293433468802,
|
|
"loss_nats": 1.2381166219711304,
|
|
"step": 2950,
|
|
"unclipped_grad_norm": 1.004768967628479
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7811238771180051,
|
|
"learning_rate": 0.00023120717301711857,
|
|
"loss_nats": 1.2345809936523438,
|
|
"step": 2960,
|
|
"unclipped_grad_norm": 0.9997458457946777
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8373286673332254,
|
|
"learning_rate": 0.00023072028040163408,
|
|
"loss_nats": 1.2735391855239868,
|
|
"step": 2970,
|
|
"unclipped_grad_norm": 1.039349913597107
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7631166056420657,
|
|
"learning_rate": 0.00023023226480788593,
|
|
"loss_nats": 1.2220993041992188,
|
|
"step": 2980,
|
|
"unclipped_grad_norm": 0.9808847904205322
|
|
},
|
|
{
|
|
"bits_per_byte": 1.9032267437163752,
|
|
"learning_rate": 0.00022974313457471416,
|
|
"loss_nats": 1.319216251373291,
|
|
"step": 2990,
|
|
"unclipped_grad_norm": 1.1331971883773804
|
|
},
|
|
{
|
|
"bits_per_byte": 1.81958332543455,
|
|
"learning_rate": 0.0002292528980600049,
|
|
"loss_nats": 1.2612390518188477,
|
|
"step": 3000,
|
|
"unclipped_grad_norm": 0.944084107875824
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7894560925870064,
|
|
"learning_rate": 0.00022876156364054758,
|
|
"loss_nats": 1.2403564453125,
|
|
"step": 3010,
|
|
"unclipped_grad_norm": 1.057902216911316
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7654951257034983,
|
|
"learning_rate": 0.0002282691397118918,
|
|
"loss_nats": 1.223747968673706,
|
|
"step": 3020,
|
|
"unclipped_grad_norm": 0.971477746963501
|
|
},
|
|
{
|
|
"bits_per_byte": 1.791092679492617,
|
|
"learning_rate": 0.00022777563468820376,
|
|
"loss_nats": 1.2414908409118652,
|
|
"step": 3030,
|
|
"unclipped_grad_norm": 1.017361044883728
|
|
},
|
|
{
|
|
"bits_per_byte": 1.760655361925568,
|
|
"learning_rate": 0.00022728105700212282,
|
|
"loss_nats": 1.2203933000564575,
|
|
"step": 3040,
|
|
"unclipped_grad_norm": 0.9738672971725464
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7859287284178103,
|
|
"learning_rate": 0.00022678541510461698,
|
|
"loss_nats": 1.2379114627838135,
|
|
"step": 3050,
|
|
"unclipped_grad_norm": 1.0311107635498047
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7871233199107035,
|
|
"learning_rate": 0.00022628871746483876,
|
|
"loss_nats": 1.2387394905090332,
|
|
"step": 3060,
|
|
"unclipped_grad_norm": 1.097463607788086
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7530494291911602,
|
|
"learning_rate": 0.0002257909725699804,
|
|
"loss_nats": 1.2151212692260742,
|
|
"step": 3070,
|
|
"unclipped_grad_norm": 0.95722895860672
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7753578147825277,
|
|
"learning_rate": 0.00022529218892512886,
|
|
"loss_nats": 1.2305842638015747,
|
|
"step": 3080,
|
|
"unclipped_grad_norm": 1.0311440229415894
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7406597990230035,
|
|
"learning_rate": 0.00022479237505312034,
|
|
"loss_nats": 1.206533432006836,
|
|
"step": 3090,
|
|
"unclipped_grad_norm": 0.9659063220024109
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8326863396384856,
|
|
"learning_rate": 0.00022429153949439496,
|
|
"loss_nats": 1.2703213691711426,
|
|
"step": 3100,
|
|
"unclipped_grad_norm": 1.0265647172927856
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8128345542320425,
|
|
"learning_rate": 0.00022378969080685053,
|
|
"loss_nats": 1.2565611600875854,
|
|
"step": 3110,
|
|
"unclipped_grad_norm": 1.0288156270980835
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6999466900495526,
|
|
"learning_rate": 0.0002232868375656965,
|
|
"loss_nats": 1.1783132553100586,
|
|
"step": 3120,
|
|
"unclipped_grad_norm": 0.9754294753074646
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7980240962703409,
|
|
"learning_rate": 0.00022278298836330733,
|
|
"loss_nats": 1.2462953329086304,
|
|
"step": 3130,
|
|
"unclipped_grad_norm": 1.0256381034851074
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7337763654048706,
|
|
"learning_rate": 0.00022227815180907572,
|
|
"loss_nats": 1.2017621994018555,
|
|
"step": 3140,
|
|
"unclipped_grad_norm": 0.9683999419212341
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7391683654747212,
|
|
"learning_rate": 0.00022177233652926546,
|
|
"loss_nats": 1.2054996490478516,
|
|
"step": 3150,
|
|
"unclipped_grad_norm": 0.9884417057037354
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7859572775378536,
|
|
"learning_rate": 0.00022126555116686416,
|
|
"loss_nats": 1.237931251525879,
|
|
"step": 3160,
|
|
"unclipped_grad_norm": 1.0163674354553223
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8529599984874952,
|
|
"learning_rate": 0.00022075780438143532,
|
|
"loss_nats": 1.2843739986419678,
|
|
"step": 3170,
|
|
"unclipped_grad_norm": 1.0605453252792358
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7071207743476438,
|
|
"learning_rate": 0.00022024910484897065,
|
|
"loss_nats": 1.1832859516143799,
|
|
"step": 3180,
|
|
"unclipped_grad_norm": 1.0476000308990479
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8011364663030052,
|
|
"learning_rate": 0.00021973946126174158,
|
|
"loss_nats": 1.2484526634216309,
|
|
"step": 3190,
|
|
"unclipped_grad_norm": 0.9400752782821655
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7046130953154133,
|
|
"learning_rate": 0.00021922888232815086,
|
|
"loss_nats": 1.18154776096344,
|
|
"step": 3200,
|
|
"unclipped_grad_norm": 0.9428834915161133
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7343896555378473,
|
|
"learning_rate": 0.0002187173767725837,
|
|
"loss_nats": 1.2021872997283936,
|
|
"step": 3210,
|
|
"unclipped_grad_norm": 0.982049822807312
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7352428614687776,
|
|
"learning_rate": 0.0002182049533352588,
|
|
"loss_nats": 1.202778697013855,
|
|
"step": 3220,
|
|
"unclipped_grad_norm": 0.9146644473075867
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7184315573469382,
|
|
"learning_rate": 0.00021769162077207884,
|
|
"loss_nats": 1.1911259889602661,
|
|
"step": 3230,
|
|
"unclipped_grad_norm": 0.9842981696128845
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7760342225483712,
|
|
"learning_rate": 0.00021717738785448102,
|
|
"loss_nats": 1.231053113937378,
|
|
"step": 3240,
|
|
"unclipped_grad_norm": 1.0283819437026978
|
|
},
|
|
{
|
|
"bits_per_byte": 1.780553754630395,
|
|
"learning_rate": 0.00021666226336928708,
|
|
"loss_nats": 1.234185814857483,
|
|
"step": 3250,
|
|
"unclipped_grad_norm": 0.9948496222496033
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8669033199235465,
|
|
"learning_rate": 0.0002161462561185532,
|
|
"loss_nats": 1.2940387725830078,
|
|
"step": 3260,
|
|
"unclipped_grad_norm": 1.0557905435562134
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7253210103578513,
|
|
"learning_rate": 0.00021562937491941957,
|
|
"loss_nats": 1.1959013938903809,
|
|
"step": 3270,
|
|
"unclipped_grad_norm": 1.0562386512756348
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6636741731213793,
|
|
"learning_rate": 0.00021511162860395975,
|
|
"loss_nats": 1.1531710624694824,
|
|
"step": 3280,
|
|
"unclipped_grad_norm": 0.9674620628356934
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7215127985197962,
|
|
"learning_rate": 0.0002145930260190298,
|
|
"loss_nats": 1.193261742591858,
|
|
"step": 3290,
|
|
"unclipped_grad_norm": 1.081762671470642
|
|
},
|
|
{
|
|
"bits_per_byte": 1.784075099406811,
|
|
"learning_rate": 0.00021407357602611698,
|
|
"loss_nats": 1.2366266250610352,
|
|
"step": 3300,
|
|
"unclipped_grad_norm": 0.9871881604194641
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7790666206483843,
|
|
"learning_rate": 0.0002135532875011885,
|
|
"loss_nats": 1.2331550121307373,
|
|
"step": 3310,
|
|
"unclipped_grad_norm": 1.018639326095581
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7964894950766919,
|
|
"learning_rate": 0.0002130321693345397,
|
|
"loss_nats": 1.2452316284179688,
|
|
"step": 3320,
|
|
"unclipped_grad_norm": 1.0376429557800293
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7442502808250664,
|
|
"learning_rate": 0.00021251023043064224,
|
|
"loss_nats": 1.2090221643447876,
|
|
"step": 3330,
|
|
"unclipped_grad_norm": 0.9751812815666199
|
|
},
|
|
{
|
|
"bits_per_byte": 1.740043757167613,
|
|
"learning_rate": 0.00021198747970799188,
|
|
"loss_nats": 1.206106424331665,
|
|
"step": 3340,
|
|
"unclipped_grad_norm": 1.0885974168777466
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7459623681144056,
|
|
"learning_rate": 0.00021146392609895618,
|
|
"loss_nats": 1.2102088928222656,
|
|
"step": 3350,
|
|
"unclipped_grad_norm": 1.028318166732788
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7533509147781228,
|
|
"learning_rate": 0.00021093957854962178,
|
|
"loss_nats": 1.2153302431106567,
|
|
"step": 3360,
|
|
"unclipped_grad_norm": 1.038756251335144
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6755667734285427,
|
|
"learning_rate": 0.00021041444601964157,
|
|
"loss_nats": 1.161414384841919,
|
|
"step": 3370,
|
|
"unclipped_grad_norm": 1.0451469421386719
|
|
},
|
|
{
|
|
"bits_per_byte": 1.794013632834871,
|
|
"learning_rate": 0.00020988853748208164,
|
|
"loss_nats": 1.2435154914855957,
|
|
"step": 3380,
|
|
"unclipped_grad_norm": 1.1703273057937622
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7554886591283476,
|
|
"learning_rate": 0.00020936186192326782,
|
|
"loss_nats": 1.216812014579773,
|
|
"step": 3390,
|
|
"unclipped_grad_norm": 1.036725401878357
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7445490146896152,
|
|
"learning_rate": 0.00020883442834263225,
|
|
"loss_nats": 1.2092292308807373,
|
|
"step": 3400,
|
|
"unclipped_grad_norm": 1.0196164846420288
|
|
},
|
|
{
|
|
"bits_per_byte": 1.794997717563108,
|
|
"learning_rate": 0.0002083062457525596,
|
|
"loss_nats": 1.2441976070404053,
|
|
"step": 3410,
|
|
"unclipped_grad_norm": 1.202675700187683
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7549782146205868,
|
|
"learning_rate": 0.00020777732317823306,
|
|
"loss_nats": 1.2164582014083862,
|
|
"step": 3420,
|
|
"unclipped_grad_norm": 0.9739375710487366
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7093242160704982,
|
|
"learning_rate": 0.00020724766965748005,
|
|
"loss_nats": 1.1848132610321045,
|
|
"step": 3430,
|
|
"unclipped_grad_norm": 1.0012179613113403
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7178566193451037,
|
|
"learning_rate": 0.00020671729424061788,
|
|
"loss_nats": 1.1907274723052979,
|
|
"step": 3440,
|
|
"unclipped_grad_norm": 1.0250310897827148
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7050728049411694,
|
|
"learning_rate": 0.00020618620599029906,
|
|
"loss_nats": 1.1818664073944092,
|
|
"step": 3450,
|
|
"unclipped_grad_norm": 1.0344855785369873
|
|
},
|
|
{
|
|
"bits_per_byte": 1.706558735044624,
|
|
"learning_rate": 0.00020565441398135649,
|
|
"loss_nats": 1.182896375656128,
|
|
"step": 3460,
|
|
"unclipped_grad_norm": 0.9980728626251221
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7653320861504804,
|
|
"learning_rate": 0.00020512192730064827,
|
|
"loss_nats": 1.223634958267212,
|
|
"step": 3470,
|
|
"unclipped_grad_norm": 1.0550833940505981
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8114688400015408,
|
|
"learning_rate": 0.00020458875504690273,
|
|
"loss_nats": 1.2556145191192627,
|
|
"step": 3480,
|
|
"unclipped_grad_norm": 1.0545322895050049
|
|
},
|
|
{
|
|
"bits_per_byte": 1.740959048835505,
|
|
"learning_rate": 0.00020405490633056245,
|
|
"loss_nats": 1.2067408561706543,
|
|
"step": 3490,
|
|
"unclipped_grad_norm": 0.9756637811660767
|
|
},
|
|
{
|
|
"bits_per_byte": 1.704705106033625,
|
|
"learning_rate": 0.00020352039027362917,
|
|
"loss_nats": 1.1816115379333496,
|
|
"step": 3500,
|
|
"unclipped_grad_norm": 0.9728481769561768
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7154590092094253,
|
|
"learning_rate": 0.0002029852160095075,
|
|
"loss_nats": 1.1890655755996704,
|
|
"step": 3510,
|
|
"unclipped_grad_norm": 0.9984562993049622
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7629102264610303,
|
|
"learning_rate": 0.000202449392682849,
|
|
"loss_nats": 1.2219562530517578,
|
|
"step": 3520,
|
|
"unclipped_grad_norm": 1.0547406673431396
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7070648799861134,
|
|
"learning_rate": 0.0002019129294493959,
|
|
"loss_nats": 1.1832472085952759,
|
|
"step": 3530,
|
|
"unclipped_grad_norm": 0.9733150601387024
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7245068444886669,
|
|
"learning_rate": 0.00020137583547582478,
|
|
"loss_nats": 1.1953370571136475,
|
|
"step": 3540,
|
|
"unclipped_grad_norm": 1.0125547647476196
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7698511022845516,
|
|
"learning_rate": 0.00020083811993958973,
|
|
"loss_nats": 1.2267673015594482,
|
|
"step": 3550,
|
|
"unclipped_grad_norm": 1.0299363136291504
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7181577609667646,
|
|
"learning_rate": 0.00020029979202876563,
|
|
"loss_nats": 1.1909362077713013,
|
|
"step": 3560,
|
|
"unclipped_grad_norm": 0.9789594411849976
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7459088815099875,
|
|
"learning_rate": 0.00019976086094189118,
|
|
"loss_nats": 1.2101718187332153,
|
|
"step": 3570,
|
|
"unclipped_grad_norm": 1.0010499954223633
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7705113436812139,
|
|
"learning_rate": 0.00019922133588781175,
|
|
"loss_nats": 1.2272249460220337,
|
|
"step": 3580,
|
|
"unclipped_grad_norm": 1.082968831062317
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7425258107848651,
|
|
"learning_rate": 0.0001986812260855219,
|
|
"loss_nats": 1.207826852798462,
|
|
"step": 3590,
|
|
"unclipped_grad_norm": 1.0055863857269287
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7755588625013865,
|
|
"learning_rate": 0.00019814054076400787,
|
|
"loss_nats": 1.2307236194610596,
|
|
"step": 3600,
|
|
"unclipped_grad_norm": 1.0473418235778809
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7693408297594415,
|
|
"learning_rate": 0.00019759928916209004,
|
|
"loss_nats": 1.226413607597351,
|
|
"step": 3610,
|
|
"unclipped_grad_norm": 1.1090856790542603
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7112244523798812,
|
|
"learning_rate": 0.00019705748052826493,
|
|
"loss_nats": 1.186130404472351,
|
|
"step": 3620,
|
|
"unclipped_grad_norm": 0.9742531776428223
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7137644641504741,
|
|
"learning_rate": 0.00019651512412054723,
|
|
"loss_nats": 1.1878910064697266,
|
|
"step": 3630,
|
|
"unclipped_grad_norm": 1.0429282188415527
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6520834023664803,
|
|
"learning_rate": 0.00019597222920631144,
|
|
"loss_nats": 1.1451369524002075,
|
|
"step": 3640,
|
|
"unclipped_grad_norm": 0.9745917320251465
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7256807980634563,
|
|
"learning_rate": 0.0001954288050621338,
|
|
"loss_nats": 1.196150779724121,
|
|
"step": 3650,
|
|
"unclipped_grad_norm": 1.0385748147964478
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7126971398192194,
|
|
"learning_rate": 0.00019488486097363363,
|
|
"loss_nats": 1.1871511936187744,
|
|
"step": 3660,
|
|
"unclipped_grad_norm": 0.9310101270675659
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7674923602279682,
|
|
"learning_rate": 0.00019434040623531452,
|
|
"loss_nats": 1.2251323461532593,
|
|
"step": 3670,
|
|
"unclipped_grad_norm": 1.0780647993087769
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6993364956042913,
|
|
"learning_rate": 0.0001937954501504058,
|
|
"loss_nats": 1.1778903007507324,
|
|
"step": 3680,
|
|
"unclipped_grad_norm": 0.9666725397109985
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7697671747509305,
|
|
"learning_rate": 0.00019325000203070333,
|
|
"loss_nats": 1.2267091274261475,
|
|
"step": 3690,
|
|
"unclipped_grad_norm": 1.0202223062515259
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6641650116069417,
|
|
"learning_rate": 0.0001927040711964106,
|
|
"loss_nats": 1.1535112857818604,
|
|
"step": 3700,
|
|
"unclipped_grad_norm": 1.0115612745285034
|
|
},
|
|
{
|
|
"bits_per_byte": 1.726182815421325,
|
|
"learning_rate": 0.00019215766697597923,
|
|
"loss_nats": 1.1964987516403198,
|
|
"step": 3710,
|
|
"unclipped_grad_norm": 0.9808560013771057
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7616059100368868,
|
|
"learning_rate": 0.00019161079870594978,
|
|
"loss_nats": 1.2210521697998047,
|
|
"step": 3720,
|
|
"unclipped_grad_norm": 1.0511683225631714
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7329405297216773,
|
|
"learning_rate": 0.0001910634757307922,
|
|
"loss_nats": 1.2011828422546387,
|
|
"step": 3730,
|
|
"unclipped_grad_norm": 1.0690499544143677
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7000292417219667,
|
|
"learning_rate": 0.0001905157074027459,
|
|
"loss_nats": 1.178370475769043,
|
|
"step": 3740,
|
|
"unclipped_grad_norm": 0.9935287237167358
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7278594742845865,
|
|
"learning_rate": 0.00018996750308166037,
|
|
"loss_nats": 1.1976609230041504,
|
|
"step": 3750,
|
|
"unclipped_grad_norm": 1.010900855064392
|
|
},
|
|
{
|
|
"bits_per_byte": 1.743004610484867,
|
|
"learning_rate": 0.00018941887213483482,
|
|
"loss_nats": 1.2081587314605713,
|
|
"step": 3760,
|
|
"unclipped_grad_norm": 1.0119491815567017
|
|
},
|
|
{
|
|
"bits_per_byte": 1.751619221466585,
|
|
"learning_rate": 0.00018886982393685845,
|
|
"loss_nats": 1.21412992477417,
|
|
"step": 3770,
|
|
"unclipped_grad_norm": 1.011305570602417
|
|
},
|
|
{
|
|
"bits_per_byte": 1.8012363882231563,
|
|
"learning_rate": 0.00018832036786945008,
|
|
"loss_nats": 1.2485219240188599,
|
|
"step": 3780,
|
|
"unclipped_grad_norm": 1.0745822191238403
|
|
},
|
|
{
|
|
"bits_per_byte": 1.698407961272283,
|
|
"learning_rate": 0.0001877705133212979,
|
|
"loss_nats": 1.1772466897964478,
|
|
"step": 3790,
|
|
"unclipped_grad_norm": 1.0157029628753662
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7299170747195092,
|
|
"learning_rate": 0.00018722026968789906,
|
|
"loss_nats": 1.199087142944336,
|
|
"step": 3800,
|
|
"unclipped_grad_norm": 1.0643014907836914
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6600432753963636,
|
|
"learning_rate": 0.00018666964637139907,
|
|
"loss_nats": 1.1506543159484863,
|
|
"step": 3810,
|
|
"unclipped_grad_norm": 0.9955329895019531
|
|
},
|
|
{
|
|
"bits_per_byte": 1.661928377232471,
|
|
"learning_rate": 0.00018611865278043115,
|
|
"loss_nats": 1.1519609689712524,
|
|
"step": 3820,
|
|
"unclipped_grad_norm": 1.0222501754760742
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7482129330835965,
|
|
"learning_rate": 0.00018556729832995555,
|
|
"loss_nats": 1.2117688655853271,
|
|
"step": 3830,
|
|
"unclipped_grad_norm": 1.0880961418151855
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6885180989344173,
|
|
"learning_rate": 0.00018501559244109864,
|
|
"loss_nats": 1.17039155960083,
|
|
"step": 3840,
|
|
"unclipped_grad_norm": 0.9980700016021729
|
|
},
|
|
{
|
|
"bits_per_byte": 1.661674014891845,
|
|
"learning_rate": 0.00018446354454099178,
|
|
"loss_nats": 1.1517846584320068,
|
|
"step": 3850,
|
|
"unclipped_grad_norm": 0.961553156375885
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6409110654011305,
|
|
"learning_rate": 0.00018391116406261048,
|
|
"loss_nats": 1.1373928785324097,
|
|
"step": 3860,
|
|
"unclipped_grad_norm": 1.019968032836914
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6069470715954888,
|
|
"learning_rate": 0.00018335846044461305,
|
|
"loss_nats": 1.1138508319854736,
|
|
"step": 3870,
|
|
"unclipped_grad_norm": 0.9610836505889893
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7034871249002144,
|
|
"learning_rate": 0.0001828054431311793,
|
|
"loss_nats": 1.180767297744751,
|
|
"step": 3880,
|
|
"unclipped_grad_norm": 0.9531551003456116
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6934515932570677,
|
|
"learning_rate": 0.00018225212157184934,
|
|
"loss_nats": 1.1738111972808838,
|
|
"step": 3890,
|
|
"unclipped_grad_norm": 1.0675544738769531
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6830007235120874,
|
|
"learning_rate": 0.00018169850522136192,
|
|
"loss_nats": 1.1665672063827515,
|
|
"step": 3900,
|
|
"unclipped_grad_norm": 1.0058032274246216
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7034874688655162,
|
|
"learning_rate": 0.000181144603539493,
|
|
"loss_nats": 1.18076753616333,
|
|
"step": 3910,
|
|
"unclipped_grad_norm": 0.9943428039550781
|
|
},
|
|
{
|
|
"bits_per_byte": 1.743468619676895,
|
|
"learning_rate": 0.00018059042599089412,
|
|
"loss_nats": 1.2084803581237793,
|
|
"step": 3920,
|
|
"unclipped_grad_norm": 1.2279266119003296
|
|
},
|
|
{
|
|
"bits_per_byte": 1.721944302990811,
|
|
"learning_rate": 0.00018003598204493048,
|
|
"loss_nats": 1.1935608386993408,
|
|
"step": 3930,
|
|
"unclipped_grad_norm": 1.008062720298767
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6512047430032224,
|
|
"learning_rate": 0.00017948128117551936,
|
|
"loss_nats": 1.1445279121398926,
|
|
"step": 3940,
|
|
"unclipped_grad_norm": 0.9797955751419067
|
|
},
|
|
{
|
|
"bits_per_byte": 1.750776850442659,
|
|
"learning_rate": 0.00017892633286096816,
|
|
"loss_nats": 1.2135460376739502,
|
|
"step": 3950,
|
|
"unclipped_grad_norm": 1.0508832931518555
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7346926889686676,
|
|
"learning_rate": 0.00017837114658381247,
|
|
"loss_nats": 1.202397346496582,
|
|
"step": 3960,
|
|
"unclipped_grad_norm": 1.0876473188400269
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6890646597988592,
|
|
"learning_rate": 0.00017781573183065386,
|
|
"loss_nats": 1.1707704067230225,
|
|
"step": 3970,
|
|
"unclipped_grad_norm": 1.0190049409866333
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6854375456921626,
|
|
"learning_rate": 0.0001772600980919981,
|
|
"loss_nats": 1.1682562828063965,
|
|
"step": 3980,
|
|
"unclipped_grad_norm": 1.0094109773635864
|
|
},
|
|
{
|
|
"bits_per_byte": 1.710786068602832,
|
|
"learning_rate": 0.0001767042548620927,
|
|
"loss_nats": 1.1858265399932861,
|
|
"step": 3990,
|
|
"unclipped_grad_norm": 1.082020878791809
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6440261871562087,
|
|
"learning_rate": 0.00017614821163876486,
|
|
"loss_nats": 1.139552116394043,
|
|
"step": 4000,
|
|
"unclipped_grad_norm": 0.9749544262886047
|
|
},
|
|
{
|
|
"bits_per_byte": 1.706837174956371,
|
|
"learning_rate": 0.00017559197792325906,
|
|
"loss_nats": 1.1830893754959106,
|
|
"step": 4010,
|
|
"unclipped_grad_norm": 1.1478910446166992
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7296654641012967,
|
|
"learning_rate": 0.0001750355632200748,
|
|
"loss_nats": 1.1989127397537231,
|
|
"step": 4020,
|
|
"unclipped_grad_norm": 1.0259206295013428
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6863480218458304,
|
|
"learning_rate": 0.00017447897703680418,
|
|
"loss_nats": 1.1688873767852783,
|
|
"step": 4030,
|
|
"unclipped_grad_norm": 1.0554102659225464
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6602591136231966,
|
|
"learning_rate": 0.0001739222288839694,
|
|
"loss_nats": 1.1508039236068726,
|
|
"step": 4040,
|
|
"unclipped_grad_norm": 1.00386643409729
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6813582891963477,
|
|
"learning_rate": 0.00017336532827486026,
|
|
"loss_nats": 1.1654287576675415,
|
|
"step": 4050,
|
|
"unclipped_grad_norm": 0.9927709698677063
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6720621109692604,
|
|
"learning_rate": 0.00017280828472537163,
|
|
"loss_nats": 1.1589851379394531,
|
|
"step": 4060,
|
|
"unclipped_grad_norm": 1.0185434818267822
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7963911210003984,
|
|
"learning_rate": 0.0001722511077538408,
|
|
"loss_nats": 1.2451634407043457,
|
|
"step": 4070,
|
|
"unclipped_grad_norm": 1.0195153951644897
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6278679011596953,
|
|
"learning_rate": 0.00017169380688088497,
|
|
"loss_nats": 1.1283520460128784,
|
|
"step": 4080,
|
|
"unclipped_grad_norm": 0.9588671326637268
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6697190193339053,
|
|
"learning_rate": 0.0001711363916292384,
|
|
"loss_nats": 1.1573610305786133,
|
|
"step": 4090,
|
|
"unclipped_grad_norm": 1.0382872819900513
|
|
},
|
|
{
|
|
"bits_per_byte": 1.786499366853373,
|
|
"learning_rate": 0.00017057887152358974,
|
|
"loss_nats": 1.238306999206543,
|
|
"step": 4100,
|
|
"unclipped_grad_norm": 1.0833542346954346
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6685632959201073,
|
|
"learning_rate": 0.0001700212560904194,
|
|
"loss_nats": 1.156559944152832,
|
|
"step": 4110,
|
|
"unclipped_grad_norm": 1.0042530298233032
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6813281922324468,
|
|
"learning_rate": 0.00016946355485783656,
|
|
"loss_nats": 1.1654078960418701,
|
|
"step": 4120,
|
|
"unclipped_grad_norm": 1.0854146480560303
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7532864212840493,
|
|
"learning_rate": 0.0001689057773554166,
|
|
"loss_nats": 1.2152855396270752,
|
|
"step": 4130,
|
|
"unclipped_grad_norm": 1.0470608472824097
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7313932018118645,
|
|
"learning_rate": 0.00016834793311403806,
|
|
"loss_nats": 1.2001103162765503,
|
|
"step": 4140,
|
|
"unclipped_grad_norm": 1.0418859720230103
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6694201134867057,
|
|
"learning_rate": 0.00016779003166571981,
|
|
"loss_nats": 1.157153844833374,
|
|
"step": 4150,
|
|
"unclipped_grad_norm": 1.0297493934631348
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6896791538103921,
|
|
"learning_rate": 0.00016723208254345838,
|
|
"loss_nats": 1.1711963415145874,
|
|
"step": 4160,
|
|
"unclipped_grad_norm": 1.0267698764801025
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7146273011098527,
|
|
"learning_rate": 0.00016667409528106481,
|
|
"loss_nats": 1.1884890794754028,
|
|
"step": 4170,
|
|
"unclipped_grad_norm": 1.1191542148590088
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6875347021367837,
|
|
"learning_rate": 0.00016611607941300184,
|
|
"loss_nats": 1.1697099208831787,
|
|
"step": 4180,
|
|
"unclipped_grad_norm": 1.0307927131652832
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7039927538937512,
|
|
"learning_rate": 0.0001655580444742211,
|
|
"loss_nats": 1.1811177730560303,
|
|
"step": 4190,
|
|
"unclipped_grad_norm": 1.089546799659729
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7112641803722306,
|
|
"learning_rate": 0.00016499999999999997,
|
|
"loss_nats": 1.1861579418182373,
|
|
"step": 4200,
|
|
"unclipped_grad_norm": 1.0752745866775513
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7401058429045744,
|
|
"learning_rate": 0.0001644419555257789,
|
|
"loss_nats": 1.2061494588851929,
|
|
"step": 4210,
|
|
"unclipped_grad_norm": 1.0523611307144165
|
|
},
|
|
{
|
|
"bits_per_byte": 1.693671731050172,
|
|
"learning_rate": 0.0001638839205869981,
|
|
"loss_nats": 1.1739637851715088,
|
|
"step": 4220,
|
|
"unclipped_grad_norm": 0.997297465801239
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7078159282224312,
|
|
"learning_rate": 0.0001633259047189352,
|
|
"loss_nats": 1.1837677955627441,
|
|
"step": 4230,
|
|
"unclipped_grad_norm": 1.0999088287353516
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7184320732948908,
|
|
"learning_rate": 0.0001627679174565416,
|
|
"loss_nats": 1.1911263465881348,
|
|
"step": 4240,
|
|
"unclipped_grad_norm": 0.9742965698242188
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6537339198668108,
|
|
"learning_rate": 0.00016220996833428016,
|
|
"loss_nats": 1.1462810039520264,
|
|
"step": 4250,
|
|
"unclipped_grad_norm": 0.9919567108154297
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6890791063415318,
|
|
"learning_rate": 0.00016165206688596191,
|
|
"loss_nats": 1.1707804203033447,
|
|
"step": 4260,
|
|
"unclipped_grad_norm": 1.0240992307662964
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6052090149258693,
|
|
"learning_rate": 0.00016109422264458335,
|
|
"loss_nats": 1.1126461029052734,
|
|
"step": 4270,
|
|
"unclipped_grad_norm": 0.9614320993423462
|
|
},
|
|
{
|
|
"bits_per_byte": 1.677325124050964,
|
|
"learning_rate": 0.00016053644514216336,
|
|
"loss_nats": 1.1626331806182861,
|
|
"step": 4280,
|
|
"unclipped_grad_norm": 1.0358916521072388
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6570518091672561,
|
|
"learning_rate": 0.00015997874390958058,
|
|
"loss_nats": 1.14858078956604,
|
|
"step": 4290,
|
|
"unclipped_grad_norm": 1.0068124532699585
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6268774530733763,
|
|
"learning_rate": 0.00015942112847641018,
|
|
"loss_nats": 1.1276655197143555,
|
|
"step": 4300,
|
|
"unclipped_grad_norm": 1.0985108613967896
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6144661530912108,
|
|
"learning_rate": 0.00015886360837076156,
|
|
"loss_nats": 1.1190626621246338,
|
|
"step": 4310,
|
|
"unclipped_grad_norm": 0.9936105608940125
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7397008237617926,
|
|
"learning_rate": 0.00015830619311911495,
|
|
"loss_nats": 1.2058687210083008,
|
|
"step": 4320,
|
|
"unclipped_grad_norm": 1.1129765510559082
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6818943591190871,
|
|
"learning_rate": 0.00015774889224615915,
|
|
"loss_nats": 1.1658003330230713,
|
|
"step": 4330,
|
|
"unclipped_grad_norm": 1.0247257947921753
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6456326770979182,
|
|
"learning_rate": 0.00015719171527462834,
|
|
"loss_nats": 1.1406656503677368,
|
|
"step": 4340,
|
|
"unclipped_grad_norm": 1.0460948944091797
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6091236840248087,
|
|
"learning_rate": 0.0001566346717251397,
|
|
"loss_nats": 1.1153595447540283,
|
|
"step": 4350,
|
|
"unclipped_grad_norm": 1.0163395404815674
|
|
},
|
|
{
|
|
"bits_per_byte": 1.686970427059303,
|
|
"learning_rate": 0.00015607777111603056,
|
|
"loss_nats": 1.1693187952041626,
|
|
"step": 4360,
|
|
"unclipped_grad_norm": 1.063033938407898
|
|
},
|
|
{
|
|
"bits_per_byte": 1.65933092325649,
|
|
"learning_rate": 0.00015552102296319576,
|
|
"loss_nats": 1.150160551071167,
|
|
"step": 4370,
|
|
"unclipped_grad_norm": 0.9919126033782959
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6024796502566765,
|
|
"learning_rate": 0.00015496443677992516,
|
|
"loss_nats": 1.1107542514801025,
|
|
"step": 4380,
|
|
"unclipped_grad_norm": 1.0177253484725952
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7322169987094975,
|
|
"learning_rate": 0.0001544080220767409,
|
|
"loss_nats": 1.2006813287734985,
|
|
"step": 4390,
|
|
"unclipped_grad_norm": 0.9770697951316833
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6952770171133256,
|
|
"learning_rate": 0.00015385178836123511,
|
|
"loss_nats": 1.1750764846801758,
|
|
"step": 4400,
|
|
"unclipped_grad_norm": 1.0351864099502563
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6960017520040616,
|
|
"learning_rate": 0.00015329574513790726,
|
|
"loss_nats": 1.1755788326263428,
|
|
"step": 4410,
|
|
"unclipped_grad_norm": 1.1030160188674927
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7127384156554266,
|
|
"learning_rate": 0.00015273990190800186,
|
|
"loss_nats": 1.1871798038482666,
|
|
"step": 4420,
|
|
"unclipped_grad_norm": 1.0353890657424927
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6726172709662455,
|
|
"learning_rate": 0.0001521842681693461,
|
|
"loss_nats": 1.159369945526123,
|
|
"step": 4430,
|
|
"unclipped_grad_norm": 1.0875210762023926
|
|
},
|
|
{
|
|
"bits_per_byte": 1.680560461678995,
|
|
"learning_rate": 0.00015162885341618752,
|
|
"loss_nats": 1.1648757457733154,
|
|
"step": 4440,
|
|
"unclipped_grad_norm": 0.9928231239318848
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6836972532480818,
|
|
"learning_rate": 0.00015107366713903178,
|
|
"loss_nats": 1.1670500040054321,
|
|
"step": 4450,
|
|
"unclipped_grad_norm": 0.9873722791671753
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6829482688035744,
|
|
"learning_rate": 0.0001505187188244806,
|
|
"loss_nats": 1.1665308475494385,
|
|
"step": 4460,
|
|
"unclipped_grad_norm": 1.0991053581237793
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6608082542274016,
|
|
"learning_rate": 0.00014996401795506952,
|
|
"loss_nats": 1.1511845588684082,
|
|
"step": 4470,
|
|
"unclipped_grad_norm": 1.0593136548995972
|
|
},
|
|
{
|
|
"bits_per_byte": 1.698860447626703,
|
|
"learning_rate": 0.00014940957400910585,
|
|
"loss_nats": 1.1775603294372559,
|
|
"step": 4480,
|
|
"unclipped_grad_norm": 1.0394423007965088
|
|
},
|
|
{
|
|
"bits_per_byte": 1.740063707155113,
|
|
"learning_rate": 0.00014885539646050699,
|
|
"loss_nats": 1.206120252609253,
|
|
"step": 4490,
|
|
"unclipped_grad_norm": 1.0491414070129395
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6564844384020596,
|
|
"learning_rate": 0.00014830149477863805,
|
|
"loss_nats": 1.148187518119812,
|
|
"step": 4500,
|
|
"unclipped_grad_norm": 1.0240522623062134
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6424668204608355,
|
|
"learning_rate": 0.00014774787842815066,
|
|
"loss_nats": 1.138471245765686,
|
|
"step": 4510,
|
|
"unclipped_grad_norm": 1.025274634361267
|
|
},
|
|
{
|
|
"bits_per_byte": 1.71951625192593,
|
|
"learning_rate": 0.00014719455686882066,
|
|
"loss_nats": 1.191877841949463,
|
|
"step": 4520,
|
|
"unclipped_grad_norm": 1.063368320465088
|
|
},
|
|
{
|
|
"bits_per_byte": 1.680697359869082,
|
|
"learning_rate": 0.00014664153955538695,
|
|
"loss_nats": 1.1649706363677979,
|
|
"step": 4530,
|
|
"unclipped_grad_norm": 1.0728412866592407
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6916681331676204,
|
|
"learning_rate": 0.00014608883593738946,
|
|
"loss_nats": 1.1725749969482422,
|
|
"step": 4540,
|
|
"unclipped_grad_norm": 1.1416125297546387
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6884355472620032,
|
|
"learning_rate": 0.00014553645545900817,
|
|
"loss_nats": 1.1703343391418457,
|
|
"step": 4550,
|
|
"unclipped_grad_norm": 1.0188941955566406
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6889325771229966,
|
|
"learning_rate": 0.0001449844075589013,
|
|
"loss_nats": 1.1706788539886475,
|
|
"step": 4560,
|
|
"unclipped_grad_norm": 1.0978926420211792
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6867642198609183,
|
|
"learning_rate": 0.0001444327016700444,
|
|
"loss_nats": 1.1691758632659912,
|
|
"step": 4570,
|
|
"unclipped_grad_norm": 1.043805480003357
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6647294586670736,
|
|
"learning_rate": 0.0001438813472195688,
|
|
"loss_nats": 1.153902530670166,
|
|
"step": 4580,
|
|
"unclipped_grad_norm": 1.061089277267456
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6460370083100966,
|
|
"learning_rate": 0.00014333035362860093,
|
|
"loss_nats": 1.1409459114074707,
|
|
"step": 4590,
|
|
"unclipped_grad_norm": 1.0722659826278687
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6445638049228057,
|
|
"learning_rate": 0.00014277973031210088,
|
|
"loss_nats": 1.1399247646331787,
|
|
"step": 4600,
|
|
"unclipped_grad_norm": 1.0390986204147339
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6450002968906956,
|
|
"learning_rate": 0.00014222948667870207,
|
|
"loss_nats": 1.1402273178100586,
|
|
"step": 4610,
|
|
"unclipped_grad_norm": 1.0198664665222168
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7089066421942032,
|
|
"learning_rate": 0.00014167963213054991,
|
|
"loss_nats": 1.1845238208770752,
|
|
"step": 4620,
|
|
"unclipped_grad_norm": 1.0599243640899658
|
|
},
|
|
{
|
|
"bits_per_byte": 1.69811834248823,
|
|
"learning_rate": 0.00014113017606314152,
|
|
"loss_nats": 1.1770459413528442,
|
|
"step": 4630,
|
|
"unclipped_grad_norm": 1.1346435546875
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7044230544862098,
|
|
"learning_rate": 0.00014058112786516518,
|
|
"loss_nats": 1.1814160346984863,
|
|
"step": 4640,
|
|
"unclipped_grad_norm": 1.0656381845474243
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7338580571640303,
|
|
"learning_rate": 0.0001400324969183396,
|
|
"loss_nats": 1.201818823814392,
|
|
"step": 4650,
|
|
"unclipped_grad_norm": 1.0118787288665771
|
|
},
|
|
{
|
|
"bits_per_byte": 1.652913906587497,
|
|
"learning_rate": 0.00013948429259725406,
|
|
"loss_nats": 1.1457126140594482,
|
|
"step": 4660,
|
|
"unclipped_grad_norm": 1.0275835990905762
|
|
},
|
|
{
|
|
"bits_per_byte": 1.66807348933045,
|
|
"learning_rate": 0.00013893652426920776,
|
|
"loss_nats": 1.1562204360961914,
|
|
"step": 4670,
|
|
"unclipped_grad_norm": 1.0584690570831299
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6481818039490068,
|
|
"learning_rate": 0.00013838920129405016,
|
|
"loss_nats": 1.1424325704574585,
|
|
"step": 4680,
|
|
"unclipped_grad_norm": 1.1420236825942993
|
|
},
|
|
{
|
|
"bits_per_byte": 1.658637833173513,
|
|
"learning_rate": 0.0001378423330240207,
|
|
"loss_nats": 1.1496801376342773,
|
|
"step": 4690,
|
|
"unclipped_grad_norm": 1.080548644065857
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6754130209386713,
|
|
"learning_rate": 0.00013729592880358936,
|
|
"loss_nats": 1.1613078117370605,
|
|
"step": 4700,
|
|
"unclipped_grad_norm": 1.063712239265442
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6541807307937524,
|
|
"learning_rate": 0.00013674999796929659,
|
|
"loss_nats": 1.1465907096862793,
|
|
"step": 4710,
|
|
"unclipped_grad_norm": 1.065613865852356
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6715305125954434,
|
|
"learning_rate": 0.00013620454984959417,
|
|
"loss_nats": 1.1586166620254517,
|
|
"step": 4720,
|
|
"unclipped_grad_norm": 1.041472315788269
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7163307892666488,
|
|
"learning_rate": 0.00013565959376468542,
|
|
"loss_nats": 1.1896698474884033,
|
|
"step": 4730,
|
|
"unclipped_grad_norm": 1.0371977090835571
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6468645888260485,
|
|
"learning_rate": 0.00013511513902636634,
|
|
"loss_nats": 1.141519546508789,
|
|
"step": 4740,
|
|
"unclipped_grad_norm": 1.118260383605957
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6331114802018514,
|
|
"learning_rate": 0.0001345711949378662,
|
|
"loss_nats": 1.1319866180419922,
|
|
"step": 4750,
|
|
"unclipped_grad_norm": 1.1252490282058716
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6043413624522664,
|
|
"learning_rate": 0.00013402777079368853,
|
|
"loss_nats": 1.1120446920394897,
|
|
"step": 4760,
|
|
"unclipped_grad_norm": 1.0504546165466309
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6005166402797284,
|
|
"learning_rate": 0.0001334848758794528,
|
|
"loss_nats": 1.10939359664917,
|
|
"step": 4770,
|
|
"unclipped_grad_norm": 0.984404981136322
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6404936635074865,
|
|
"learning_rate": 0.00013294251947173504,
|
|
"loss_nats": 1.13710355758667,
|
|
"step": 4780,
|
|
"unclipped_grad_norm": 1.0672088861465454
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6438352864137509,
|
|
"learning_rate": 0.00013240071083790996,
|
|
"loss_nats": 1.1394197940826416,
|
|
"step": 4790,
|
|
"unclipped_grad_norm": 1.0523202419281006
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6719128300283113,
|
|
"learning_rate": 0.0001318594592359921,
|
|
"loss_nats": 1.158881664276123,
|
|
"step": 4800,
|
|
"unclipped_grad_norm": 1.1081420183181763
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6392536685947656,
|
|
"learning_rate": 0.0001313187739144781,
|
|
"loss_nats": 1.1362440586090088,
|
|
"step": 4810,
|
|
"unclipped_grad_norm": 1.0507419109344482
|
|
},
|
|
{
|
|
"bits_per_byte": 1.648858383697501,
|
|
"learning_rate": 0.0001307786641121882,
|
|
"loss_nats": 1.1429015398025513,
|
|
"step": 4820,
|
|
"unclipped_grad_norm": 1.1054445505142212
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6256828615804833,
|
|
"learning_rate": 0.00013023913905810876,
|
|
"loss_nats": 1.1268374919891357,
|
|
"step": 4830,
|
|
"unclipped_grad_norm": 1.0212544202804565
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5951660880287355,
|
|
"learning_rate": 0.00012970020797123432,
|
|
"loss_nats": 1.1056848764419556,
|
|
"step": 4840,
|
|
"unclipped_grad_norm": 1.0648442506790161
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6564994008926848,
|
|
"learning_rate": 0.00012916188006041027,
|
|
"loss_nats": 1.148197889328003,
|
|
"step": 4850,
|
|
"unclipped_grad_norm": 1.047589898109436
|
|
},
|
|
{
|
|
"bits_per_byte": 1.641553764567405,
|
|
"learning_rate": 0.00012862416452417516,
|
|
"loss_nats": 1.137838363647461,
|
|
"step": 4860,
|
|
"unclipped_grad_norm": 1.0177857875823975
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6551041056462348,
|
|
"learning_rate": 0.00012808707055060405,
|
|
"loss_nats": 1.1472307443618774,
|
|
"step": 4870,
|
|
"unclipped_grad_norm": 1.0283644199371338
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6180980827121316,
|
|
"learning_rate": 0.000127550607317151,
|
|
"loss_nats": 1.1215801239013672,
|
|
"step": 4880,
|
|
"unclipped_grad_norm": 1.0304375886917114
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5903827345602883,
|
|
"learning_rate": 0.00012701478399049248,
|
|
"loss_nats": 1.1023693084716797,
|
|
"step": 4890,
|
|
"unclipped_grad_norm": 1.0921995639801025
|
|
},
|
|
{
|
|
"bits_per_byte": 1.567706994026678,
|
|
"learning_rate": 0.0001264796097263708,
|
|
"loss_nats": 1.0866516828536987,
|
|
"step": 4900,
|
|
"unclipped_grad_norm": 1.0604755878448486
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6439632415059928,
|
|
"learning_rate": 0.0001259450936694375,
|
|
"loss_nats": 1.1395084857940674,
|
|
"step": 4910,
|
|
"unclipped_grad_norm": 1.0503017902374268
|
|
},
|
|
{
|
|
"bits_per_byte": 1.642120275419347,
|
|
"learning_rate": 0.00012541124495309727,
|
|
"loss_nats": 1.1382310390472412,
|
|
"step": 4920,
|
|
"unclipped_grad_norm": 1.1088591814041138
|
|
},
|
|
{
|
|
"bits_per_byte": 1.622944209848144,
|
|
"learning_rate": 0.00012487807269935167,
|
|
"loss_nats": 1.124939203262329,
|
|
"step": 4930,
|
|
"unclipped_grad_norm": 1.099297046661377
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6933755769253864,
|
|
"learning_rate": 0.00012434558601864349,
|
|
"loss_nats": 1.1737585067749023,
|
|
"step": 4940,
|
|
"unclipped_grad_norm": 1.08061945438385
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5670388414280758,
|
|
"learning_rate": 0.0001238137940097009,
|
|
"loss_nats": 1.086188554763794,
|
|
"step": 4950,
|
|
"unclipped_grad_norm": 1.1187851428985596
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6577792957804058,
|
|
"learning_rate": 0.0001232827057593821,
|
|
"loss_nats": 1.1490850448608398,
|
|
"step": 4960,
|
|
"unclipped_grad_norm": 1.095150351524353
|
|
},
|
|
{
|
|
"bits_per_byte": 1.7221861105979241,
|
|
"learning_rate": 0.00012275233034251992,
|
|
"loss_nats": 1.1937284469604492,
|
|
"step": 4970,
|
|
"unclipped_grad_norm": 1.2121388912200928
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5930121773093298,
|
|
"learning_rate": 0.0001222226768217669,
|
|
"loss_nats": 1.1041918992996216,
|
|
"step": 4980,
|
|
"unclipped_grad_norm": 1.1045044660568237
|
|
},
|
|
{
|
|
"bits_per_byte": 1.665033523993799,
|
|
"learning_rate": 0.00012169375424744034,
|
|
"loss_nats": 1.1541132926940918,
|
|
"step": 4990,
|
|
"unclipped_grad_norm": 1.1089035272598267
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6784994215910551,
|
|
"learning_rate": 0.00012116557165736772,
|
|
"loss_nats": 1.1634471416473389,
|
|
"step": 5000,
|
|
"unclipped_grad_norm": 1.0390889644622803
|
|
},
|
|
{
|
|
"bits_per_byte": 1.642436207548982,
|
|
"learning_rate": 0.00012063813807673214,
|
|
"loss_nats": 1.138450026512146,
|
|
"step": 5010,
|
|
"unclipped_grad_norm": 1.1114050149917603
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6556807634745778,
|
|
"learning_rate": 0.00012011146251791832,
|
|
"loss_nats": 1.1476304531097412,
|
|
"step": 5020,
|
|
"unclipped_grad_norm": 1.0220290422439575
|
|
},
|
|
{
|
|
"bits_per_byte": 1.636998288111351,
|
|
"learning_rate": 0.0001195855539803584,
|
|
"loss_nats": 1.1346807479858398,
|
|
"step": 5030,
|
|
"unclipped_grad_norm": 1.1270323991775513
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6052772920382619,
|
|
"learning_rate": 0.00011906042145037817,
|
|
"loss_nats": 1.112693428993225,
|
|
"step": 5040,
|
|
"unclipped_grad_norm": 1.009588360786438
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6583182894082098,
|
|
"learning_rate": 0.0001185360739010438,
|
|
"loss_nats": 1.149458646774292,
|
|
"step": 5050,
|
|
"unclipped_grad_norm": 1.060039758682251
|
|
},
|
|
{
|
|
"bits_per_byte": 1.609498090255737,
|
|
"learning_rate": 0.00011801252029200808,
|
|
"loss_nats": 1.1156190633773804,
|
|
"step": 5060,
|
|
"unclipped_grad_norm": 1.0725975036621094
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6927082842400387,
|
|
"learning_rate": 0.00011748976956935777,
|
|
"loss_nats": 1.1732959747314453,
|
|
"step": 5070,
|
|
"unclipped_grad_norm": 1.120245099067688
|
|
},
|
|
{
|
|
"bits_per_byte": 1.690623166580978,
|
|
"learning_rate": 0.00011696783066546024,
|
|
"loss_nats": 1.1718506813049316,
|
|
"step": 5080,
|
|
"unclipped_grad_norm": 1.1778441667556763
|
|
},
|
|
{
|
|
"bits_per_byte": 1.67411111227164,
|
|
"learning_rate": 0.00011644671249881147,
|
|
"loss_nats": 1.1604053974151611,
|
|
"step": 5090,
|
|
"unclipped_grad_norm": 1.3122999668121338
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5883206625764432,
|
|
"learning_rate": 0.00011592642397388293,
|
|
"loss_nats": 1.1009399890899658,
|
|
"step": 5100,
|
|
"unclipped_grad_norm": 1.0812361240386963
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6343635139001327,
|
|
"learning_rate": 0.00011540697398097015,
|
|
"loss_nats": 1.1328544616699219,
|
|
"step": 5110,
|
|
"unclipped_grad_norm": 1.054042935371399
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6096652573923755,
|
|
"learning_rate": 0.00011488837139604016,
|
|
"loss_nats": 1.1157349348068237,
|
|
"step": 5120,
|
|
"unclipped_grad_norm": 1.183487892150879
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6704048861455463,
|
|
"learning_rate": 0.00011437062508058039,
|
|
"loss_nats": 1.1578364372253418,
|
|
"step": 5130,
|
|
"unclipped_grad_norm": 1.1154927015304565
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6086259662332116,
|
|
"learning_rate": 0.00011385374388144673,
|
|
"loss_nats": 1.1150145530700684,
|
|
"step": 5140,
|
|
"unclipped_grad_norm": 1.1280518770217896
|
|
},
|
|
{
|
|
"bits_per_byte": 1.607841897327928,
|
|
"learning_rate": 0.00011333773663071286,
|
|
"loss_nats": 1.1144710779190063,
|
|
"step": 5150,
|
|
"unclipped_grad_norm": 1.1053173542022705
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6377722100402334,
|
|
"learning_rate": 0.00011282261214551897,
|
|
"loss_nats": 1.1352171897888184,
|
|
"step": 5160,
|
|
"unclipped_grad_norm": 1.0772578716278076
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6387738369988585,
|
|
"learning_rate": 0.00011230837922792112,
|
|
"loss_nats": 1.135911464691162,
|
|
"step": 5170,
|
|
"unclipped_grad_norm": 1.08511221408844
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6731652076918944,
|
|
"learning_rate": 0.0001117950466647412,
|
|
"loss_nats": 1.1597497463226318,
|
|
"step": 5180,
|
|
"unclipped_grad_norm": 1.0842925310134888
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6308702022958073,
|
|
"learning_rate": 0.00011128262322741627,
|
|
"loss_nats": 1.1304330825805664,
|
|
"step": 5190,
|
|
"unclipped_grad_norm": 1.1379977464675903
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5988843529403896,
|
|
"learning_rate": 0.00011077111767184913,
|
|
"loss_nats": 1.1082621812820435,
|
|
"step": 5200,
|
|
"unclipped_grad_norm": 1.1327811479568481
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6538405491103456,
|
|
"learning_rate": 0.00011026053873825837,
|
|
"loss_nats": 1.1463549137115479,
|
|
"step": 5210,
|
|
"unclipped_grad_norm": 1.131410837173462
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6579122383695226,
|
|
"learning_rate": 0.0001097508951510293,
|
|
"loss_nats": 1.1491771936416626,
|
|
"step": 5220,
|
|
"unclipped_grad_norm": 1.111109972000122
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6745703059494434,
|
|
"learning_rate": 0.0001092421956185646,
|
|
"loss_nats": 1.1607236862182617,
|
|
"step": 5230,
|
|
"unclipped_grad_norm": 1.1052273511886597
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6223493218588094,
|
|
"learning_rate": 0.00010873444883313581,
|
|
"loss_nats": 1.124526858329773,
|
|
"step": 5240,
|
|
"unclipped_grad_norm": 1.0814346075057983
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5820224859191963,
|
|
"learning_rate": 0.00010822766347073447,
|
|
"loss_nats": 1.0965744256973267,
|
|
"step": 5250,
|
|
"unclipped_grad_norm": 1.0832712650299072
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5670176875620196,
|
|
"learning_rate": 0.00010772184819092424,
|
|
"loss_nats": 1.0861738920211792,
|
|
"step": 5260,
|
|
"unclipped_grad_norm": 1.0310815572738647
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6681894056371316,
|
|
"learning_rate": 0.0001072170116366926,
|
|
"loss_nats": 1.1563007831573486,
|
|
"step": 5270,
|
|
"unclipped_grad_norm": 1.1380984783172607
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6634579909292448,
|
|
"learning_rate": 0.00010671316243430345,
|
|
"loss_nats": 1.153021216392517,
|
|
"step": 5280,
|
|
"unclipped_grad_norm": 1.1410281658172607
|
|
},
|
|
{
|
|
"bits_per_byte": 1.627283848077365,
|
|
"learning_rate": 0.00010621030919314944,
|
|
"loss_nats": 1.127947211265564,
|
|
"step": 5290,
|
|
"unclipped_grad_norm": 1.0713688135147095
|
|
},
|
|
{
|
|
"bits_per_byte": 1.57530983107337,
|
|
"learning_rate": 0.000105708460505605,
|
|
"loss_nats": 1.0919215679168701,
|
|
"step": 5300,
|
|
"unclipped_grad_norm": 1.103183388710022
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6444668067077193,
|
|
"learning_rate": 0.00010520762494687964,
|
|
"loss_nats": 1.139857530593872,
|
|
"step": 5310,
|
|
"unclipped_grad_norm": 1.1310855150222778
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6034007893346978,
|
|
"learning_rate": 0.00010470781107487111,
|
|
"loss_nats": 1.1113927364349365,
|
|
"step": 5320,
|
|
"unclipped_grad_norm": 1.0775246620178223
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6848084331553064,
|
|
"learning_rate": 0.00010420902743001955,
|
|
"loss_nats": 1.1678202152252197,
|
|
"step": 5330,
|
|
"unclipped_grad_norm": 1.1329472064971924
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5891140185448733,
|
|
"learning_rate": 0.00010371128253516117,
|
|
"loss_nats": 1.1014899015426636,
|
|
"step": 5340,
|
|
"unclipped_grad_norm": 1.0905873775482178
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6324792719772796,
|
|
"learning_rate": 0.000103214584895383,
|
|
"loss_nats": 1.1315484046936035,
|
|
"step": 5350,
|
|
"unclipped_grad_norm": 1.13417649269104
|
|
},
|
|
{
|
|
"bits_per_byte": 1.556942083961222,
|
|
"learning_rate": 0.00010271894299787715,
|
|
"loss_nats": 1.0791900157928467,
|
|
"step": 5360,
|
|
"unclipped_grad_norm": 1.11483633518219
|
|
},
|
|
{
|
|
"bits_per_byte": 1.585282417066301,
|
|
"learning_rate": 0.00010222436531179622,
|
|
"loss_nats": 1.0988340377807617,
|
|
"step": 5370,
|
|
"unclipped_grad_norm": 1.0698448419570923
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6547109533063624,
|
|
"learning_rate": 0.0001017308602881082,
|
|
"loss_nats": 1.1469582319259644,
|
|
"step": 5380,
|
|
"unclipped_grad_norm": 1.1219947338104248
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5562904416971028,
|
|
"learning_rate": 0.00010123843635945242,
|
|
"loss_nats": 1.0787383317947388,
|
|
"step": 5390,
|
|
"unclipped_grad_norm": 1.115169644355774
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6265400231123834,
|
|
"learning_rate": 0.00010074710193999505,
|
|
"loss_nats": 1.1274316310882568,
|
|
"step": 5400,
|
|
"unclipped_grad_norm": 1.079226016998291
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5897746039068374,
|
|
"learning_rate": 0.0001002568654252858,
|
|
"loss_nats": 1.1019477844238281,
|
|
"step": 5410,
|
|
"unclipped_grad_norm": 1.0649707317352295
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5705084194265821,
|
|
"learning_rate": 9.976773519211404e-05,
|
|
"loss_nats": 1.0885934829711914,
|
|
"step": 5420,
|
|
"unclipped_grad_norm": 1.115608811378479
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6107165873370999,
|
|
"learning_rate": 9.92797195983659e-05,
|
|
"loss_nats": 1.1164636611938477,
|
|
"step": 5430,
|
|
"unclipped_grad_norm": 1.1144897937774658
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5986757379848928,
|
|
"learning_rate": 9.879282698288142e-05,
|
|
"loss_nats": 1.1081175804138184,
|
|
"step": 5440,
|
|
"unclipped_grad_norm": 1.0857055187225342
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5965995634236771,
|
|
"learning_rate": 9.830706566531197e-05,
|
|
"loss_nats": 1.1066784858703613,
|
|
"step": 5450,
|
|
"unclipped_grad_norm": 1.0530359745025635
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5899876904112564,
|
|
"learning_rate": 9.782244394597826e-05,
|
|
"loss_nats": 1.1020954847335815,
|
|
"step": 5460,
|
|
"unclipped_grad_norm": 1.1017006635665894
|
|
},
|
|
{
|
|
"bits_per_byte": 1.611714258694755,
|
|
"learning_rate": 9.733897010572829e-05,
|
|
"loss_nats": 1.1171551942825317,
|
|
"step": 5470,
|
|
"unclipped_grad_norm": 1.11431884765625
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5834313677950644,
|
|
"learning_rate": 9.685665240579612e-05,
|
|
"loss_nats": 1.0975509881973267,
|
|
"step": 5480,
|
|
"unclipped_grad_norm": 1.0543001890182495
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6635541292310771,
|
|
"learning_rate": 9.637549908766041e-05,
|
|
"loss_nats": 1.153087854385376,
|
|
"step": 5490,
|
|
"unclipped_grad_norm": 1.154997706413269
|
|
},
|
|
{
|
|
"bits_per_byte": 1.62576197359988,
|
|
"learning_rate": 9.589551837290389e-05,
|
|
"loss_nats": 1.126892328262329,
|
|
"step": 5500,
|
|
"unclipped_grad_norm": 1.1008871793746948
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5806631350467766,
|
|
"learning_rate": 9.541671846307257e-05,
|
|
"loss_nats": 1.0956321954727173,
|
|
"step": 5510,
|
|
"unclipped_grad_norm": 1.0707919597625732
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5578388015028208,
|
|
"learning_rate": 9.493910753953597e-05,
|
|
"loss_nats": 1.0798115730285645,
|
|
"step": 5520,
|
|
"unclipped_grad_norm": 1.0223692655563354
|
|
},
|
|
{
|
|
"bits_per_byte": 1.573020741990386,
|
|
"learning_rate": 9.446269376334689e-05,
|
|
"loss_nats": 1.0903348922729492,
|
|
"step": 5530,
|
|
"unclipped_grad_norm": 1.1420265436172485
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6833484724321321,
|
|
"learning_rate": 9.39874852751024e-05,
|
|
"loss_nats": 1.1668082475662231,
|
|
"step": 5540,
|
|
"unclipped_grad_norm": 1.1607446670532227
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6178791487975832,
|
|
"learning_rate": 9.351349019480433e-05,
|
|
"loss_nats": 1.121428370475769,
|
|
"step": 5550,
|
|
"unclipped_grad_norm": 1.1443098783493042
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6447739677221602,
|
|
"learning_rate": 9.304071662172086e-05,
|
|
"loss_nats": 1.1400704383850098,
|
|
"step": 5560,
|
|
"unclipped_grad_norm": 1.1718472242355347
|
|
},
|
|
{
|
|
"bits_per_byte": 1.59615963180277,
|
|
"learning_rate": 9.256917263424787e-05,
|
|
"loss_nats": 1.1063735485076904,
|
|
"step": 5570,
|
|
"unclipped_grad_norm": 1.093506097793579
|
|
},
|
|
{
|
|
"bits_per_byte": 1.624393163681663,
|
|
"learning_rate": 9.209886628977114e-05,
|
|
"loss_nats": 1.1259435415267944,
|
|
"step": 5580,
|
|
"unclipped_grad_norm": 1.0647783279418945
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6292150412639035,
|
|
"learning_rate": 9.162980562452843e-05,
|
|
"loss_nats": 1.1292858123779297,
|
|
"step": 5590,
|
|
"unclipped_grad_norm": 1.1650937795639038
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6101528282075717,
|
|
"learning_rate": 9.116199865347238e-05,
|
|
"loss_nats": 1.1160728931427002,
|
|
"step": 5600,
|
|
"unclipped_grad_norm": 1.1434698104858398
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5665055232277503,
|
|
"learning_rate": 9.069545337013333e-05,
|
|
"loss_nats": 1.085818886756897,
|
|
"step": 5610,
|
|
"unclipped_grad_norm": 1.1304930448532104
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6202220684502873,
|
|
"learning_rate": 9.0230177746483e-05,
|
|
"loss_nats": 1.1230523586273193,
|
|
"step": 5620,
|
|
"unclipped_grad_norm": 1.1116455793380737
|
|
},
|
|
{
|
|
"bits_per_byte": 1.538340096478599,
|
|
"learning_rate": 8.976617973279797e-05,
|
|
"loss_nats": 1.066296100616455,
|
|
"step": 5630,
|
|
"unclipped_grad_norm": 1.058684229850769
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5357409226761094,
|
|
"learning_rate": 8.930346725752419e-05,
|
|
"loss_nats": 1.0644944906234741,
|
|
"step": 5640,
|
|
"unclipped_grad_norm": 1.033333420753479
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6143326945541414,
|
|
"learning_rate": 8.88420482271411e-05,
|
|
"loss_nats": 1.1189701557159424,
|
|
"step": 5650,
|
|
"unclipped_grad_norm": 1.1008256673812866
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5570710709493691,
|
|
"learning_rate": 8.838193052602691e-05,
|
|
"loss_nats": 1.0792794227600098,
|
|
"step": 5660,
|
|
"unclipped_grad_norm": 1.1397353410720825
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6224767610030988,
|
|
"learning_rate": 8.792312201632352e-05,
|
|
"loss_nats": 1.12461519241333,
|
|
"step": 5670,
|
|
"unclipped_grad_norm": 1.1256861686706543
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6278986860541997,
|
|
"learning_rate": 8.746563053780261e-05,
|
|
"loss_nats": 1.128373384475708,
|
|
"step": 5680,
|
|
"unclipped_grad_norm": 1.2497283220291138
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6402167714395974,
|
|
"learning_rate": 8.700946390773114e-05,
|
|
"loss_nats": 1.1369116306304932,
|
|
"step": 5690,
|
|
"unclipped_grad_norm": 1.099176049232483
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6346018818542285,
|
|
"learning_rate": 8.655462992073833e-05,
|
|
"loss_nats": 1.1330196857452393,
|
|
"step": 5700,
|
|
"unclipped_grad_norm": 1.132989525794983
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6142821316547877,
|
|
"learning_rate": 8.610113634868199e-05,
|
|
"loss_nats": 1.1189351081848145,
|
|
"step": 5710,
|
|
"unclipped_grad_norm": 1.169878363609314
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6389509791292471,
|
|
"learning_rate": 8.564899094051613e-05,
|
|
"loss_nats": 1.1360342502593994,
|
|
"step": 5720,
|
|
"unclipped_grad_norm": 1.1713305711746216
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5731861893005161,
|
|
"learning_rate": 8.519820142215816e-05,
|
|
"loss_nats": 1.090449571609497,
|
|
"step": 5730,
|
|
"unclipped_grad_norm": 1.0991312265396118
|
|
},
|
|
{
|
|
"bits_per_byte": 1.642317195554585,
|
|
"learning_rate": 8.474877549635728e-05,
|
|
"loss_nats": 1.1383675336837769,
|
|
"step": 5740,
|
|
"unclipped_grad_norm": 1.074776530265808
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5380594207923908,
|
|
"learning_rate": 8.430072084256247e-05,
|
|
"loss_nats": 1.0661015510559082,
|
|
"step": 5750,
|
|
"unclipped_grad_norm": 1.1045373678207397
|
|
},
|
|
{
|
|
"bits_per_byte": 1.574528857855802,
|
|
"learning_rate": 8.385404511679161e-05,
|
|
"loss_nats": 1.09138023853302,
|
|
"step": 5760,
|
|
"unclipped_grad_norm": 1.1376982927322388
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5630001728378644,
|
|
"learning_rate": 8.340875595150034e-05,
|
|
"loss_nats": 1.083389163017273,
|
|
"step": 5770,
|
|
"unclipped_grad_norm": 1.1196208000183105
|
|
},
|
|
{
|
|
"bits_per_byte": 1.658872245526639,
|
|
"learning_rate": 8.296486095545203e-05,
|
|
"loss_nats": 1.149842619895935,
|
|
"step": 5780,
|
|
"unclipped_grad_norm": 1.1231145858764648
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4960170579104213,
|
|
"learning_rate": 8.252236771358729e-05,
|
|
"loss_nats": 1.0369600057601929,
|
|
"step": 5790,
|
|
"unclipped_grad_norm": 1.1306540966033936
|
|
},
|
|
{
|
|
"bits_per_byte": 1.632525363327711,
|
|
"learning_rate": 8.208128378689484e-05,
|
|
"loss_nats": 1.1315803527832031,
|
|
"step": 5800,
|
|
"unclipped_grad_norm": 1.1228681802749634
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5271961366506415,
|
|
"learning_rate": 8.16416167122819e-05,
|
|
"loss_nats": 1.058571696281433,
|
|
"step": 5810,
|
|
"unclipped_grad_norm": 1.0998179912567139
|
|
},
|
|
{
|
|
"bits_per_byte": 1.544303422914616,
|
|
"learning_rate": 8.120337400244578e-05,
|
|
"loss_nats": 1.0704295635223389,
|
|
"step": 5820,
|
|
"unclipped_grad_norm": 1.1354378461837769
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5973088198758354,
|
|
"learning_rate": 8.076656314574521e-05,
|
|
"loss_nats": 1.1071701049804688,
|
|
"step": 5830,
|
|
"unclipped_grad_norm": 1.112725853919983
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5755839714188453,
|
|
"learning_rate": 8.03311916060725e-05,
|
|
"loss_nats": 1.092111587524414,
|
|
"step": 5840,
|
|
"unclipped_grad_norm": 1.105390191078186
|
|
},
|
|
{
|
|
"bits_per_byte": 1.551858620767019,
|
|
"learning_rate": 7.989726682272606e-05,
|
|
"loss_nats": 1.0756664276123047,
|
|
"step": 5850,
|
|
"unclipped_grad_norm": 1.0705221891403198
|
|
},
|
|
{
|
|
"bits_per_byte": 1.58110633433805,
|
|
"learning_rate": 7.946479621028318e-05,
|
|
"loss_nats": 1.0959393978118896,
|
|
"step": 5860,
|
|
"unclipped_grad_norm": 1.1176000833511353
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6256357383341469,
|
|
"learning_rate": 7.903378715847343e-05,
|
|
"loss_nats": 1.1268048286437988,
|
|
"step": 5870,
|
|
"unclipped_grad_norm": 1.1718623638153076
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5672180473502748,
|
|
"learning_rate": 7.860424703205224e-05,
|
|
"loss_nats": 1.0863127708435059,
|
|
"step": 5880,
|
|
"unclipped_grad_norm": 1.1114801168441772
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6374564498932493,
|
|
"learning_rate": 7.817618317067534e-05,
|
|
"loss_nats": 1.1349983215332031,
|
|
"step": 5890,
|
|
"unclipped_grad_norm": 1.2559460401535034
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6402117839427224,
|
|
"learning_rate": 7.774960288877295e-05,
|
|
"loss_nats": 1.1369081735610962,
|
|
"step": 5900,
|
|
"unclipped_grad_norm": 1.1818214654922485
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5692250848858438,
|
|
"learning_rate": 7.732451347542522e-05,
|
|
"loss_nats": 1.0877039432525635,
|
|
"step": 5910,
|
|
"unclipped_grad_norm": 1.0690149068832397
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5417845650100792,
|
|
"learning_rate": 7.690092219423722e-05,
|
|
"loss_nats": 1.0686836242675781,
|
|
"step": 5920,
|
|
"unclipped_grad_norm": 1.1236122846603394
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6662781624380931,
|
|
"learning_rate": 7.647883628321536e-05,
|
|
"loss_nats": 1.1549760103225708,
|
|
"step": 5930,
|
|
"unclipped_grad_norm": 1.2468876838684082
|
|
},
|
|
{
|
|
"bits_per_byte": 1.596861664983592,
|
|
"learning_rate": 7.605826295464316e-05,
|
|
"loss_nats": 1.1068601608276367,
|
|
"step": 5940,
|
|
"unclipped_grad_norm": 1.2251354455947876
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5910101272706358,
|
|
"learning_rate": 7.563920939495851e-05,
|
|
"loss_nats": 1.102804183959961,
|
|
"step": 5950,
|
|
"unclipped_grad_norm": 1.11275053024292
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5416916943786134,
|
|
"learning_rate": 7.522168276463047e-05,
|
|
"loss_nats": 1.0686192512512207,
|
|
"step": 5960,
|
|
"unclipped_grad_norm": 1.1575579643249512
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5659371205666486,
|
|
"learning_rate": 7.480569019803719e-05,
|
|
"loss_nats": 1.0854249000549316,
|
|
"step": 5970,
|
|
"unclipped_grad_norm": 1.102811336517334
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5505245513442762,
|
|
"learning_rate": 7.439123880334388e-05,
|
|
"loss_nats": 1.0747417211532593,
|
|
"step": 5980,
|
|
"unclipped_grad_norm": 1.1679552793502808
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5609080038901184,
|
|
"learning_rate": 7.39783356623813e-05,
|
|
"loss_nats": 1.0819389820098877,
|
|
"step": 5990,
|
|
"unclipped_grad_norm": 1.1159348487854004
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6522259759840456,
|
|
"learning_rate": 7.356698783052497e-05,
|
|
"loss_nats": 1.1452357769012451,
|
|
"step": 6000,
|
|
"unclipped_grad_norm": 1.1946688890457153
|
|
},
|
|
{
|
|
"bits_per_byte": 1.610257393659296,
|
|
"learning_rate": 7.315720233657431e-05,
|
|
"loss_nats": 1.116145372390747,
|
|
"step": 6010,
|
|
"unclipped_grad_norm": 1.2001700401306152
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5550891428808262,
|
|
"learning_rate": 7.274898618263286e-05,
|
|
"loss_nats": 1.0779056549072266,
|
|
"step": 6020,
|
|
"unclipped_grad_norm": 1.11931574344635
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5522581364649735,
|
|
"learning_rate": 7.234234634398831e-05,
|
|
"loss_nats": 1.0759433507919312,
|
|
"step": 6030,
|
|
"unclipped_grad_norm": 1.0901384353637695
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5778765001548465,
|
|
"learning_rate": 7.193728976899362e-05,
|
|
"loss_nats": 1.093700647354126,
|
|
"step": 6040,
|
|
"unclipped_grad_norm": 1.1465646028518677
|
|
},
|
|
{
|
|
"bits_per_byte": 1.585459043248737,
|
|
"learning_rate": 7.153382337894797e-05,
|
|
"loss_nats": 1.0989564657211304,
|
|
"step": 6050,
|
|
"unclipped_grad_norm": 1.093497395515442
|
|
},
|
|
{
|
|
"bits_per_byte": 1.580860743112618,
|
|
"learning_rate": 7.113195406797887e-05,
|
|
"loss_nats": 1.0957691669464111,
|
|
"step": 6060,
|
|
"unclipped_grad_norm": 1.1618996858596802
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4692863104347675,
|
|
"learning_rate": 7.073168870292397e-05,
|
|
"loss_nats": 1.0184316635131836,
|
|
"step": 6070,
|
|
"unclipped_grad_norm": 1.095646619796753
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5593341906520728,
|
|
"learning_rate": 7.033303412321404e-05,
|
|
"loss_nats": 1.0808480978012085,
|
|
"step": 6080,
|
|
"unclipped_grad_norm": 1.2226048707962036
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5548665973306097,
|
|
"learning_rate": 6.993599714075586e-05,
|
|
"loss_nats": 1.0777513980865479,
|
|
"step": 6090,
|
|
"unclipped_grad_norm": 1.160715103149414
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6437169623499572,
|
|
"learning_rate": 6.954058453981609e-05,
|
|
"loss_nats": 1.1393377780914307,
|
|
"step": 6100,
|
|
"unclipped_grad_norm": 1.216336965560913
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5953482176559992,
|
|
"learning_rate": 6.914680307690504e-05,
|
|
"loss_nats": 1.1058111190795898,
|
|
"step": 6110,
|
|
"unclipped_grad_norm": 1.1638948917388916
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5315409343593887,
|
|
"learning_rate": 6.875465948066139e-05,
|
|
"loss_nats": 1.0615832805633545,
|
|
"step": 6120,
|
|
"unclipped_grad_norm": 1.1416045427322388
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6181004904692438,
|
|
"learning_rate": 6.836416045173733e-05,
|
|
"loss_nats": 1.121581792831421,
|
|
"step": 6130,
|
|
"unclipped_grad_norm": 1.1373443603515625
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5591868015202834,
|
|
"learning_rate": 6.797531266268377e-05,
|
|
"loss_nats": 1.0807459354400635,
|
|
"step": 6140,
|
|
"unclipped_grad_norm": 1.0817177295684814
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5489426549216403,
|
|
"learning_rate": 6.758812275783663e-05,
|
|
"loss_nats": 1.0736452341079712,
|
|
"step": 6150,
|
|
"unclipped_grad_norm": 1.156706690788269
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6171114182441317,
|
|
"learning_rate": 6.720259735320298e-05,
|
|
"loss_nats": 1.1208962202072144,
|
|
"step": 6160,
|
|
"unclipped_grad_norm": 1.2064051628112793
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5388670513208427,
|
|
"learning_rate": 6.681874303634842e-05,
|
|
"loss_nats": 1.0666613578796387,
|
|
"step": 6170,
|
|
"unclipped_grad_norm": 1.0822783708572388
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5490690621700245,
|
|
"learning_rate": 6.643656636628402e-05,
|
|
"loss_nats": 1.073732852935791,
|
|
"step": 6180,
|
|
"unclipped_grad_norm": 1.109446406364441
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5625440748477764,
|
|
"learning_rate": 6.605607387335466e-05,
|
|
"loss_nats": 1.0830730199813843,
|
|
"step": 6190,
|
|
"unclipped_grad_norm": 1.1676242351531982
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5465116801516945,
|
|
"learning_rate": 6.567727205912724e-05,
|
|
"loss_nats": 1.071960210800171,
|
|
"step": 6200,
|
|
"unclipped_grad_norm": 1.094281792640686
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5807004552820139,
|
|
"learning_rate": 6.530016739627959e-05,
|
|
"loss_nats": 1.0956580638885498,
|
|
"step": 6210,
|
|
"unclipped_grad_norm": 1.2013437747955322
|
|
},
|
|
{
|
|
"bits_per_byte": 1.592342648849521,
|
|
"learning_rate": 6.492476632848993e-05,
|
|
"loss_nats": 1.1037278175354004,
|
|
"step": 6220,
|
|
"unclipped_grad_norm": 1.1159899234771729
|
|
},
|
|
{
|
|
"bits_per_byte": 1.550343453612918,
|
|
"learning_rate": 6.455107527032678e-05,
|
|
"loss_nats": 1.0746161937713623,
|
|
"step": 6230,
|
|
"unclipped_grad_norm": 1.156030535697937
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5936101609863798,
|
|
"learning_rate": 6.417910060713926e-05,
|
|
"loss_nats": 1.1046063899993896,
|
|
"step": 6240,
|
|
"unclipped_grad_norm": 1.2167954444885254
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5477754086702344,
|
|
"learning_rate": 6.380884869494799e-05,
|
|
"loss_nats": 1.07283616065979,
|
|
"step": 6250,
|
|
"unclipped_grad_norm": 1.1424623727798462
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5145149958892723,
|
|
"learning_rate": 6.34403258603367e-05,
|
|
"loss_nats": 1.0497817993164062,
|
|
"step": 6260,
|
|
"unclipped_grad_norm": 1.159928798675537
|
|
},
|
|
{
|
|
"bits_per_byte": 1.558105374611658,
|
|
"learning_rate": 6.30735384003437e-05,
|
|
"loss_nats": 1.0799963474273682,
|
|
"step": 6270,
|
|
"unclipped_grad_norm": 1.1427968740463257
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5868821996846267,
|
|
"learning_rate": 6.270849258235484e-05,
|
|
"loss_nats": 1.099942922592163,
|
|
"step": 6280,
|
|
"unclipped_grad_norm": 1.2086946964263916
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5572024656946284,
|
|
"learning_rate": 6.234519464399581e-05,
|
|
"loss_nats": 1.0793704986572266,
|
|
"step": 6290,
|
|
"unclipped_grad_norm": 1.26358163356781
|
|
},
|
|
{
|
|
"bits_per_byte": 1.525056844456559,
|
|
"learning_rate": 6.198365079302614e-05,
|
|
"loss_nats": 1.057088851928711,
|
|
"step": 6300,
|
|
"unclipped_grad_norm": 1.1142038106918335
|
|
},
|
|
{
|
|
"bits_per_byte": 1.585295831713068,
|
|
"learning_rate": 6.16238672072326e-05,
|
|
"loss_nats": 1.0988433361053467,
|
|
"step": 6310,
|
|
"unclipped_grad_norm": 1.202506422996521
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5589988244828903,
|
|
"learning_rate": 6.126585003432406e-05,
|
|
"loss_nats": 1.0806156396865845,
|
|
"step": 6320,
|
|
"unclipped_grad_norm": 1.1080162525177002
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5721742433828394,
|
|
"learning_rate": 6.090960539182614e-05,
|
|
"loss_nats": 1.0897481441497803,
|
|
"step": 6330,
|
|
"unclipped_grad_norm": 1.1463266611099243
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4897746036320538,
|
|
"learning_rate": 6.0555139366976935e-05,
|
|
"loss_nats": 1.0326330661773682,
|
|
"step": 6340,
|
|
"unclipped_grad_norm": 1.0811859369277954
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5979289893148465,
|
|
"learning_rate": 6.020245801662274e-05,
|
|
"loss_nats": 1.1075999736785889,
|
|
"step": 6350,
|
|
"unclipped_grad_norm": 1.1469974517822266
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6121533304024078,
|
|
"learning_rate": 5.985156736711483e-05,
|
|
"loss_nats": 1.1174595355987549,
|
|
"step": 6360,
|
|
"unclipped_grad_norm": 1.2663604021072388
|
|
},
|
|
{
|
|
"bits_per_byte": 1.522211219515383,
|
|
"learning_rate": 5.950247341420618e-05,
|
|
"loss_nats": 1.0551164150238037,
|
|
"step": 6370,
|
|
"unclipped_grad_norm": 1.136039137840271
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5209657211578345,
|
|
"learning_rate": 5.915518212294935e-05,
|
|
"loss_nats": 1.054253101348877,
|
|
"step": 6380,
|
|
"unclipped_grad_norm": 1.1138842105865479
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6340888576067047,
|
|
"learning_rate": 5.880969942759425e-05,
|
|
"loss_nats": 1.1326640844345093,
|
|
"step": 6390,
|
|
"unclipped_grad_norm": 1.227125883102417
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5932569086215076,
|
|
"learning_rate": 5.846603123148688e-05,
|
|
"loss_nats": 1.1043615341186523,
|
|
"step": 6400,
|
|
"unclipped_grad_norm": 1.1935430765151978
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6221133616618257,
|
|
"learning_rate": 5.812418340696858e-05,
|
|
"loss_nats": 1.1243633031845093,
|
|
"step": 6410,
|
|
"unclipped_grad_norm": 1.289966106414795
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5341647016809516,
|
|
"learning_rate": 5.778416179527541e-05,
|
|
"loss_nats": 1.0634019374847412,
|
|
"step": 6420,
|
|
"unclipped_grad_norm": 1.2202966213226318
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5828500664351481,
|
|
"learning_rate": 5.7445972206438635e-05,
|
|
"loss_nats": 1.097148060798645,
|
|
"step": 6430,
|
|
"unclipped_grad_norm": 1.1407660245895386
|
|
},
|
|
{
|
|
"bits_per_byte": 1.518648426920109,
|
|
"learning_rate": 5.710962041918516e-05,
|
|
"loss_nats": 1.0526468753814697,
|
|
"step": 6440,
|
|
"unclipped_grad_norm": 1.1754008531570435
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5227140967865058,
|
|
"learning_rate": 5.677511218083908e-05,
|
|
"loss_nats": 1.0554649829864502,
|
|
"step": 6450,
|
|
"unclipped_grad_norm": 1.1674573421478271
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5506523344538674,
|
|
"learning_rate": 5.644245320722316e-05,
|
|
"loss_nats": 1.0748302936553955,
|
|
"step": 6460,
|
|
"unclipped_grad_norm": 1.1624079942703247
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5647150118496176,
|
|
"learning_rate": 5.61116491825615e-05,
|
|
"loss_nats": 1.0845777988433838,
|
|
"step": 6470,
|
|
"unclipped_grad_norm": 1.1915463209152222
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5303090226312583,
|
|
"learning_rate": 5.5782705759382104e-05,
|
|
"loss_nats": 1.0607293844223022,
|
|
"step": 6480,
|
|
"unclipped_grad_norm": 1.1169558763504028
|
|
},
|
|
{
|
|
"bits_per_byte": 1.610437975442702,
|
|
"learning_rate": 5.5455628558420546e-05,
|
|
"loss_nats": 1.1162705421447754,
|
|
"step": 6490,
|
|
"unclipped_grad_norm": 1.212812900543213
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6629645727039195,
|
|
"learning_rate": 5.513042316852368e-05,
|
|
"loss_nats": 1.152679204940796,
|
|
"step": 6500,
|
|
"unclipped_grad_norm": 1.249900221824646
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6085310318099353,
|
|
"learning_rate": 5.480709514655443e-05,
|
|
"loss_nats": 1.1149487495422363,
|
|
"step": 6510,
|
|
"unclipped_grad_norm": 1.2940583229064941
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5857588090091908,
|
|
"learning_rate": 5.448565001729653e-05,
|
|
"loss_nats": 1.0991642475128174,
|
|
"step": 6520,
|
|
"unclipped_grad_norm": 1.1899638175964355
|
|
},
|
|
{
|
|
"bits_per_byte": 1.557970024265429,
|
|
"learning_rate": 5.41660932733604e-05,
|
|
"loss_nats": 1.0799025297164917,
|
|
"step": 6530,
|
|
"unclipped_grad_norm": 1.18255615234375
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5442463246745295,
|
|
"learning_rate": 5.384843037508902e-05,
|
|
"loss_nats": 1.070389986038208,
|
|
"step": 6540,
|
|
"unclipped_grad_norm": 1.1805294752120972
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5888496812104973,
|
|
"learning_rate": 5.353266675046498e-05,
|
|
"loss_nats": 1.101306676864624,
|
|
"step": 6550,
|
|
"unclipped_grad_norm": 1.1853868961334229
|
|
},
|
|
{
|
|
"bits_per_byte": 1.598041465968511,
|
|
"learning_rate": 5.321880779501729e-05,
|
|
"loss_nats": 1.107677936553955,
|
|
"step": 6560,
|
|
"unclipped_grad_norm": 1.167456865310669
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5763221209563485,
|
|
"learning_rate": 5.290685887172962e-05,
|
|
"loss_nats": 1.092623233795166,
|
|
"step": 6570,
|
|
"unclipped_grad_norm": 1.2579692602157593
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5598688847136053,
|
|
"learning_rate": 5.2596825310948295e-05,
|
|
"loss_nats": 1.0812187194824219,
|
|
"step": 6580,
|
|
"unclipped_grad_norm": 1.218462347984314
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5745766690327418,
|
|
"learning_rate": 5.228871241029154e-05,
|
|
"loss_nats": 1.0914133787155151,
|
|
"step": 6590,
|
|
"unclipped_grad_norm": 1.1964205503463745
|
|
},
|
|
{
|
|
"bits_per_byte": 1.483007602268555,
|
|
"learning_rate": 5.198252543455865e-05,
|
|
"loss_nats": 1.0279425382614136,
|
|
"step": 6600,
|
|
"unclipped_grad_norm": 1.1824510097503662
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5325055850480782,
|
|
"learning_rate": 5.167826961564027e-05,
|
|
"loss_nats": 1.0622519254684448,
|
|
"step": 6610,
|
|
"unclipped_grad_norm": 1.1718802452087402
|
|
},
|
|
{
|
|
"bits_per_byte": 1.648197798335537,
|
|
"learning_rate": 5.137595015242884e-05,
|
|
"loss_nats": 1.1424436569213867,
|
|
"step": 6620,
|
|
"unclipped_grad_norm": 1.2758471965789795
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5936395700196773,
|
|
"learning_rate": 5.1075572210729925e-05,
|
|
"loss_nats": 1.1046267747879028,
|
|
"step": 6630,
|
|
"unclipped_grad_norm": 1.1752488613128662
|
|
},
|
|
{
|
|
"bits_per_byte": 1.550369594975849,
|
|
"learning_rate": 5.07771409231737e-05,
|
|
"loss_nats": 1.074634313583374,
|
|
"step": 6640,
|
|
"unclipped_grad_norm": 1.1907187700271606
|
|
},
|
|
{
|
|
"bits_per_byte": 1.539030778804464,
|
|
"learning_rate": 5.048066138912756e-05,
|
|
"loss_nats": 1.066774845123291,
|
|
"step": 6650,
|
|
"unclipped_grad_norm": 1.172040343284607
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5824082430050816,
|
|
"learning_rate": 5.018613867460863e-05,
|
|
"loss_nats": 1.096841812133789,
|
|
"step": 6660,
|
|
"unclipped_grad_norm": 1.2395251989364624
|
|
},
|
|
{
|
|
"bits_per_byte": 1.55066678099654,
|
|
"learning_rate": 4.989357781219755e-05,
|
|
"loss_nats": 1.0748403072357178,
|
|
"step": 6670,
|
|
"unclipped_grad_norm": 1.1819844245910645
|
|
},
|
|
{
|
|
"bits_per_byte": 1.567913545190364,
|
|
"learning_rate": 4.960298380095222e-05,
|
|
"loss_nats": 1.0867948532104492,
|
|
"step": 6680,
|
|
"unclipped_grad_norm": 1.207129955291748
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5601953077849429,
|
|
"learning_rate": 4.931436160632256e-05,
|
|
"loss_nats": 1.0814449787139893,
|
|
"step": 6690,
|
|
"unclipped_grad_norm": 1.284845232963562
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5747911313983678,
|
|
"learning_rate": 4.9027716160065485e-05,
|
|
"loss_nats": 1.091562032699585,
|
|
"step": 6700,
|
|
"unclipped_grad_norm": 1.1885749101638794
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5878793550943293,
|
|
"learning_rate": 4.874305236016084e-05,
|
|
"loss_nats": 1.1006340980529785,
|
|
"step": 6710,
|
|
"unclipped_grad_norm": 1.178633689880371
|
|
},
|
|
{
|
|
"bits_per_byte": 1.6366173465396898,
|
|
"learning_rate": 4.846037507072752e-05,
|
|
"loss_nats": 1.1344166994094849,
|
|
"step": 6720,
|
|
"unclipped_grad_norm": 1.265533685684204
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5154492056487592,
|
|
"learning_rate": 4.817968912194049e-05,
|
|
"loss_nats": 1.050429344177246,
|
|
"step": 6730,
|
|
"unclipped_grad_norm": 1.138245940208435
|
|
},
|
|
{
|
|
"bits_per_byte": 1.547962697777024,
|
|
"learning_rate": 4.790099930994813e-05,
|
|
"loss_nats": 1.0729659795761108,
|
|
"step": 6740,
|
|
"unclipped_grad_norm": 1.167292833328247
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5245414124519232,
|
|
"learning_rate": 4.762431039679047e-05,
|
|
"loss_nats": 1.0567315816879272,
|
|
"step": 6750,
|
|
"unclipped_grad_norm": 1.1478474140167236
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5232747602283185,
|
|
"learning_rate": 4.734962711031755e-05,
|
|
"loss_nats": 1.0558536052703857,
|
|
"step": 6760,
|
|
"unclipped_grad_norm": 1.185871958732605
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5451932611501802,
|
|
"learning_rate": 4.707695414410892e-05,
|
|
"loss_nats": 1.0710463523864746,
|
|
"step": 6770,
|
|
"unclipped_grad_norm": 1.2135790586471558
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5261805790972967,
|
|
"learning_rate": 4.680629615739322e-05,
|
|
"loss_nats": 1.0578677654266357,
|
|
"step": 6780,
|
|
"unclipped_grad_norm": 1.2295221090316772
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5602444948230898,
|
|
"learning_rate": 4.653765777496863e-05,
|
|
"loss_nats": 1.0814790725708008,
|
|
"step": 6790,
|
|
"unclipped_grad_norm": 1.174469232559204
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5335944072106906,
|
|
"learning_rate": 4.6271043587123986e-05,
|
|
"loss_nats": 1.0630066394805908,
|
|
"step": 6800,
|
|
"unclipped_grad_norm": 1.1970314979553223
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5348973477736272,
|
|
"learning_rate": 4.600645814956006e-05,
|
|
"loss_nats": 1.0639097690582275,
|
|
"step": 6810,
|
|
"unclipped_grad_norm": 1.162747859954834
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5590328770477613,
|
|
"learning_rate": 4.574390598331204e-05,
|
|
"loss_nats": 1.0806392431259155,
|
|
"step": 6820,
|
|
"unclipped_grad_norm": 1.2186238765716553
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5711721004762618,
|
|
"learning_rate": 4.548339157467199e-05,
|
|
"loss_nats": 1.0890535116195679,
|
|
"step": 6830,
|
|
"unclipped_grad_norm": 1.1893293857574463
|
|
},
|
|
{
|
|
"bits_per_byte": 1.526197777362383,
|
|
"learning_rate": 4.52249193751124e-05,
|
|
"loss_nats": 1.0578796863555908,
|
|
"step": 6840,
|
|
"unclipped_grad_norm": 1.2071653604507446
|
|
},
|
|
{
|
|
"bits_per_byte": 1.624858204769596,
|
|
"learning_rate": 4.4968493801209985e-05,
|
|
"loss_nats": 1.1262658834457397,
|
|
"step": 6850,
|
|
"unclipped_grad_norm": 1.2998149394989014
|
|
},
|
|
{
|
|
"bits_per_byte": 1.574837222748799,
|
|
"learning_rate": 4.471411923457034e-05,
|
|
"loss_nats": 1.0915939807891846,
|
|
"step": 6860,
|
|
"unclipped_grad_norm": 1.2379318475723267
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5010148737444944,
|
|
"learning_rate": 4.446180002175291e-05,
|
|
"loss_nats": 1.0404242277145386,
|
|
"step": 6870,
|
|
"unclipped_grad_norm": 1.1776995658874512
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5287346934452601,
|
|
"learning_rate": 4.421154047419693e-05,
|
|
"loss_nats": 1.0596381425857544,
|
|
"step": 6880,
|
|
"unclipped_grad_norm": 1.2149008512496948
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5161942344922967,
|
|
"learning_rate": 4.396334486814753e-05,
|
|
"loss_nats": 1.05094575881958,
|
|
"step": 6890,
|
|
"unclipped_grad_norm": 1.1354871988296509
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5760503883679853,
|
|
"learning_rate": 4.371721744458285e-05,
|
|
"loss_nats": 1.0924348831176758,
|
|
"step": 6900,
|
|
"unclipped_grad_norm": 1.2098510265350342
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5705858116194704,
|
|
"learning_rate": 4.347316240914145e-05,
|
|
"loss_nats": 1.0886471271514893,
|
|
"step": 6910,
|
|
"unclipped_grad_norm": 1.2706652879714966
|
|
},
|
|
{
|
|
"bits_per_byte": 1.523291786510754,
|
|
"learning_rate": 4.3231183932050546e-05,
|
|
"loss_nats": 1.0558654069900513,
|
|
"step": 6920,
|
|
"unclipped_grad_norm": 1.1377208232879639
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5043128130574397,
|
|
"learning_rate": 4.299128614805464e-05,
|
|
"loss_nats": 1.0427101850509644,
|
|
"step": 6930,
|
|
"unclipped_grad_norm": 1.1997718811035156
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5789452003473081,
|
|
"learning_rate": 4.2753473156344965e-05,
|
|
"loss_nats": 1.0944414138793945,
|
|
"step": 6940,
|
|
"unclipped_grad_norm": 1.2278647422790527
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4992432804579567,
|
|
"learning_rate": 4.251774902048942e-05,
|
|
"loss_nats": 1.039196252822876,
|
|
"step": 6950,
|
|
"unclipped_grad_norm": 1.2732465267181396
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4942526878952196,
|
|
"learning_rate": 4.228411776836306e-05,
|
|
"loss_nats": 1.0357370376586914,
|
|
"step": 6960,
|
|
"unclipped_grad_norm": 1.1388111114501953
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4775614276636824,
|
|
"learning_rate": 4.2052583392079427e-05,
|
|
"loss_nats": 1.024167537689209,
|
|
"step": 6970,
|
|
"unclipped_grad_norm": 1.2496774196624756
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5972561931846714,
|
|
"learning_rate": 4.182314984792214e-05,
|
|
"loss_nats": 1.1071336269378662,
|
|
"step": 6980,
|
|
"unclipped_grad_norm": 1.2329373359680176
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5648880263963856,
|
|
"learning_rate": 4.159582105627748e-05,
|
|
"loss_nats": 1.0846977233886719,
|
|
"step": 6990,
|
|
"unclipped_grad_norm": 1.225949764251709
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5748815942727217,
|
|
"learning_rate": 4.137060090156724e-05,
|
|
"loss_nats": 1.0916247367858887,
|
|
"step": 7000,
|
|
"unclipped_grad_norm": 1.2655893564224243
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4699690815586928,
|
|
"learning_rate": 4.114749323218249e-05,
|
|
"loss_nats": 1.0189049243927002,
|
|
"step": 7010,
|
|
"unclipped_grad_norm": 1.1935946941375732
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5527095909234883,
|
|
"learning_rate": 4.0926501860417706e-05,
|
|
"loss_nats": 1.076256275177002,
|
|
"step": 7020,
|
|
"unclipped_grad_norm": 1.2986186742782593
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5324563980099315,
|
|
"learning_rate": 4.070763056240571e-05,
|
|
"loss_nats": 1.0622178316116333,
|
|
"step": 7030,
|
|
"unclipped_grad_norm": 1.2135052680969238
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5215922539549278,
|
|
"learning_rate": 4.049088307805306e-05,
|
|
"loss_nats": 1.0546873807907104,
|
|
"step": 7040,
|
|
"unclipped_grad_norm": 1.1919881105422974
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5129705916845244,
|
|
"learning_rate": 4.027626311097629e-05,
|
|
"loss_nats": 1.0487112998962402,
|
|
"step": 7050,
|
|
"unclipped_grad_norm": 1.177833914756775
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5002927185935215,
|
|
"learning_rate": 4.006377432843846e-05,
|
|
"loss_nats": 1.0399236679077148,
|
|
"step": 7060,
|
|
"unclipped_grad_norm": 1.2103830575942993
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5379128915738558,
|
|
"learning_rate": 3.985342036128656e-05,
|
|
"loss_nats": 1.065999984741211,
|
|
"step": 7070,
|
|
"unclipped_grad_norm": 1.2223129272460938
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5494862920810175,
|
|
"learning_rate": 3.964520480388956e-05,
|
|
"loss_nats": 1.0740220546722412,
|
|
"step": 7080,
|
|
"unclipped_grad_norm": 1.2125953435897827
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5050088268454815,
|
|
"learning_rate": 3.943913121407678e-05,
|
|
"loss_nats": 1.0431926250457764,
|
|
"step": 7090,
|
|
"unclipped_grad_norm": 1.274321436882019
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5086218383748073,
|
|
"learning_rate": 3.923520311307735e-05,
|
|
"loss_nats": 1.0456969738006592,
|
|
"step": 7100,
|
|
"unclipped_grad_norm": 1.244695782661438
|
|
},
|
|
{
|
|
"bits_per_byte": 1.480645764524256,
|
|
"learning_rate": 3.9033423985459816e-05,
|
|
"loss_nats": 1.0263054370880127,
|
|
"step": 7110,
|
|
"unclipped_grad_norm": 1.1544733047485352
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5449419944972695,
|
|
"learning_rate": 3.88337972790728e-05,
|
|
"loss_nats": 1.070872187614441,
|
|
"step": 7120,
|
|
"unclipped_grad_norm": 1.25104558467865
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5693447848108444,
|
|
"learning_rate": 3.8636326404985845e-05,
|
|
"loss_nats": 1.0877869129180908,
|
|
"step": 7130,
|
|
"unclipped_grad_norm": 1.2125219106674194
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5401648324042534,
|
|
"learning_rate": 3.844101473743143e-05,
|
|
"loss_nats": 1.0675609111785889,
|
|
"step": 7140,
|
|
"unclipped_grad_norm": 1.2392476797103882
|
|
},
|
|
{
|
|
"bits_per_byte": 1.581065058501843,
|
|
"learning_rate": 3.824786561374701e-05,
|
|
"loss_nats": 1.0959107875823975,
|
|
"step": 7150,
|
|
"unclipped_grad_norm": 1.2786449193954468
|
|
},
|
|
{
|
|
"bits_per_byte": 1.568793580414829,
|
|
"learning_rate": 3.805688233431824e-05,
|
|
"loss_nats": 1.0874048471450806,
|
|
"step": 7160,
|
|
"unclipped_grad_norm": 1.2588245868682861
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5794673396753276,
|
|
"learning_rate": 3.786806816252236e-05,
|
|
"loss_nats": 1.0948033332824707,
|
|
"step": 7170,
|
|
"unclipped_grad_norm": 1.2507553100585938
|
|
},
|
|
{
|
|
"bits_per_byte": 1.519351835962138,
|
|
"learning_rate": 3.768142632467269e-05,
|
|
"loss_nats": 1.0531344413757324,
|
|
"step": 7180,
|
|
"unclipped_grad_norm": 1.194468379020691
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5739864245749806,
|
|
"learning_rate": 3.7496960009963236e-05,
|
|
"loss_nats": 1.0910042524337769,
|
|
"step": 7190,
|
|
"unclipped_grad_norm": 1.2753334045410156
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5095894127685614,
|
|
"learning_rate": 3.7314672370414325e-05,
|
|
"loss_nats": 1.0463676452636719,
|
|
"step": 7200,
|
|
"unclipped_grad_norm": 1.1532927751541138
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5104264523303108,
|
|
"learning_rate": 3.713456652081882e-05,
|
|
"loss_nats": 1.0469478368759155,
|
|
"step": 7210,
|
|
"unclipped_grad_norm": 1.1723830699920654
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5078061246617651,
|
|
"learning_rate": 3.695664553868873e-05,
|
|
"loss_nats": 1.0451315641403198,
|
|
"step": 7220,
|
|
"unclipped_grad_norm": 1.254270076751709
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5518125294165879,
|
|
"learning_rate": 3.678091246420277e-05,
|
|
"loss_nats": 1.075634479522705,
|
|
"step": 7230,
|
|
"unclipped_grad_norm": 1.2554386854171753
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5582962753541159,
|
|
"learning_rate": 3.660737030015427e-05,
|
|
"loss_nats": 1.0801286697387695,
|
|
"step": 7240,
|
|
"unclipped_grad_norm": 1.2507786750793457
|
|
},
|
|
{
|
|
"bits_per_byte": 1.571778683285855,
|
|
"learning_rate": 3.6436022011900056e-05,
|
|
"loss_nats": 1.0894739627838135,
|
|
"step": 7250,
|
|
"unclipped_grad_norm": 1.2601356506347656
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5335579468887077,
|
|
"learning_rate": 3.626687052730954e-05,
|
|
"loss_nats": 1.062981367111206,
|
|
"step": 7260,
|
|
"unclipped_grad_norm": 1.203548789024353
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5322409037484004,
|
|
"learning_rate": 3.6099918736714955e-05,
|
|
"loss_nats": 1.0620684623718262,
|
|
"step": 7270,
|
|
"unclipped_grad_norm": 1.2632365226745605
|
|
},
|
|
{
|
|
"bits_per_byte": 1.522663877852454,
|
|
"learning_rate": 3.5935169492861716e-05,
|
|
"loss_nats": 1.0554301738739014,
|
|
"step": 7280,
|
|
"unclipped_grad_norm": 1.226701259613037
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5219511817472784,
|
|
"learning_rate": 3.5772625610859886e-05,
|
|
"loss_nats": 1.054936170578003,
|
|
"step": 7290,
|
|
"unclipped_grad_norm": 1.2358945608139038
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5307491262348163,
|
|
"learning_rate": 3.5612289868135887e-05,
|
|
"loss_nats": 1.0610344409942627,
|
|
"step": 7300,
|
|
"unclipped_grad_norm": 1.1983592510223389
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5145411372522035,
|
|
"learning_rate": 3.545416500438522e-05,
|
|
"loss_nats": 1.049799919128418,
|
|
"step": 7310,
|
|
"unclipped_grad_norm": 1.224237084388733
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4925153191562035,
|
|
"learning_rate": 3.52982537215255e-05,
|
|
"loss_nats": 1.0345327854156494,
|
|
"step": 7320,
|
|
"unclipped_grad_norm": 1.2309412956237793
|
|
},
|
|
{
|
|
"bits_per_byte": 1.55153787312316,
|
|
"learning_rate": 3.5144558683650396e-05,
|
|
"loss_nats": 1.0754441022872925,
|
|
"step": 7330,
|
|
"unclipped_grad_norm": 1.2790515422821045
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5015385609163716,
|
|
"learning_rate": 3.499308251698404e-05,
|
|
"loss_nats": 1.0407872200012207,
|
|
"step": 7340,
|
|
"unclipped_grad_norm": 1.2610660791397095
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5181352306899345,
|
|
"learning_rate": 3.484382780983616e-05,
|
|
"loss_nats": 1.0522911548614502,
|
|
"step": 7350,
|
|
"unclipped_grad_norm": 1.2389053106307983
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5177651240252776,
|
|
"learning_rate": 3.469679711255795e-05,
|
|
"loss_nats": 1.052034616470337,
|
|
"step": 7360,
|
|
"unclipped_grad_norm": 1.2198708057403564
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4782792832483838,
|
|
"learning_rate": 3.4551992937498284e-05,
|
|
"loss_nats": 1.024665117263794,
|
|
"step": 7370,
|
|
"unclipped_grad_norm": 1.2335501909255981
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4776752801785535,
|
|
"learning_rate": 3.440941775896104e-05,
|
|
"loss_nats": 1.0242464542388916,
|
|
"step": 7380,
|
|
"unclipped_grad_norm": 1.1802135705947876
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4876678161589845,
|
|
"learning_rate": 3.4269074013162596e-05,
|
|
"loss_nats": 1.031172752380371,
|
|
"step": 7390,
|
|
"unclipped_grad_norm": 1.2134222984313965
|
|
},
|
|
{
|
|
"bits_per_byte": 1.508026778402822,
|
|
"learning_rate": 3.41309640981904e-05,
|
|
"loss_nats": 1.0452845096588135,
|
|
"step": 7400,
|
|
"unclipped_grad_norm": 1.23512601852417
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5449191208047048,
|
|
"learning_rate": 3.3995090373961795e-05,
|
|
"loss_nats": 1.0708563327789307,
|
|
"step": 7410,
|
|
"unclipped_grad_norm": 1.250113606452942
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5275736385692853,
|
|
"learning_rate": 3.3861455162183876e-05,
|
|
"loss_nats": 1.058833360671997,
|
|
"step": 7420,
|
|
"unclipped_grad_norm": 1.2959712743759155
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4779425412179943,
|
|
"learning_rate": 3.3730060746313664e-05,
|
|
"loss_nats": 1.0244317054748535,
|
|
"step": 7430,
|
|
"unclipped_grad_norm": 1.2527908086776733
|
|
},
|
|
{
|
|
"bits_per_byte": 1.468023269846831,
|
|
"learning_rate": 3.3600909371519225e-05,
|
|
"loss_nats": 1.0175561904907227,
|
|
"step": 7440,
|
|
"unclipped_grad_norm": 1.2249972820281982
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5643605556061893,
|
|
"learning_rate": 3.3474003244641194e-05,
|
|
"loss_nats": 1.0843321084976196,
|
|
"step": 7450,
|
|
"unclipped_grad_norm": 1.2696515321731567
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5085103936170483,
|
|
"learning_rate": 3.334934453415514e-05,
|
|
"loss_nats": 1.0456197261810303,
|
|
"step": 7460,
|
|
"unclipped_grad_norm": 1.2498137950897217
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4827952036947394,
|
|
"learning_rate": 3.3226935370134486e-05,
|
|
"loss_nats": 1.0277953147888184,
|
|
"step": 7470,
|
|
"unclipped_grad_norm": 1.2631312608718872
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5577199614910746,
|
|
"learning_rate": 3.31067778442141e-05,
|
|
"loss_nats": 1.0797291994094849,
|
|
"step": 7480,
|
|
"unclipped_grad_norm": 1.1877262592315674
|
|
},
|
|
{
|
|
"bits_per_byte": 1.530344623039987,
|
|
"learning_rate": 3.298887400955457e-05,
|
|
"loss_nats": 1.0607540607452393,
|
|
"step": 7490,
|
|
"unclipped_grad_norm": 1.268275499343872
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5391859071555423,
|
|
"learning_rate": 3.287322588080714e-05,
|
|
"loss_nats": 1.0668823719024658,
|
|
"step": 7500,
|
|
"unclipped_grad_norm": 1.3035756349563599
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5412744644676202,
|
|
"learning_rate": 3.275983543407924e-05,
|
|
"loss_nats": 1.0683300495147705,
|
|
"step": 7510,
|
|
"unclipped_grad_norm": 1.2108540534973145
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5426088778556648,
|
|
"learning_rate": 3.2648704606900735e-05,
|
|
"loss_nats": 1.069254994392395,
|
|
"step": 7520,
|
|
"unclipped_grad_norm": 1.265081524848938
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5294915890917073,
|
|
"learning_rate": 3.253983529819086e-05,
|
|
"loss_nats": 1.0601627826690674,
|
|
"step": 7530,
|
|
"unclipped_grad_norm": 1.2194392681121826
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5287204188852386,
|
|
"learning_rate": 3.243322936822571e-05,
|
|
"loss_nats": 1.0596282482147217,
|
|
"step": 7540,
|
|
"unclipped_grad_norm": 1.2046699523925781
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5311642923539992,
|
|
"learning_rate": 3.232888863860648e-05,
|
|
"loss_nats": 1.0613222122192383,
|
|
"step": 7550,
|
|
"unclipped_grad_norm": 1.2151964902877808
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4454386801181778,
|
|
"learning_rate": 3.222681489222838e-05,
|
|
"loss_nats": 1.0019017457962036,
|
|
"step": 7560,
|
|
"unclipped_grad_norm": 1.2129703760147095
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4692137337561033,
|
|
"learning_rate": 3.212700987325009e-05,
|
|
"loss_nats": 1.0183813571929932,
|
|
"step": 7570,
|
|
"unclipped_grad_norm": 1.2061578035354614
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4652694836412157,
|
|
"learning_rate": 3.202947528706403e-05,
|
|
"loss_nats": 1.0156474113464355,
|
|
"step": 7580,
|
|
"unclipped_grad_norm": 1.1768887042999268
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5398244787381958,
|
|
"learning_rate": 3.1934212800267116e-05,
|
|
"loss_nats": 1.0673249959945679,
|
|
"step": 7590,
|
|
"unclipped_grad_norm": 1.2310807704925537
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5671884663343265,
|
|
"learning_rate": 3.1841224040632484e-05,
|
|
"loss_nats": 1.0862922668457031,
|
|
"step": 7600,
|
|
"unclipped_grad_norm": 1.2284492254257202
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5395315922837765,
|
|
"learning_rate": 3.175051059708143e-05,
|
|
"loss_nats": 1.067121982574463,
|
|
"step": 7610,
|
|
"unclipped_grad_norm": 1.2651493549346924
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5411196800818436,
|
|
"learning_rate": 3.1662074019656414e-05,
|
|
"loss_nats": 1.0682227611541748,
|
|
"step": 7620,
|
|
"unclipped_grad_norm": 1.2471308708190918
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5165897945892812,
|
|
"learning_rate": 3.157591581949455e-05,
|
|
"loss_nats": 1.0512199401855469,
|
|
"step": 7630,
|
|
"unclipped_grad_norm": 1.253797173500061
|
|
},
|
|
{
|
|
"bits_per_byte": 1.497296092884888,
|
|
"learning_rate": 3.149203746880175e-05,
|
|
"loss_nats": 1.037846565246582,
|
|
"step": 7640,
|
|
"unclipped_grad_norm": 1.318769931793213
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4637429656321574,
|
|
"learning_rate": 3.141044040082763e-05,
|
|
"loss_nats": 1.0145893096923828,
|
|
"step": 7650,
|
|
"unclipped_grad_norm": 1.2420055866241455
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4840704550508872,
|
|
"learning_rate": 3.133112600984091e-05,
|
|
"loss_nats": 1.0286792516708374,
|
|
"step": 7660,
|
|
"unclipped_grad_norm": 1.299368977546692
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5379041204586616,
|
|
"learning_rate": 3.125409565110571e-05,
|
|
"loss_nats": 1.0659939050674438,
|
|
"step": 7670,
|
|
"unclipped_grad_norm": 1.2423371076583862
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5171365274363742,
|
|
"learning_rate": 3.117935064085835e-05,
|
|
"loss_nats": 1.0515989065170288,
|
|
"step": 7680,
|
|
"unclipped_grad_norm": 1.2243461608886719
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5051288707357837,
|
|
"learning_rate": 3.110689225628479e-05,
|
|
"loss_nats": 1.0432758331298828,
|
|
"step": 7690,
|
|
"unclipped_grad_norm": 1.245628833770752
|
|
},
|
|
{
|
|
"bits_per_byte": 1.544223278999314,
|
|
"learning_rate": 3.1036721735498926e-05,
|
|
"loss_nats": 1.0703740119934082,
|
|
"step": 7700,
|
|
"unclipped_grad_norm": 1.2915515899658203
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4916550619365878,
|
|
"learning_rate": 3.096884027752132e-05,
|
|
"loss_nats": 1.0339365005493164,
|
|
"step": 7710,
|
|
"unclipped_grad_norm": 1.201813817024231
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4940050328779773,
|
|
"learning_rate": 3.090324904225885e-05,
|
|
"loss_nats": 1.0355653762817383,
|
|
"step": 7720,
|
|
"unclipped_grad_norm": 1.2110096216201782
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4993191248069873,
|
|
"learning_rate": 3.083994915048472e-05,
|
|
"loss_nats": 1.0392488241195679,
|
|
"step": 7730,
|
|
"unclipped_grad_norm": 1.2087814807891846
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5356127956012164,
|
|
"learning_rate": 3.077894168381942e-05,
|
|
"loss_nats": 1.0644056797027588,
|
|
"step": 7740,
|
|
"unclipped_grad_norm": 1.3031466007232666
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4696118735928507,
|
|
"learning_rate": 3.0720227684712254e-05,
|
|
"loss_nats": 1.0186573266983032,
|
|
"step": 7750,
|
|
"unclipped_grad_norm": 1.249332070350647
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4759148657643217,
|
|
"learning_rate": 3.0663808156423456e-05,
|
|
"loss_nats": 1.0230262279510498,
|
|
"step": 7760,
|
|
"unclipped_grad_norm": 1.2120082378387451
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5360676897127488,
|
|
"learning_rate": 3.060968406300711e-05,
|
|
"loss_nats": 1.0647209882736206,
|
|
"step": 7770,
|
|
"unclipped_grad_norm": 1.2807056903839111
|
|
},
|
|
{
|
|
"bits_per_byte": 1.529660132089553,
|
|
"learning_rate": 3.055785632929463e-05,
|
|
"loss_nats": 1.0602796077728271,
|
|
"step": 7780,
|
|
"unclipped_grad_norm": 1.2968212366104126
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5315347429839576,
|
|
"learning_rate": 3.0508325840878984e-05,
|
|
"loss_nats": 1.0615789890289307,
|
|
"step": 7790,
|
|
"unclipped_grad_norm": 1.2608845233917236
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5176234103209667,
|
|
"learning_rate": 3.046109344409957e-05,
|
|
"loss_nats": 1.051936388015747,
|
|
"step": 7800,
|
|
"unclipped_grad_norm": 1.2756648063659668
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5054119541791038,
|
|
"learning_rate": 3.0416159946027727e-05,
|
|
"loss_nats": 1.0434720516204834,
|
|
"step": 7810,
|
|
"unclipped_grad_norm": 1.2882802486419678
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5431284374439214,
|
|
"learning_rate": 3.0373526114452946e-05,
|
|
"loss_nats": 1.069615125656128,
|
|
"step": 7820,
|
|
"unclipped_grad_norm": 1.3333698511123657
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5271158207526887,
|
|
"learning_rate": 3.0333192677869803e-05,
|
|
"loss_nats": 1.058516025543213,
|
|
"step": 7830,
|
|
"unclipped_grad_norm": 1.229974389076233
|
|
},
|
|
{
|
|
"bits_per_byte": 1.501238623173267,
|
|
"learning_rate": 3.029516032546544e-05,
|
|
"loss_nats": 1.0405793190002441,
|
|
"step": 7840,
|
|
"unclipped_grad_norm": 1.2474827766418457
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4945333635814277,
|
|
"learning_rate": 3.02594297071078e-05,
|
|
"loss_nats": 1.0359315872192383,
|
|
"step": 7850,
|
|
"unclipped_grad_norm": 1.2343677282333374
|
|
},
|
|
{
|
|
"bits_per_byte": 1.534035198744852,
|
|
"learning_rate": 3.0226001433334603e-05,
|
|
"loss_nats": 1.0633121728897095,
|
|
"step": 7860,
|
|
"unclipped_grad_norm": 1.256583571434021
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5241123157380205,
|
|
"learning_rate": 3.019487607534278e-05,
|
|
"loss_nats": 1.056434154510498,
|
|
"step": 7870,
|
|
"unclipped_grad_norm": 1.3308851718902588
|
|
},
|
|
{
|
|
"bits_per_byte": 1.498756569556015,
|
|
"learning_rate": 3.016605416497886e-05,
|
|
"loss_nats": 1.0388588905334473,
|
|
"step": 7880,
|
|
"unclipped_grad_norm": 1.301889181137085
|
|
},
|
|
{
|
|
"bits_per_byte": 1.516417124007815,
|
|
"learning_rate": 3.0139536194729734e-05,
|
|
"loss_nats": 1.051100254058838,
|
|
"step": 7890,
|
|
"unclipped_grad_norm": 1.2672597169876099
|
|
},
|
|
{
|
|
"bits_per_byte": 1.4738032627770283,
|
|
"learning_rate": 3.0115322617714362e-05,
|
|
"loss_nats": 1.0215625762939453,
|
|
"step": 7900,
|
|
"unclipped_grad_norm": 1.2230888605117798
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5374244608454053,
|
|
"learning_rate": 3.0093413847675965e-05,
|
|
"loss_nats": 1.0656614303588867,
|
|
"step": 7910,
|
|
"unclipped_grad_norm": 1.3816039562225342
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5497416863175488,
|
|
"learning_rate": 3.0073810258974985e-05,
|
|
"loss_nats": 1.0741990804672241,
|
|
"step": 7920,
|
|
"unclipped_grad_norm": 1.2244949340820312
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5130077399371107,
|
|
"learning_rate": 3.0056512186582658e-05,
|
|
"loss_nats": 1.0487370491027832,
|
|
"step": 7930,
|
|
"unclipped_grad_norm": 1.2602943181991577
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5360826522033737,
|
|
"learning_rate": 3.0041519926075316e-05,
|
|
"loss_nats": 1.0647313594818115,
|
|
"step": 7940,
|
|
"unclipped_grad_norm": 1.278623342514038
|
|
},
|
|
{
|
|
"bits_per_byte": 1.479992574416279,
|
|
"learning_rate": 3.0028833733629313e-05,
|
|
"loss_nats": 1.0258526802062988,
|
|
"step": 7950,
|
|
"unclipped_grad_norm": 1.2290197610855103
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5513340736818877,
|
|
"learning_rate": 3.0018453826016686e-05,
|
|
"loss_nats": 1.0753028392791748,
|
|
"step": 7960,
|
|
"unclipped_grad_norm": 1.3293349742889404
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5048769161522697,
|
|
"learning_rate": 3.001038038060143e-05,
|
|
"loss_nats": 1.043101191520691,
|
|
"step": 7970,
|
|
"unclipped_grad_norm": 1.2828478813171387
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5449772509406965,
|
|
"learning_rate": 3.0004613535336433e-05,
|
|
"loss_nats": 1.0708966255187988,
|
|
"step": 7980,
|
|
"unclipped_grad_norm": 1.325577974319458
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5031080745881458,
|
|
"learning_rate": 3.000115338876117e-05,
|
|
"loss_nats": 1.0418751239776611,
|
|
"step": 7990,
|
|
"unclipped_grad_norm": 1.3532280921936035
|
|
},
|
|
{
|
|
"bits_per_byte": 1.5250520289423348,
|
|
"learning_rate": 3e-05,
|
|
"loss_nats": 1.0570855140686035,
|
|
"step": 8000,
|
|
"unclipped_grad_norm": 1.3144081830978394
|
|
}
|
|
]
|
|
}
|