wuyang/llm-atlas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
d81aacfc68d71c372fb9527fc10e589d0eae4c4d
llm-atlas/experiments/k3/attnres_gradient/results/raw/formal-depth-32-baseline-seed-2026073003.json
T
wuyang 26fc824409 research: audit AttnRes gradient scale study
2026-07-30 09:49:53 +08:00

8615 lines
250 KiB
JSON
Raw Blame History

{
"architecture": "baseline",
"batch_size": 32,
"canonical_sha256_without_self": "5f1f6749be2f106aaf597f0478348ce6e226aa739e557d119b2388ec4460aa16",
"depth": 32,
"diagnostics": [
{
"activation_grad_rms_by_block": [
0.0002897916710935533,
0.0002662435290403664,
0.00024211003619711846,
0.00022891115804668516,
0.00021424151782412082,
0.00020374931045807898,
0.00019545880786608905,
0.00018862746946979314,
0.00018024678865913302,
0.00017406532424502075,
0.0001698832056717947,
0.00016633344057481736,
0.00016165875422302634,
0.0001582534605404362,
0.00015452370280399919,
0.0001515774056315422,
0.00014964381989557296,
0.00014728632231708616,
0.00014459768135566264,
0.00014173447561915964,
0.00013920740457251668,
0.00013723202573601156,
0.00013503432273864746,
0.0001331205276073888,
0.00013141198724042624,
0.00013009003305342048,
0.0001286487968172878,
0.00012732617324218154,
0.00012632299331016839,
0.00012548104859888554,
0.00012477955897338688,
0.00012385686568450183
],
"activation_grad_statistics": {
"first_quartile_mean": 0.00022864168749947567,
"first_to_last_ratio": 1.796936959436678,
"imbalance_abs_log_ratio": 0.5860835261825511,
"last_quartile_mean": 0.00012723968211503234,
"mean": 0.00016535780059712124,
"normalized": [
1.7525128542293782,
1.6101056501655084,
1.4641585417974736,
1.3843384298779209,
1.2956238958819981,
1.2321723542664615,
1.1820356049746095,
1.1407231396924917,
1.0900410383317047,
1.052658680851196,
1.0273673516358577,
1.0059001751001344,
0.9776300461137163,
0.9570365593214796,
0.9344808787127115,
0.9166631697094613,
0.9049698251621409,
0.8907128770776013,
0.874453341986335,
0.8571381277892198,
0.8418556854882369,
0.8299095974937676,
0.816619005883163,
0.8050453448623477,
0.7947129604160569,
0.786718452855893,
0.7780025880407573,
0.7700040323613145,
0.7639373096038118,
0.7588456555769529,
0.7546034025779078,
0.7490234221623898
],
"population_cv": 0.25804396357350073
},
"activation_output_rms_by_block": [
0.028274519369006157,
0.02845057100057602,
0.02867918275296688,
0.02882254309952259,
0.0290658138692379,
0.029274335131049156,
0.029572302475571632,
0.02980491891503334,
0.03008633479475975,
0.03044024296104908,
0.030671589076519012,
0.030981140211224556,
0.0313686840236187,
0.031716227531433105,
0.03204340860247612,
0.03237249702215195,
0.032637808471918106,
0.03308072313666344,
0.03343126177787781,
0.033881161361932755,
0.03435666114091873,
0.03482793644070625,
0.03533180430531502,
0.03546451777219772,
0.036012887954711914,
0.03628316894173622,
0.03689942881464958,
0.03742557764053345,
0.03777860850095749,
0.03856590390205383,
0.038892220705747604,
0.039198413491249084
],
"activation_output_statistics": {
"first_quartile_mean": 0.02899302332662046,
"first_to_last_ratio": 0.7704348189669382,
"imbalance_abs_log_ratio": 0.2608002235603661,
"last_quartile_mean": 0.0376320262439549,
"mean": 0.032990387349855155,
"normalized": [
0.8570532703712039,
0.8623897227657417,
0.8693193701798957,
0.8736648889225345,
0.8810388784163674,
0.887359545694382,
0.8963914901017818,
0.9034425270294439,
0.9119727657545023,
0.9227003805150147,
0.9297129115597873,
0.9390959821925394,
0.9508431559460433,
0.961377846070436,
0.9712953128638184,
0.9812705949417742,
0.9893126784418129,
1.002738245715344,
1.0133637239037931,
1.0270010170729782,
1.0414142997647942,
1.0556995306349188,
1.070972702764284,
1.0749954947817073,
1.0916176149374623,
1.099810334354729,
1.1184903172957619,
1.1344388668021435,
1.1451398887901616,
1.1690042767025335,
1.1788955459450936,
1.1881768187672155
],
"population_cv": 0.10167083190861884
},
"bits_per_byte": 8.0691508062416,
"branch_output_rms_by_sublayer": [
0.002122601494193077,
0.002755566267296672,
0.002245423151180148,
0.0027348215226083994,
0.0024954702239483595,
0.002720654010772705,
0.0023770928382873535,
0.002733660163357854,
0.0023630354553461075,
0.0027867795433849096,
0.0024109049700200558,
0.0027537678834050894,
0.002701795194298029,
0.0027542903553694487,
0.0029824089724570513,
0.002726059639826417,
0.0029706284403800964,
0.0027311695739626884,
0.003079906338825822,
0.0027289879508316517,
0.0028494365978986025,
0.0027595392893999815,
0.0032860173378139734,
0.002710360335186124,
0.0035252906382083893,
0.0027479089330881834,
0.003967344760894775,
0.002725270576775074,
0.0035556324291974306,
0.002750981133431196,
0.004272372927516699,
0.0027044012676924467,
0.004134721122682095,
0.0027556782588362694,
0.0041320133022964,
0.0027166390791535378,
0.0040069944225251675,
0.0026826481334865093,
0.004499548580497503,
0.002729416359215975,
0.0049178507179021835,
0.002681323792785406,
0.004439458716660738,
0.002806412521749735,
0.004581923596560955,
0.0027696131728589535,
0.004871132783591747,
0.0027697959449142218,
0.0049627418629825115,
0.0028166540432721376,
0.0051520452834665775,
0.0027135310228914022,
0.005508258938789368,
0.002744151744991541,
0.005101518705487251,
0.002700615907087922,
0.005637416150420904,
0.0027939171995967627,
0.005635067820549011,
0.0027390308678150177,
0.00545669998973608,
0.002716719638556242,
0.005571703426539898,
0.002707480685785413
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.006421208939522194,
0.005746349410576503,
0.005404164999391892,
0.005101115831054325,
0.00485388847333966,
0.0045855144167447825,
0.004662942770173713,
0.005059104716899598,
0.00461737020280362,
0.004446534400320367,
0.004114212128857249,
0.004678028852169936,
0.004404060169276539,
0.004656150674369456,
0.004513829588134647,
0.004839518675960965,
0.004348161380019098,
0.0047201153472655015,
0.004169246752947831,
0.004575240937826365,
0.0047925402340120975,
0.004508705150890575,
0.004155367099556682,
0.004478773084339979,
0.004234175381969994,
0.004428191743895327,
0.004153362780335295,
0.00410615213012631,
0.004175626130102292,
0.004125604583408443,
0.0038633837013440408,
0.003947405979968418
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.005229286194712833,
"first_to_last_ratio": 1.266404707857162,
"imbalance_abs_log_ratio": 0.236181947089728,
"last_quartile_mean": 0.004129237803893765,
"mean": 0.004590188958362615,
"normalized": [
1.3988986069568536,
1.2518764396632391,
1.1773295279154767,
1.1113084618795224,
1.057448509716931,
0.9989816232707989,
1.0158498511653973,
1.1021560904769923,
1.0059215959707901,
0.9687039990411438,
0.8963056131625672,
1.0191364439686714,
0.9594507348663767,
1.0143701526462583,
0.983364656461723,
1.0543179637831923,
0.9472728507390573,
1.0283052375580706,
0.9082952337620236,
0.9967434847079623,
1.044083430439357,
0.9822482672911342,
0.9052714686148694,
0.9757273883421174,
0.9224403222564466,
0.9647079421050512,
0.9048348157364001,
0.894549694440255,
0.9096850190654888,
0.8987875272307099,
0.8416611464993293,
0.859965900265795
],
"population_cv": 0.11345956482154346
},
"depth_weights": [],
"layer_input_rms_by_sublayer": [
0.028097067028284073,
0.028159458190202713,
0.028274519369006157,
0.02833542972803116,
0.02845057100057602,
0.02851351536810398,
0.02867918275296688,
0.028714356943964958,
0.02882254309952259,
0.028916271403431892,
0.0290658138692379,
0.02915445901453495,
0.029274335131049156,
0.02944180741906166,
0.029572302475571632,
0.02970401756465435,
0.02980491891503334,
0.02993646264076233,
0.03008633479475975,
0.030266879126429558,
0.03044024296104908,
0.03057776764035225,
0.030671589076519012,
0.030867453664541245,
0.030981140211224556,
0.03126408904790878,
0.0313686840236187,
0.031607914716005325,
0.031716227531433105,
0.03191319853067398,
0.03204340860247612,
0.032251372933387756,
0.03237249702215195,
0.03250224143266678,
0.032637808471918106,
0.03294040635228157,
0.03308072313666344,
0.0333157554268837,
0.03343126177787781,
0.03377276286482811,
0.033881161361932755,
0.034277379512786865,
0.03435666114091873,
0.03472030907869339,
0.03482793644070625,
0.0352315790951252,
0.03533180430531502,
0.03532300516963005,
0.03546451777219772,
0.03583687171339989,
0.036012887954711914,
0.03619367629289627,
0.03628316894173622,
0.03677845373749733,
0.03689942881464958,
0.03733044117689133,
0.03742557764053345,
0.0377187617123127,
0.03777860850095749,
0.03849533945322037,
0.03856590390205383,
0.03877026215195656,
0.038892220705747604,
0.03908585384488106
],
"loss_nats": 5.593109130859375,
"loss_scale": 1.0,
"output_weights": null,
"step": 0,
"stream_state_rms_by_sublayer": [
0.028159458190202713,
0.028274519369006157,
0.02833542972803116,
0.02845057100057602,
0.02851351536810398,
0.02867918275296688,
0.028714356943964958,
0.02882254309952259,
0.028916271403431892,
0.0290658138692379,
0.02915445901453495,
0.029274335131049156,
0.02944180741906166,
0.029572302475571632,
0.02970401756465435,
0.02980491891503334,
0.02993646264076233,
0.03008633479475975,
0.030266879126429558,
0.03044024296104908,
0.03057776764035225,
0.030671589076519012,
0.030867453664541245,
0.030981140211224556,
0.03126408904790878,
0.0313686840236187,
0.031607914716005325,
0.031716227531433105,
0.03191319853067398,
0.03204340860247612,
0.032251372933387756,
0.03237249702215195,
0.03250224143266678,
0.032637808471918106,
0.03294040635228157,
0.03308072313666344,
0.0333157554268837,
0.03343126177787781,
0.03377276286482811,
0.033881161361932755,
0.034277379512786865,
0.03435666114091873,
0.03472030907869339,
0.03482793644070625,
0.0352315790951252,
0.03533180430531502,
0.03532300516963005,
0.03546451777219772,
0.03583687171339989,
0.036012887954711914,
0.03619367629289627,
0.03628316894173622,
0.03677845373749733,
0.03689942881464958,
0.03733044117689133,
0.03742557764053345,
0.0377187617123127,
0.03777860850095749,
0.03849533945322037,
0.03856590390205383,
0.03877026215195656,
0.038892220705747604,
0.03908585384488106,
0.039198413491249084
]
},
{
"activation_grad_rms_by_block": [
7.023746729828417e-05,
6.667003617621958e-05,
6.32333685643971e-05,
6.012299854774028e-05,
5.672227416653186e-05,
5.335300011211075e-05,
5.0173275667475536e-05,
4.719031130662188e-05,
4.408728636917658e-05,
4.096491466043517e-05,
3.824968007393181e-05,
3.570610351744108e-05,
3.324189310660586e-05,
3.1057134037837386e-05,
2.914190736191813e-05,
2.7255719032837078e-05,
2.56736693700077e-05,
2.4158533051377162e-05,
2.2910870029591024e-05,
2.187793143093586e-05,
2.0998755644541234e-05,
2.0234831026755273e-05,
1.963501563295722e-05,
1.908456215460319e-05,
1.8524928236729465e-05,
1.8092870959662832e-05,
1.7702994227875024e-05,
1.74019587575458e-05,
1.7127178580267355e-05,
1.6885946024558507e-05,
1.666023126745131e-05,
1.649606565479189e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 5.8462841479922645e-05,
"first_to_last_ratio": 3.3673800283354014,
"imbalance_abs_log_ratio": 1.2141350023382471,
"last_quartile_mean": 1.7361521713610273e-05,
"mean": 3.315230350153797e-05,
"normalized": [
2.1186300763392136,
2.011022738529366,
1.9073597272498315,
1.8135390967614395,
1.710960270495245,
1.6093301061157226,
1.5134174813870156,
1.4234398917237432,
1.3298408168570044,
1.235658169530656,
1.1537563316575383,
1.0770323551057208,
1.0027023644092703,
0.9368016927208878,
0.8790311466763148,
0.8221365079977823,
0.7744158522443748,
0.7287135583280486,
0.691079279861448,
0.6599219094963015,
0.6334026123876152,
0.610359730382432,
0.5922670088986827,
0.5756632311754095,
0.5587825363589011,
0.5457500399277982,
0.5339898697251538,
0.5249094910324558,
0.516621072181992,
0.5093445776331997,
0.5025361591142052,
0.49758429769523016
],
"population_cv": 0.5033867184228629
},
"activation_output_rms_by_block": [
0.029364673420786858,
0.03058232180774212,
0.03253435716032982,
0.035004038363695145,
0.03789948299527168,
0.041081421077251434,
0.04464675113558769,
0.04816310852766037,
0.05221380665898323,
0.056122854351997375,
0.06031070277094841,
0.06517986208200455,
0.07041798532009125,
0.07559357583522797,
0.08127844333648682,
0.08752159774303436,
0.09348469227552414,
0.10084687173366547,
0.10794128477573395,
0.11579020321369171,
0.12420836836099625,
0.13254153728485107,
0.14211823046207428,
0.151936337351799,
0.16282601654529572,
0.17539408802986145,
0.19021794199943542,
0.2045798897743225,
0.2205442190170288,
0.23628318309783936,
0.2528626322746277,
0.27021101117134094
],
"activation_output_statistics": {
"first_quartile_mean": 0.03740951931104064,
"first_to_last_ratio": 0.17471705179812935,
"imbalance_abs_log_ratio": 1.74458746043398,
"last_quartile_mean": 0.214114872738719,
"mean": 0.11030317156109959,
"normalized": [
0.2662178521722837,
0.2772569580268309,
0.29495395916434053,
0.31734389744455865,
0.34359377394944846,
0.37244097785978386,
0.40476398369793726,
0.43664300714129206,
0.47336632229165543,
0.5088054455524842,
0.5467721545752731,
0.590915575314168,
0.6384039944045038,
0.685325496677626,
0.736864064615447,
0.793464018344695,
0.8475249709727586,
0.9142699190458359,
0.9785873175545334,
1.0497450034748341,
1.126063435920283,
1.2016112991949022,
1.2884328569224464,
1.3774430526472923,
1.476168039783901,
1.5901092012817277,
1.7245011118657556,
1.8547054167069055,
1.9994367876798904,
2.142125015570893,
2.292433016167273,
2.4497120739784397
],
"population_cv": 0.6367597776091559
},
"bits_per_byte": 5.184434896455056,
"branch_output_rms_by_sublayer": [
0.0026808311231434345,
0.00477010989561677,
0.0025234855711460114,
0.004875619895756245,
0.0027965111657977104,
0.005397462286055088,
0.002715951530262828,
0.00578050222247839,
0.0028346737381070852,
0.005865047685801983,
0.0025751085486263037,
0.0059114922769367695,
0.0034423002507537603,
0.006014656741172075,
0.0034009802620857954,
0.005813058000057936,
0.0035280236043035984,
0.006192575208842754,
0.003798100398853421,
0.006026907358318567,
0.0032950215972959995,
0.006300882901996374,
0.004497972317039967,
0.006683246698230505,
0.004819076508283615,
0.006906723603606224,
0.004967817571014166,
0.006679704412817955,
0.005188894923776388,
0.006651155184954405,
0.005964669398963451,
0.006957803387194872,
0.005442580208182335,
0.006974066607654095,
0.00629971781745553,
0.007549752946943045,
0.007111458573490381,
0.007369632367044687,
0.008486796170473099,
0.0074090128764510155,
0.008528640493750572,
0.007484533358365297,
0.007537186611443758,
0.007801020983606577,
0.00965037476271391,
0.007585691753774881,
0.009028015658259392,
0.007420822512358427,
0.010029759258031845,
0.00862929131835699,
0.01243194192647934,
0.0086217625066638,
0.0149146169424057,
0.00852242112159729,
0.014407102949917316,
0.008064002729952335,
0.01547334622591734,
0.008129187859594822,
0.015585782937705517,
0.008083954453468323,
0.014871877618134022,
0.009126133285462856,
0.016274450346827507,
0.008773879148066044
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.00026099601887828554,
0.00022299818257146475,
0.00021593606328411676,
0.0001980902028289896,
0.00019039206376599677,
0.0001764249768268048,
0.00019262760932023515,
0.00017741019012756782,
0.0001866267032514166,
0.00018194420611625288,
0.00016335812718032435,
0.00018628060025801724,
0.00018197064718745634,
0.00018231380553041538,
0.00016812738233283717,
0.0001758928024359223,
0.00015172497688937343,
0.0001676233054310719,
0.0001547068922126687,
0.0001733627635453279,
0.0001619137110647745,
0.00014190991701657472,
0.0001580141865100647,
0.00014951069781870242,
0.00014861801255322757,
0.00015877260520542335,
0.00016243531213177502,
0.00014972906733594748,
0.0001506252547372875,
0.00014728724932094392,
0.00014196626632039671,
0.00013988030321371655
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.00020435941345043265,
"first_to_last_ratio": 1.3631752910956885,
"imbalance_abs_log_ratio": 0.30981675126526836,
"last_quartile_mean": 0.00014991425885233975,
"mean": 0.00017248344072510563,
"normalized": [
1.513165656836857,
1.2928671971871588,
1.251923444803397,
1.1484592491675452,
1.1038280716433115,
1.0228516783125923,
1.1167890002103689,
1.028563608087539,
1.0819977991327974,
1.0548502821567973,
0.9470945529239257,
1.0799912123442663,
1.0550035784447904,
1.0569930931571387,
0.9747450632132802,
1.0197663131978583,
0.8796495260735443,
0.9718225977311089,
0.8969376513031868,
1.0050980129833083,
0.9387203222761739,
0.8227451656808188,
0.9161122125450802,
0.8668118933050746,
0.8616364094341472,
0.9205092647616311,
0.941744386875116,
0.8680779250837024,
0.8732737131406461,
0.853921099334295,
0.823071859672921,
0.8109781589796199
],
"population_cv": 0.15033028761203962
},
"depth_weights": [],
"layer_input_rms_by_sublayer": [
0.028610490262508392,
0.028607122600078583,
0.029364673420786858,
0.029366537928581238,
0.03058232180774212,
0.03066444955766201,
0.03253435716032982,
0.03263945132493973,
0.035004038363695145,
0.035203028470277786,
0.03789948299527168,
0.03799305483698845,
0.041081421077251434,
0.04133254662156105,
0.04464675113558769,
0.04496326297521591,
0.04816310852766037,
0.048516932874917984,
0.05221380665898323,
0.05251964181661606,
0.056122854351997375,
0.056522201746702194,
0.06031070277094841,
0.060862768441438675,
0.06517986208200455,
0.06596556305885315,
0.07041798532009125,
0.0712207704782486,
0.07559357583522797,
0.07684765011072159,
0.08127844333648682,
0.08279602974653244,
0.08752159774303436,
0.08871174603700638,
0.09348469227552414,
0.09565582871437073,
0.10084687173366547,
0.10279740393161774,
0.10794128477573395,
0.11048463732004166,
0.11579020321369171,
0.11891037970781326,
0.12420836836099625,
0.1270110011100769,
0.13254153728485107,
0.13657133281230927,
0.14211823046207428,
0.14646445214748383,
0.151936337351799,
0.15653157234191895,
0.16282601654529572,
0.16900379955768585,
0.17539408802986145,
0.18384796380996704,
0.19021794199943542,
0.1985616832971573,
0.2045798897743225,
0.21447749435901642,
0.2205442190170288,
0.23017354309558868,
0.23628318309783936,
0.24575340747833252,
0.2528626322746277,
0.26334208250045776
],
"loss_nats": 3.593576431274414,
"loss_scale": 1.0,
"output_weights": null,
"step": 100,
"stream_state_rms_by_sublayer": [
0.028607122600078583,
0.029364673420786858,
0.029366537928581238,
0.03058232180774212,
0.03066444955766201,
0.03253435716032982,
0.03263945132493973,
0.035004038363695145,
0.035203028470277786,
0.03789948299527168,
0.03799305483698845,
0.041081421077251434,
0.04133254662156105,
0.04464675113558769,
0.04496326297521591,
0.04816310852766037,
0.048516932874917984,
0.05221380665898323,
0.05251964181661606,
0.056122854351997375,
0.056522201746702194,
0.06031070277094841,
0.060862768441438675,
0.06517986208200455,
0.06596556305885315,
0.07041798532009125,
0.0712207704782486,
0.07559357583522797,
0.07684765011072159,
0.08127844333648682,
0.08279602974653244,
0.08752159774303436,
0.08871174603700638,
0.09348469227552414,
0.09565582871437073,
0.10084687173366547,
0.10279740393161774,
0.10794128477573395,
0.11048463732004166,
0.11579020321369171,
0.11891037970781326,
0.12420836836099625,
0.1270110011100769,
0.13254153728485107,
0.13657133281230927,
0.14211823046207428,
0.14646445214748383,
0.151936337351799,
0.15653157234191895,
0.16282601654529572,
0.16900379955768585,
0.17539408802986145,
0.18384796380996704,
0.19021794199943542,
0.1985616832971573,
0.2045798897743225,
0.21447749435901642,
0.2205442190170288,
0.23017354309558868,
0.23628318309783936,
0.24575340747833252,
0.2528626322746277,
0.26334208250045776,
0.27021101117134094
]
},
{
"activation_grad_rms_by_block": [
0.0001506932603660971,
0.00015340163372457027,
0.00015237691695801914,
0.000148982901009731,
0.0001392244448652491,
0.00012851000064983964,
0.00011962141434196383,
0.0001119004882639274,
0.00010405662760604173,
9.720238449517637e-05,
9.135912114288658e-05,
8.468296437058598e-05,
7.889774860814214e-05,
7.427640230162069e-05,
6.881110311951488e-05,
6.43495368422009e-05,
5.969387711957097e-05,
5.503925422090106e-05,
5.034419518779032e-05,
4.6255379857029766e-05,
4.248250479577109e-05,
3.87999571103137e-05,
3.563905556802638e-05,
3.2581570849288255e-05,
3.028466016985476e-05,
2.807797682180535e-05,
2.6328352760174312e-05,
2.5215304049197584e-05,
2.4513139578630216e-05,
2.4017937903408892e-05,
2.3832226361264475e-05,
2.3948901798576117e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 0.00013808888252242468,
"first_to_last_ratio": 5.356993010635397,
"imbalance_abs_log_ratio": 1.678402812181787,
"last_quartile_mean": 2.5777312430363963e-05,
"mean": 7.298128883803656e-05,
"normalized": [
2.0648204870774824,
2.1019310040551114,
2.087890189171378,
2.041384899907163,
1.907673145841839,
1.7608623072556988,
1.6390696334156716,
1.533276406124618,
1.425798711735127,
1.3318808977310936,
1.2518156721736573,
1.1603380225103768,
1.0810681732852876,
1.0177458288857888,
0.9428595221471582,
0.881726506433839,
0.8179339947263247,
0.7541556897281809,
0.6898233230645799,
0.6337977938383883,
0.5821013231220159,
0.5316425309563984,
0.488331408439747,
0.44643731794864266,
0.4149647211227507,
0.3847284320247795,
0.3607548342781861,
0.3455036825282238,
0.3358825250816124,
0.32909720129375364,
0.3265525553289974,
0.3281512587661287
],
"population_cv": 0.6061250608697893
},
"activation_output_rms_by_block": [
0.048818644136190414,
0.05689374357461929,
0.06500593572854996,
0.07087047398090363,
0.07611831277608871,
0.0806790366768837,
0.08478696644306183,
0.08861920237541199,
0.0918315127491951,
0.09433750063180923,
0.09700070321559906,
0.09972338378429413,
0.10201811045408249,
0.10424874722957611,
0.10649008303880692,
0.10806933790445328,
0.11109450459480286,
0.11401215940713882,
0.11832662671804428,
0.12293888628482819,
0.12948566675186157,
0.1387748122215271,
0.14864373207092285,
0.16069285571575165,
0.1741981953382492,
0.18964411318302155,
0.20650440454483032,
0.22468066215515137,
0.24238750338554382,
0.2638821303844452,
0.2911032736301422,
0.32429978251457214
],
"activation_output_statistics": {
"first_quartile_mean": 0.07147403946146369,
"first_to_last_ratio": 0.2983212272448851,
"imbalance_abs_log_ratio": 1.2095844292661684,
"last_quartile_mean": 0.23958750814199448,
"mean": 0.13550565636251122,
"normalized": [
0.3602701573253034,
0.41986249948426085,
0.4797285771941724,
0.523007495652488,
0.5617353165867361,
0.5953923905659501,
0.6257079499045831,
0.6539889533344176,
0.67769505136762,
0.6961886548811884,
0.7158424660598531,
0.7359352108336301,
0.7528697560872202,
0.7693313329348029,
0.7858718661449788,
0.7975263970925391,
0.8198514185828341,
0.8413830275994386,
0.8732227856340641,
0.907260180755379,
0.9555738869339544,
1.024125604397429,
1.0969559209636535,
1.185875630799193,
1.285541873412471,
1.3995291287005507,
1.523954129209041,
1.6580906523494112,
1.7887629925727762,
1.9473883044298541,
2.148273964677672,
2.393256423532534
],
"population_cv": 0.5081939166956976
},
"bits_per_byte": 3.1154419867739596,
"branch_output_rms_by_sublayer": [
0.010533316060900688,
0.01737932860851288,
0.0117434561252594,
0.012238939292728901,
0.01211429014801979,
0.011732303537428379,
0.014587659388780594,
0.009741203859448433,
0.012628368102014065,
0.009234351105988026,
0.014413262717425823,
0.00937508512288332,
0.015033986419439316,
0.00873780157417059,
0.016592979431152344,
0.008810308761894703,
0.018142949789762497,
0.00861456897109747,
0.01592753641307354,
0.008246068842709064,
0.013729470781981945,
0.008380995132029057,
0.017536180093884468,
0.008834624662995338,
0.021067099645733833,
0.009005429223179817,
0.022672412917017937,
0.009119905531406403,
0.019366111606359482,
0.0090357456356287,
0.0166512168943882,
0.009071468375623226,
0.014951958321034908,
0.01051315013319254,
0.017114169895648956,
0.010944797657430172,
0.01693827658891678,
0.012414379045367241,
0.01647246815264225,
0.013282178901135921,
0.01672390289604664,
0.014548475854098797,
0.017381425946950912,
0.017058325931429863,
0.02400100603699684,
0.018193069845438004,
0.017901180312037468,
0.02045683190226555,
0.01391419768333435,
0.021929442882537842,
0.014953838661313057,
0.02380906604230404,
0.010647366754710674,
0.02548205293715,
0.005664720665663481,
0.026819046586751938,
0.008504491299390793,
0.024410424754023552,
0.008407330140471458,
0.02826099656522274,
0.01054995134472847,
0.033158741891384125,
0.02235519140958786,
0.03220592811703682
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.0005931392614171548,
0.0004012275990867258,
0.00038098222807989436,
0.0003649111519840816,
0.00034656151495199636,
0.00031005860528648927,
0.00027821812913176474,
0.0002561842642921938,
0.0002305368063696604,
0.00020943363497294808,
0.0001942700943631853,
0.0001900904574854747,
0.00018678033884870243,
0.00017357948080148642,
0.00015815775362626997,
0.00014490835987546657,
0.0001375100278294081,
0.0001295570650502436,
0.00012744106699623042,
0.00011906627060222178,
0.00011368520556157287,
0.0001091753996736031,
0.00010941555658600093,
9.636520203422524e-05,
9.391483800802352e-05,
9.101856383525677e-05,
7.998410113295477e-05,
7.102545922046693e-05,
6.763159533414917e-05,
6.887797791066819e-05,
7.001610738319118e-05,
7.664349539138475e-05
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.00036641034427878756,
"first_to_last_ratio": 4.73465560322638,
"imbalance_abs_log_ratio": 1.5549089897216843,
"last_quartile_mean": 7.738901727701191e-05,
"mean": 0.00018688648791009675,
"normalized": [
3.173794253667441,
2.146905341170195,
2.038575567128047,
1.9525817840807467,
1.8543957823141959,
1.6590744935805388,
1.4887011481836172,
1.3708014268823612,
1.2335659412710565,
1.1206462131906392,
1.0395085088047697,
1.017143933792149,
0.9994320131830752,
0.9287963123636216,
0.8462770925544549,
0.7753816848715992,
0.7357943817509076,
0.6932393374130293,
0.681916966932018,
0.6371047577259816,
0.6083115308810451,
0.5841802737826761,
0.5854653153877886,
0.5156349349375247,
0.5025234250921448,
0.4870259206703161,
0.4279822582541749,
0.38004598414110224,
0.3618859559642637,
0.3685551517443171,
0.3746451023086966,
0.4101072059755049
],
"population_cv": 0.6522632271008562
},
"depth_weights": [],
"layer_input_rms_by_sublayer": [
0.04385505989193916,
0.04404326155781746,
0.048818644136190414,
0.05170967057347298,
0.05689374357461929,
0.059983257204294205,
0.06500593572854996,
0.06660757958889008,
0.07087047398090363,
0.07236418128013611,
0.07611831277608871,
0.07718922197818756,
0.0806790366768837,
0.08160495012998581,
0.08478696644306183,
0.08585704118013382,
0.08861920237541199,
0.08891960233449936,
0.0918315127491951,
0.0918683186173439,
0.09433750063180923,
0.09460321068763733,
0.09700070321559906,
0.09746499359607697,
0.09972338378429413,
0.0994088351726532,
0.10201811045408249,
0.10191889107227325,
0.10424874722957611,
0.10386203229427338,
0.10649008303880692,
0.10544375330209732,
0.10806933790445328,
0.10778804868459702,
0.11109450459480286,
0.11027765274047852,
0.11401215940713882,
0.11356210708618164,
0.11832662671804428,
0.11750734597444534,
0.12293888628482819,
0.1225726455450058,
0.12948566675186157,
0.12988287210464478,
0.1387748122215271,
0.13870327174663544,
0.14864373207092285,
0.14834089577198029,
0.16069285571575165,
0.16013014316558838,
0.1741981953382492,
0.1741078794002533,
0.18964411318302155,
0.18942901492118835,
0.20650440454483032,
0.20662079751491547,
0.22468066215515137,
0.2257053554058075,
0.24238750338554382,
0.24448354542255402,
0.2638821303844452,
0.2678608000278473,
0.2911032736301422,
0.30181384086608887
],
"loss_nats": 2.1594598293304443,
"loss_scale": 1.0,
"output_weights": null,
"step": 500,
"stream_state_rms_by_sublayer": [
0.04404326155781746,
0.048818644136190414,
0.05170967057347298,
0.05689374357461929,
0.059983257204294205,
0.06500593572854996,
0.06660757958889008,
0.07087047398090363,
0.07236418128013611,
0.07611831277608871,
0.07718922197818756,
0.0806790366768837,
0.08160495012998581,
0.08478696644306183,
0.08585704118013382,
0.08861920237541199,
0.08891960233449936,
0.0918315127491951,
0.0918683186173439,
0.09433750063180923,
0.09460321068763733,
0.09700070321559906,
0.09746499359607697,
0.09972338378429413,
0.0994088351726532,
0.10201811045408249,
0.10191889107227325,
0.10424874722957611,
0.10386203229427338,
0.10649008303880692,
0.10544375330209732,
0.10806933790445328,
0.10778804868459702,
0.11109450459480286,
0.11027765274047852,
0.11401215940713882,
0.11356210708618164,
0.11832662671804428,
0.11750734597444534,
0.12293888628482819,
0.1225726455450058,
0.12948566675186157,
0.12988287210464478,
0.1387748122215271,
0.13870327174663544,
0.14864373207092285,
0.14834089577198029,
0.16069285571575165,
0.16013014316558838,
0.1741981953382492,
0.1741078794002533,
0.18964411318302155,
0.18942901492118835,
0.20650440454483032,
0.20662079751491547,
0.22468066215515137,
0.2257053554058075,
0.24238750338554382,
0.24448354542255402,
0.2638821303844452,
0.2678608000278473,
0.2911032736301422,
0.30181384086608887,
0.32429978251457214
]
},
{
"activation_grad_rms_by_block": [
0.00010946426482405514,
0.00010930908320005983,
0.00012598594184964895,
0.00012082153261872008,
0.00011562183499336243,
0.00011443938274169341,
0.00011117216490674764,
0.00010893856233451515,
0.00010224300058325753,
9.949541708920151e-05,
9.518271690467373e-05,
9.17384895728901e-05,
8.801270450931042e-05,
8.554522355552763e-05,
8.286484808195382e-05,
8.027214062167332e-05,
7.707033364567906e-05,
7.422274211421609e-05,
7.329088111873716e-05,
7.053956505842507e-05,
6.641655636485666e-05,
6.314766505965963e-05,
5.928793325438164e-05,
5.485894507728517e-05,
5.013645932194777e-05,
4.561002060654573e-05,
4.094795440323651e-05,
3.615552486735396e-05,
3.135877341264859e-05,
2.6529152819421142e-05,
2.2670756152365357e-05,
2.0441382730496116e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 0.00011446909593360033,
"first_to_last_ratio": 3.3439937416939496,
"imbalance_abs_log_ratio": 1.2071658237176983,
"last_quartile_mean": 3.4231253039251897e-05,
"mean": 7.668099857482957e-05,
"normalized": [
1.427527899460436,
1.4255041696332367,
1.642987756956567,
1.5756384875558933,
1.5078290207779732,
1.4924086131979242,
1.4498006934307157,
1.4206721920582042,
1.3333551007878848,
1.297523753451233,
1.2412816561300926,
1.196365348364192,
1.1477772348442066,
1.115598716050207,
1.0806438312235815,
1.046832228499728,
1.0050773343864263,
0.9679417781940518,
0.955789341308797,
0.91990931742482,
0.866140994500009,
0.8235112509396527,
0.7731763325503017,
0.7154177188205337,
0.6538315913168741,
0.5948021211804766,
0.5340039274955088,
0.47150566032432917,
0.4089510308351939,
0.3459677535828191,
0.29565024678495777,
0.26657689793317285
],
"population_cv": 0.3996524250074872
},
"activation_output_rms_by_block": [
0.09914769232273102,
0.13720469176769257,
0.1984461098909378,
0.2206047922372818,
0.22331808507442474,
0.2258162647485733,
0.22567619383335114,
0.2251615822315216,
0.22496475279331207,
0.22249442338943481,
0.21478813886642456,
0.205490842461586,
0.19144563376903534,
0.188800647854805,
0.18874111771583557,
0.190367192029953,
0.19439123570919037,
0.198538675904274,
0.20280276238918304,
0.2084609866142273,
0.21558508276939392,
0.21985429525375366,
0.22511647641658783,
0.2311476618051529,
0.24068132042884827,
0.2547627091407776,
0.2762547731399536,
0.30550503730773926,
0.34050679206848145,
0.39206719398498535,
0.46342816948890686,
0.575099527835846
],
"activation_output_statistics": {
"first_quartile_mean": 0.19442192651331425,
"first_to_last_ratio": 0.5460704265504183,
"imbalance_abs_log_ratio": 0.6050073251991793,
"last_quartile_mean": 0.3560381904244423,
"mean": 0.2414584644138813,
"normalized": [
0.41062007316000754,
0.5682331000519887,
0.8218643747803495,
0.9136345365766327,
0.9248716362729685,
0.9352178450099977,
0.9346377414482436,
0.932506477990246,
0.9316913090596918,
0.9214604421904198,
0.8895448721907656,
0.851040128000468,
0.7928719096004871,
0.7819177029602238,
0.7816711589464782,
0.7884055441669947,
0.8050711172253067,
0.8222477368362661,
0.8399074469452479,
0.8633409771748842,
0.8928454146045669,
0.9105263540353832,
0.9323196723007322,
0.9572978208332561,
0.9967814589274413,
1.0550995168431603,
1.144108879390653,
1.2652488205345183,
1.410208554481745,
1.6237459098263265,
1.9192873217919157,
2.381774145842633
],
"population_cv": 0.36935148393532197
},
"bits_per_byte": 2.073311222061611,
"branch_output_rms_by_sublayer": [
0.01824937015771866,
0.06101369485259056,
0.05977629870176315,
0.0147639699280262,
0.08494830876588821,
0.003444893751293421,
0.03928658738732338,
0.004586480092257261,
0.025128526613116264,
0.007272258400917053,
0.024004649370908737,
0.006204464007169008,
0.019449269399046898,
0.007294370327144861,
0.02213231660425663,
0.010247048921883106,
0.041971735656261444,
0.012659681960940361,
0.02208700217306614,
0.013693951070308685,
0.029153602197766304,
0.01590428315103054,
0.02919897623360157,
0.017466455698013306,
0.036670926958322525,
0.021206999197602272,
0.02140921540558338,
0.021084947511553764,
0.023707622662186623,
0.022131068632006645,
0.02047598734498024,
0.022354252636432648,
0.022473404183983803,
0.02369093894958496,
0.02454194240272045,
0.02377566508948803,
0.020227128639817238,
0.025439493358135223,
0.02723041921854019,
0.027927791699767113,
0.03427847847342491,
0.028011852875351906,
0.03502028062939644,
0.030455492436885834,
0.029505647718906403,
0.032503895461559296,
0.033919259905815125,
0.03810396417975426,
0.036724433302879333,
0.04227631166577339,
0.033869534730911255,
0.04830062761902809,
0.021475639194250107,
0.057604189962148666,
0.010480014607310295,
0.06417365372180939,
0.010398799553513527,
0.06654922664165497,
0.012935257516801357,
0.0871838852763176,
0.010905168019235134,
0.11424379050731659,
0.02395492233335972,
0.16713054478168488
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.00035888588058483766,
0.00018018950310225553,
0.0002078431087578172,
0.0001449691382379752,
0.00012177099939920197,
0.00012306582379429542,
0.00012204348242118167,
0.00015411618529826362,
0.00023712554584861813,
0.0001667297606409996,
0.00020124714337423614,
0.00019657670547036952,
0.00021589641757185544,
0.00019691649893044806,
0.0002141921139315525,
0.00021492094770207205,
0.00022724548428173296,
0.00022961612017464663,
0.00022356249125846063,
0.00024496232741861765,
0.0002428330999302831,
0.0002472582804061272,
0.00023198586071939664,
0.00023918771709558322,
0.0002286007324212131,
0.00020791382420134994,
0.00018978221308207498,
0.00017514362262291224,
0.0001585770655414503,
0.0001491152323589239,
0.00013960034115182495,
0.00014044566543596396
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.00017661051519947854,
"first_to_last_ratio": 1.0170643451662862,
"imbalance_abs_log_ratio": 0.016920384648356147,
"last_quartile_mean": 0.00017364733710196415,
"mean": 0.0001978849791614544,
"normalized": [
1.813608501795493,
0.9105769617573595,
1.0503228170150194,
0.7325929378383735,
0.6153625197587583,
0.6219058380064613,
0.6167394965416064,
0.7788169973857398,
1.1982998752782916,
0.8425589519099616,
1.0169904973436037,
0.9933887165330652,
1.0910197352357174,
0.9951058426207471,
1.0824071379202214,
1.0860902561316592,
1.1483715704177997,
1.160351438232726,
1.1297597837178734,
1.2379025859196358,
1.227142661152444,
1.2495050481036718,
1.1723267814588358,
1.2087209353087376,
1.155220236472308,
1.0506801733127658,
0.959053152423619,
0.8850779041698386,
0.8013597909928638,
0.7535449784556955,
0.7054620403397319,
0.7097338364493765
],
"population_cv": 0.24613765810690832
},
"depth_weights": [],
"layer_input_rms_by_sublayer": [
0.0564301572740078,
0.061963923275470734,
0.09914769232273102,
0.12944035232067108,
0.13720469176769257,
0.1976913958787918,
0.1984461098909378,
0.21992529928684235,
0.2206047922372818,
0.22257371246814728,
0.22331808507442474,
0.22564378380775452,
0.2258162647485733,
0.2261907458305359,
0.22567619383335114,
0.22598126530647278,
0.2251615822315216,
0.22548764944076538,
0.22496475279331207,
0.22340621054172516,
0.22249442338943481,
0.21654370427131653,
0.21478813886642456,
0.20714853703975677,
0.205490842461586,
0.19201314449310303,
0.19144563376903534,
0.18731488287448883,
0.188800647854805,
0.18594272434711456,
0.18874111771583557,
0.18583883345127106,
0.190367192029953,
0.18933312594890594,
0.19439123570919037,
0.1937587559223175,
0.198538675904274,
0.19723950326442719,
0.20280276238918304,
0.2025505006313324,
0.2084609866142273,
0.2101445198059082,
0.21558508276939392,
0.21427761018276215,
0.21985429525375366,
0.21906179189682007,
0.22511647641658783,
0.22344982624053955,
0.2311476618051529,
0.2303154021501541,
0.24068132042884827,
0.23970884084701538,
0.2547627091407776,
0.25386181473731995,
0.2762547731399536,
0.2756711542606354,
0.30550503730773926,
0.30545076727867126,
0.34050679206848145,
0.34091928601264954,
0.39206719398498535,
0.3925974369049072,
0.46342816948890686,
0.46681874990463257
],
"loss_nats": 1.4371098279953003,
"loss_scale": 1.0,
"output_weights": null,
"step": 2000,
"stream_state_rms_by_sublayer": [
0.061963923275470734,
0.09914769232273102,
0.12944035232067108,
0.13720469176769257,
0.1976913958787918,
0.1984461098909378,
0.21992529928684235,
0.2206047922372818,
0.22257371246814728,
0.22331808507442474,
0.22564378380775452,
0.2258162647485733,
0.2261907458305359,
0.22567619383335114,
0.22598126530647278,
0.2251615822315216,
0.22548764944076538,
0.22496475279331207,
0.22340621054172516,
0.22249442338943481,
0.21654370427131653,
0.21478813886642456,
0.20714853703975677,
0.205490842461586,
0.19201314449310303,
0.19144563376903534,
0.18731488287448883,
0.188800647854805,
0.18594272434711456,
0.18874111771583557,
0.18583883345127106,
0.190367192029953,
0.18933312594890594,
0.19439123570919037,
0.1937587559223175,
0.198538675904274,
0.19723950326442719,
0.20280276238918304,
0.2025505006313324,
0.2084609866142273,
0.2101445198059082,
0.21558508276939392,
0.21427761018276215,
0.21985429525375366,
0.21906179189682007,
0.22511647641658783,
0.22344982624053955,
0.2311476618051529,
0.2303154021501541,
0.24068132042884827,
0.23970884084701538,
0.2547627091407776,
0.25386181473731995,
0.2762547731399536,
0.2756711542606354,
0.30550503730773926,
0.30545076727867126,
0.34050679206848145,
0.34091928601264954,
0.39206719398498535,
0.3925974369049072,
0.46342816948890686,
0.46681874990463257,
0.575099527835846
]
},
{
"activation_grad_rms_by_block": [
0.00010021305934060365,
0.00010125436529051512,
0.000114263420982752,
0.00011024689592886716,
0.00010690757335396484,
0.00010680041305022314,
0.00010475025192135945,
0.00010345032933400944,
9.713075269246474e-05,
9.490915545029566e-05,
9.159108594758436e-05,
8.836394408717752e-05,
8.501022239215672e-05,
8.242073090514168e-05,
7.99787012510933e-05,
7.76422893977724e-05,
7.462384382961318e-05,
7.162441761465743e-05,
7.150838064262643e-05,
6.863063026685268e-05,
6.474268593592569e-05,
6.149821274448186e-05,
5.7751010899664834e-05,
5.37736268597655e-05,
4.920278661302291e-05,
4.476245885598473e-05,
4.037534017697908e-05,
3.581849159672856e-05,
3.1086256058188155e-05,
2.5875511710182764e-05,
2.1458168703247793e-05,
1.863625948317349e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 0.00010598578865028685,
"first_to_last_ratio": 3.1730458332581706,
"imbalance_abs_log_ratio": 1.1546919572676668,
"last_quartile_mean": 3.3401909149688436e-05,
"mean": 7.300941479115863e-05,
"normalized": [
1.3726046103404645,
1.3868672359606,
1.565050498070684,
1.5100367020366523,
1.4642984560247596,
1.4628306959550723,
1.434749918500163,
1.4169450560578547,
1.330386676434875,
1.2999577619103044,
1.2545106163304838,
1.2103088942698703,
1.1643734254729483,
1.12890551363689,
1.0954573664214418,
1.0634558518222064,
1.0221126144220245,
0.9810298825094963,
0.9794405399245307,
0.9400243853915095,
0.8867717419886232,
0.8423326350498187,
0.7910077222897892,
0.7365300353876748,
0.6739238597346121,
0.613105295858438,
0.5530155294693376,
0.4906009957645373,
0.425784210804997,
0.35441335592401246,
0.29390961103616947,
0.255258305199158
],
"population_cv": 0.383084243994172
},
"activation_output_rms_by_block": [
0.11517385393381119,
0.17733007669448853,
0.24607165157794952,
0.2600470185279846,
0.2621006965637207,
0.26366734504699707,
0.2635972201824188,
0.26321178674697876,
0.2646143436431885,
0.2614729404449463,
0.2522071599960327,
0.23732660710811615,
0.2133668214082718,
0.2109174132347107,
0.21198545396327972,
0.21621736884117126,
0.2226637899875641,
0.22916270792484283,
0.23635683953762054,
0.24468488991260529,
0.25434690713882446,
0.26204079389572144,
0.2696947753429413,
0.27905911207199097,
0.29096299409866333,
0.30846187472343445,
0.3323265314102173,
0.36396095156669617,
0.403833270072937,
0.4673614203929901,
0.5606572031974792,
0.7190070152282715
],
"activation_output_statistics": {
"first_quartile_mean": 0.23139995615929365,
"first_to_last_ratio": 0.5371134119256165,
"imbalance_abs_log_ratio": 0.6215460113796702,
"last_quartile_mean": 0.43082140758633614,
"mean": 0.28637152607552707,
"normalized": [
0.4021833298588333,
0.6192308262090269,
0.859274156722729,
0.9080756808880515,
0.9152470573998156,
0.9207177426482616,
0.92047286891539,
0.9191269479688416,
0.9240246307637429,
0.9130549535709983,
0.8806991513867063,
0.8287367475407597,
0.7450699597557016,
0.7365167065495323,
0.740246269831209,
0.7550239781316335,
0.7775346698709116,
0.8002286786864554,
0.8253503509119278,
0.8544316303572463,
0.8881710784044343,
0.9150378792429639,
0.9417653320456605,
0.9744652895357776,
1.0160332561203216,
1.0771387747610122,
1.1604733751447416,
1.2709397353656795,
1.410172567109311,
1.6320107896122644,
1.9577966108600218,
2.510748973830038
],
"population_cv": 0.3918463566300126
},
"bits_per_byte": 1.9262875534052675,
"branch_output_rms_by_sublayer": [
0.021954918280243874,
0.07429475337266922,
0.09672636538743973,
0.017902610823512077,
0.0940650999546051,
0.0039193108677864075,
0.036259762942790985,
0.004245741758495569,
0.02603325992822647,
0.008932216092944145,
0.02122427336871624,
0.005480592139065266,
0.02764410711824894,
0.00817549042403698,
0.02432539500296116,
0.009956736117601395,
0.05137033388018608,
0.014394877478480339,
0.02661585807800293,
0.015817945823073387,
0.030927525833249092,
0.018779242411255836,
0.03579767048358917,
0.02186598815023899,
0.05378979817032814,
0.028890088200569153,
0.025482553988695145,
0.027309151366353035,
0.0287014190107584,
0.028864959254860878,
0.027393119409680367,
0.02912190742790699,
0.029581313952803612,
0.03136448562145233,
0.0320778414607048,
0.03278070688247681,
0.028201179578900337,
0.033851705491542816,
0.035640619695186615,
0.03695352375507355,
0.04368399456143379,
0.038013845682144165,
0.04277802258729935,
0.04267064854502678,
0.03937495872378349,
0.04597146809101105,
0.04081331565976143,
0.053922027349472046,
0.05011079087853432,
0.0602298267185688,
0.04528728500008583,
0.06855001300573349,
0.028651293367147446,
0.07740508019924164,
0.014390980824828148,
0.08345231413841248,
0.015424116514623165,
0.08889877796173096,
0.01913117803633213,
0.11831489205360413,
0.01693635806441307,
0.153398334980011,
0.032383937388658524,
0.24250756204128265
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.0003320807822012488,
0.00017565239824668803,
0.00017577232236880354,
0.00012480421087066412,
0.00013414970937401124,
0.0001166490159724099,
0.00011491447201106907,
0.00015823331904477844,
0.000249176250949333,
0.0001712841712530518,
0.00020010962025834788,
0.00021729074089037623,
0.0002489951018894741,
0.00022387049947650086,
0.00024825867897268754,
0.000256508706958601,
0.0002733879022225402,
0.00028045470195987973,
0.0002710271146389,
0.0002966387758490912,
0.00030658370753967526,
0.0003196905817287839,
0.0002949547776417074,
0.00030211663360406435,
0.0002930543360663917,
0.00026176751657805116,
0.0002415189744957938,
0.00021641616631898222,
0.00019981422740886822,
0.0001881612684369175,
0.00017039642606795488,
0.0001626151127852574
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.00016653202876120915,
"first_to_last_ratio": 0.7684272928714567,
"imbalance_abs_log_ratio": 0.2634093296250088,
"last_quartile_mean": 0.0002167180035197771,
"mean": 0.00022582338200252827,
"normalized": [
1.4705332072191308,
0.7778308724679426,
0.7783619251917662,
0.5526629251760424,
0.5940470299594988,
0.5165497697271398,
0.5088687937982548,
0.7006950194511165,
1.103412094618897,
0.758487317540635,
0.8861333068517568,
0.9622154223513644,
1.1026099232128512,
0.991352168634134,
1.0993488662299287,
1.1358819652950252,
1.2106270829806252,
1.2419205640837485,
1.2001729503628886,
1.313587517902686,
1.3576260563498372,
1.4156664331827458,
1.306130370673508,
1.3378447834984684,
1.2977147603923083,
1.159169233304284,
1.069503841250105,
0.9583425967668807,
0.8848252365941058,
0.8332231444253673,
0.7545561693254915,
0.7200986511814654
],
"population_cv": 0.2750960622596733
},
"depth_weights": [],
"layer_input_rms_by_sublayer": [
0.06031418219208717,
0.06744465231895447,
0.11517385393381119,
0.16882504522800446,
0.17733007669448853,
0.2454802393913269,
0.24607165157794952,
0.25892511010169983,
0.2600470185279846,
0.2610686123371124,
0.2621006965637207,
0.2635994553565979,
0.26366734504699707,
0.26440703868865967,
0.2635972201824188,
0.2638950049877167,
0.26321178674697876,
0.2655138075351715,
0.2646143436431885,
0.2625099718570709,
0.2614729404449463,
0.25542622804641724,
0.2522071599960327,
0.24079175293445587,
0.23732660710811615,
0.21542933583259583,
0.2133668214082718,
0.20939168334007263,
0.2109174132347107,
0.20824690163135529,
0.21198545396327972,
0.21042729914188385,
0.21621736884117126,
0.21581067144870758,
0.2226637899875641,
0.2222982496023178,
0.22916270792484283,
0.22995305061340332,
0.23635683953762054,
0.23728354275226593,
0.24468488991260529,
0.24780811369419098,
0.25434690713882446,
0.25493237376213074,
0.26204079389572144,
0.26210466027259827,
0.2696947753429413,
0.2699442505836487,
0.27905911207199097,
0.27968016266822815,
0.29096299409866333,
0.29178300499916077,
0.30846187472343445,
0.30848607420921326,
0.3323265314102173,
0.33218035101890564,
0.36396095156669617,
0.3638138175010681,
0.403833270072937,
0.40415704250335693,
0.4673614203929901,
0.4678023159503937,
0.5606572031974792,
0.5632834434509277
],
"loss_nats": 1.3352007865905762,
"loss_scale": 1.0,
"output_weights": null,
"step": 4000,
"stream_state_rms_by_sublayer": [
0.06744465231895447,
0.11517385393381119,
0.16882504522800446,
0.17733007669448853,
0.2454802393913269,
0.24607165157794952,
0.25892511010169983,
0.2600470185279846,
0.2610686123371124,
0.2621006965637207,
0.2635994553565979,
0.26366734504699707,
0.26440703868865967,
0.2635972201824188,
0.2638950049877167,
0.26321178674697876,
0.2655138075351715,
0.2646143436431885,
0.2625099718570709,
0.2614729404449463,
0.25542622804641724,
0.2522071599960327,
0.24079175293445587,
0.23732660710811615,
0.21542933583259583,
0.2133668214082718,
0.20939168334007263,
0.2109174132347107,
0.20824690163135529,
0.21198545396327972,
0.21042729914188385,
0.21621736884117126,
0.21581067144870758,
0.2226637899875641,
0.2222982496023178,
0.22916270792484283,
0.22995305061340332,
0.23635683953762054,
0.23728354275226593,
0.24468488991260529,
0.24780811369419098,
0.25434690713882446,
0.25493237376213074,
0.26204079389572144,
0.26210466027259827,
0.2696947753429413,
0.2699442505836487,
0.27905911207199097,
0.27968016266822815,
0.29096299409866333,
0.29178300499916077,
0.30846187472343445,
0.30848607420921326,
0.3323265314102173,
0.33218035101890564,
0.36396095156669617,
0.3638138175010681,
0.403833270072937,
0.40415704250335693,
0.4673614203929901,
0.4678023159503937,
0.5606572031974792,
0.5632834434509277,
0.7190070152282715
]
},
{
"activation_grad_rms_by_block": [
0.00018638069741427898,
0.00018956589337904006,
0.00020923581905663013,
0.00020349136320874095,
0.00019885810615960509,
0.0001991111203096807,
0.00019671664631459862,
0.00019502552459016442,
0.00018589819956105202,
0.00018270809960085899,
0.00017816871695686132,
0.00017320679035037756,
0.0001670292258495465,
0.00016241244156844914,
0.00015707056445535272,
0.0001517482742201537,
0.0001448393304599449,
0.00013789460354018956,
0.00013790397497359663,
0.00013200659304857254,
0.00012367918679956347,
0.00011618347343755886,
0.00010830583050847054,
9.980145841836929e-05,
8.979889389593154e-05,
8.034214988583699e-05,
7.120905502233654e-05,
6.192603905219585e-05,
5.214559132582508e-05,
4.129187436774373e-05,
3.1231873435899615e-05,
2.3505688659497537e-05
],
"activation_grad_statistics": {
"first_quartile_mean": 0.00019729814630409237,
"first_to_last_ratio": 3.49624785701179,
"imbalance_abs_log_ratio": 1.2516903525947785,
"last_quartile_mean": 5.643139570565836e-05,
"mean": 0.00013714665936959136,
"normalized": [
1.3589882412812453,
1.3822129846282736,
1.525635549698432,
1.4837500537316024,
1.449966824373825,
1.4518116680706283,
1.434352445905914,
1.4220216918634343,
1.3554701253063843,
1.3322096246506874,
1.2991108771867432,
1.2629311654147486,
1.217887673074309,
1.1842245543201309,
1.1452744469121132,
1.1064671565292232,
1.0560908382727927,
1.0054536083783316,
1.0055219398524642,
0.9625213888209478,
0.9018024016630627,
0.8471476736772742,
0.7897081198062669,
0.7276987924978771,
0.6547654482340389,
0.5858119348669365,
0.519218297767195,
0.451531516238494,
0.38021772871113024,
0.3010782366668357,
0.22772609686200243,
0.17139089473665517
],
"population_cv": 0.4031953558435787
},
"activation_output_rms_by_block": [
0.1026567816734314,
0.17673343420028687,
0.23246681690216064,
0.24062344431877136,
0.2416554093360901,
0.24258241057395935,
0.24218299984931946,
0.24160927534103394,
0.24130181968212128,
0.23599836230278015,
0.22310413420200348,
0.20329499244689941,
0.176778644323349,
0.17310521006584167,
0.1729687601327896,
0.17606647312641144,
0.1812635064125061,
0.18683874607086182,
0.19253391027450562,
0.1985689103603363,
0.20644955337047577,
0.2135031372308731,
0.22015690803527832,
0.228074312210083,
0.2383035272359848,
0.2532499134540558,
0.27322494983673096,
0.2992607355117798,
0.3339390158653259,
0.3918353319168091,
0.48234984278678894,
0.651733934879303
],
"activation_output_statistics": {
"first_quartile_mean": 0.21506382152438164,
"first_to_last_ratio": 0.5884305856918151,
"imbalance_abs_log_ratio": 0.5302963104844547,
"last_quartile_mean": 0.3654871564358473,
"mean": 0.2460754751227796,
"normalized": [
0.41717599700746566,
0.7182082412402355,
0.9446972185512233,
0.9778440708027163,
0.9820377638832796,
0.9858049058034841,
0.9841817829610285,
0.9818502848383519,
0.9806008484255633,
0.9590486909962422,
0.906649206267652,
0.8261489380258848,
0.7183919658598449,
0.7034638863522286,
0.7029093819548116,
0.7154978489366439,
0.7366175207913933,
0.7592741453720182,
0.7824181185810598,
0.8069431147547724,
0.838968423275288,
0.8676327339176954,
0.8946722867259763,
0.9268469850410127,
0.9684164060521798,
1.0291554382967116,
1.1103298681041054,
1.2161339335521484,
1.357059315637638,
1.5923380081712835,
1.9601703198830358,
2.648512349937025
],
"population_cv": 0.40510862299171807
},
"bits_per_byte": 1.8611962476373152,
"branch_output_rms_by_sublayer": [
0.026356825605034828,
0.057217150926589966,
0.10627161711454391,
0.01789308711886406,
0.07707435637712479,
0.0034156402107328176,
0.02677960880100727,
0.0038151287008076906,
0.02153952419757843,
0.007133069448173046,
0.01632758229970932,
0.004304513800889254,
0.01752857305109501,
0.006448030937463045,
0.01845136471092701,
0.007711905520409346,
0.044129498302936554,
0.011710400693118572,
0.023254960775375366,
0.01295715756714344,
0.02913394197821617,
0.01598021760582924,
0.03491654247045517,
0.01944364421069622,
0.04857943207025528,
0.02654583752155304,
0.023438163101673126,
0.02489112876355648,
0.025785095989704132,
0.0264157485216856,
0.025251252576708794,
0.02720639854669571,
0.027512386441230774,
0.029633820056915283,
0.02885596640408039,
0.031438712030649185,
0.0244715828448534,
0.03218274191021919,
0.031743232160806656,
0.03524785488843918,
0.038931526243686676,
0.037403788417577744,
0.03771914169192314,
0.041983578354120255,
0.03549705073237419,
0.04549537971615791,
0.0359940230846405,
0.052616480737924576,
0.04251093044877052,
0.059319522231817245,
0.03818616643548012,
0.06705058366060257,
0.024311494082212448,
0.07359735667705536,
0.01473314967006445,
0.07876058667898178,
0.015392928384244442,
0.0848938599228859,
0.018577447161078453,
0.11232978105545044,
0.018359949812293053,
0.1471690982580185,
0.03667483106255531,
0.24789714813232422
],
"capture": {
"all_gradients_finite": true,
"all_gradients_present": true,
"count": 32,
"dtypes": [
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32",
"torch.float32"
],
"position": "post-MLP Transformer-block output; Block AttnRes is captured before aggregation-partial reset",
"shape": [
16,
256,
192
],
"storage_unique": true
},
"core_parameter_grad_rms_by_block": [
0.0005542142737822822,
0.0003797328422683109,
0.000340619215402376,
0.0001793765938241596,
0.00020684039436717747,
0.00016374338231517116,
0.00018705111879548875,
0.0002256278252430331,
0.000426778694633347,
0.00027155262856832466,
0.0003392540980461625,
0.0003897844238340517,
0.00045204461946776503,
0.0004208641344247883,
0.00047738343422760874,
0.00048450285457785075,
0.0005178591522769467,
0.0005253914687520978,
0.0005086607978429822,
0.0005610796431247281,
0.0005682688177114847,
0.000574059459111366,
0.0005453911942509562,
0.0005508569019095279,
0.0005214428582378581,
0.000461966711188474,
0.00041497592078977314,
0.00037256322877604213,
0.00033686502618798456,
0.00030674152201740854,
0.00025927963978097604,
0.00021950976285567438
],
"core_parameter_grad_statistics": {
"first_quartile_mean": 0.0002796507057497499,
"first_to_last_ratio": 0.7732247282264533,
"imbalance_abs_log_ratio": 0.25718555049038655,
"last_quartile_mean": 0.00036166808372927383,
"mean": 0.00039825883245600557,
"normalized": [
1.3915931766397287,
0.9534825377922003,
0.8552709636138532,
0.4504020481302817,
0.5193617253674505,
0.41114815032572916,
0.46967224214958675,
0.5665356468094339,
1.0716113739435822,
0.681849607436697,
0.8518432496626144,
0.9787213542266133,
1.1350523394046785,
1.056760328024313,
1.1986763263570401,
1.2165526916000595,
1.3003080159789122,
1.319221134436827,
1.2772115930389878,
1.4088316376177517,
1.4268831508570738,
1.441423045337684,
1.3694390426638032,
1.383163051306287,
1.3093064503358134,
1.159966015918821,
1.0419754365036318,
0.9354801410893958,
0.8458444577627716,
0.7702064512311685,
0.6510329932472191,
0.5511736211899907
],
"population_cv": 0.3252608102570666
},
"depth_weights": [],
"layer_input_rms_by_sublayer": [
0.061885204166173935,
0.07089564949274063,
0.1026567816734314,
0.16996370255947113,
0.17673343420028687,
0.23203739523887634,
0.23246681690216064,
0.23961465060710907,
0.24062344431877136,
0.24108263850212097,
0.2416554093360901,
0.24260269105434418,
0.24258241057395935,
0.2429492026567459,
0.24218299984931946,
0.24250134825706482,
0.24160927534103394,
0.242857426404953,
0.24130181968212128,
0.23791606724262238,
0.23599836230278015,
0.22686637938022614,
0.22310413420200348,
0.20835906267166138,
0.20329499244689941,
0.1809774935245514,
0.176778644323349,
0.17327234148979187,
0.17310521006584167,
0.17050857841968536,
0.1729687601327896,
0.17174123227596283,
0.17606647312641144,
0.1756037324666977,
0.1812635064125061,
0.1811160296201706,
0.18683874607086182,
0.1874236911535263,
0.19253391027450562,
0.19280622899532318,
0.1985689103603363,
0.20097090303897858,
0.20644955337047577,
0.2074177861213684,
0.2135031372308731,
0.21357181668281555,
0.22015690803527832,
0.22022640705108643,
0.228074312210083,
0.22874467074871063,
0.2383035272359848,
0.23907209932804108,
0.2532499134540558,
0.2534840703010559,
0.27322494983673096,
0.27303415536880493,
0.2992607355117798,
0.29903557896614075,
0.3339390158653259,
0.3341987431049347,
0.3918353319168091,
0.3921641707420349,
0.48234984278678894,
0.4832552373409271
],
"loss_nats": 1.2900829315185547,
"loss_scale": 1.0,
"output_weights": null,
"step": 8000,
"stream_state_rms_by_sublayer": [
0.07089564949274063,
0.1026567816734314,
0.16996370255947113,
0.17673343420028687,
0.23203739523887634,
0.23246681690216064,
0.23961465060710907,
0.24062344431877136,
0.24108263850212097,
0.2416554093360901,
0.24260269105434418,
0.24258241057395935,
0.2429492026567459,
0.24218299984931946,
0.24250134825706482,
0.24160927534103394,
0.242857426404953,
0.24130181968212128,
0.23791606724262238,
0.23599836230278015,
0.22686637938022614,
0.22310413420200348,
0.20835906267166138,
0.20329499244689941,
0.1809774935245514,
0.176778644323349,
0.17327234148979187,
0.17310521006584167,
0.17050857841968536,
0.1729687601327896,
0.17174123227596283,
0.17606647312641144,
0.1756037324666977,
0.1812635064125061,
0.1811160296201706,
0.18683874607086182,
0.1874236911535263,
0.19253391027450562,
0.19280622899532318,
0.1985689103603363,
0.20097090303897858,
0.20644955337047577,
0.2074177861213684,
0.2135031372308731,
0.21357181668281555,
0.22015690803527832,
0.22022640705108643,
0.228074312210083,
0.22874467074871063,
0.2383035272359848,
0.23907209932804108,
0.2532499134540558,
0.2534840703010559,
0.27322494983673096,
0.27303415536880493,
0.2992607355117798,
0.29903557896614075,
0.3339390158653259,
0.3341987431049347,
0.3918353319168091,
0.3921641707420349,
0.48234984278678894,
0.4832552373409271,
0.651733934879303
]
}
],
"environment": {
"autocast": "cuda-bfloat16-forward-fp32-cross-entropy",
"compile": false,
"compute_capability": [
12,
0
],
"cublas_workspace_config": ":4096:8",
"cuda": "12.8",
"deterministic_algorithms": true,
"gpu": "NVIDIA GeForce RTX 5090",
"python": "3.10.14",
"torch": "2.11.0+cu128"
},
"evaluations": [
{
"bits_per_byte": 8.06581270897968,
"cross_entropy_nats": 5.590795338153839,
"step": 0
},
{
"bits_per_byte": 5.193364149696528,
"cross_entropy_nats": 3.599765717983246,
"step": 100
},
{
"bits_per_byte": 3.118855971380574,
"cross_entropy_nats": 2.1618262231349945,
"step": 500
},
{
"bits_per_byte": 1.9576097217110056,
"cross_entropy_nats": 1.3569116592407227,
"step": 2000
},
{
"bits_per_byte": 1.792987261872353,
"cross_entropy_nats": 1.2428040653467178,
"step": 4000
},
{
"bits_per_byte": 1.709514213904039,
"cross_entropy_nats": 1.1849449574947357,
"step": 8000
}
],
"gradient_gate": null,
"hashes": {
"final_mixer_parameters": null,
"final_model_state": "411fcebe23a13021e3c3b6a3b03dca473d4b287d53d6c87cd6f8b3e6b7611d22",
"final_optimizer_state": "e774625d2eec7b16bb949a8bd26f25b05b7288a59bf24a716a88b489cd22bcd6",
"final_public_parameters": "411fcebe23a13021e3c3b6a3b03dca473d4b287d53d6c87cd6f8b3e6b7611d22",
"initial_mixer_parameters": null,
"initial_public_parameter_elements": 18985152,
"initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670",
"initial_public_parameter_tensors": 227,
"initial_public_parameters": "3f93c11463780b386fff00c53b0424f687e1136b1b80f7b626f058adef96c764"
},
"manifest": {
"diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716",
"file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371",
"formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e",
"input_gate_tensor_hashes": {
"0": "3eff5c07fa0cfbd5ceed5922cbd09525ad514f801cc0a440c03fd2fa15df4c84",
"1": "e83e4001c5811b19c69a54c7ea7a802af996a34aa57072cde43a76bc8369e2d0",
"7999": "d04c4ca879f0209183a48ac54a0c4ffe3a0f62a78888479d4641f6c304426442"
},
"path": "experiments/k3/attnres_gradient/manifest.json",
"validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338"
},
"model": {
"attnres_aggregation_groups": 8,
"context": 256,
"d_ff": 768,
"d_head": 32,
"d_model": 192,
"heads": 6,
"layers": 32,
"parameters": {
"core": 18985152,
"embedding": 98304,
"mixer": 0,
"total": 18985152
},
"sublayers": 64,
"sublayers_per_attnres_group": 8,
"transformer_blocks_per_attnres_group": 4,
"vocabulary": 256
},
"optimizer": {
"betas": [
0.9,
0.95
],
"epsilon": 1e-08,
"grad_clip": 1.0,
"min_lr": 3e-05,
"name": "AdamW",
"peak_lr": 0.0003,
"warmup_steps": 400,
"weight_decay_ndim_ge_2": 0.1
},
"protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1",
"run_kind": "formal",
"schema_version": 1,
"seed": 2026073003,
"steps": 8000,
"target_bytes_seen": 65536000,
"timing": {
"mean_ms": 42.39461278025134,
"measured_steps": 7980,
"median_ms": 42.272048493032344,
"p95_ms": 43.13050716154976,
"peak_allocated_bytes": 5938562048,
"peak_reserved_bytes": 6325010432,
"warmup_steps_excluded": 20
},
"training_history": [
{
"bits_per_byte": 8.087939566883062,
"learning_rate": 7.499999999999999e-07,
"loss_nats": 5.606132507324219,
"step": 1,
"unclipped_grad_norm": 19.928020477294922
},
{
"bits_per_byte": 7.3282798152711734,
"learning_rate": 7.499999999999999e-06,
"loss_nats": 5.07957649230957,
"step": 10,
"unclipped_grad_norm": 11.731513977050781
},
{
"bits_per_byte": 6.791570117070589,
"learning_rate": 1.4999999999999999e-05,
"loss_nats": 4.707557678222656,
"step": 20,
"unclipped_grad_norm": 3.350315809249878
},
{
"bits_per_byte": 6.667928349712299,
"learning_rate": 2.2499999999999998e-05,
"loss_nats": 4.621855735778809,
"step": 30,
"unclipped_grad_norm": 3.0359153747558594
},
{
"bits_per_byte": 6.473404964696806,
"learning_rate": 2.9999999999999997e-05,
"loss_nats": 4.487022399902344,
"step": 40,
"unclipped_grad_norm": 3.083479404449463
},
{
"bits_per_byte": 6.159992036931666,
"learning_rate": 3.75e-05,
"loss_nats": 4.269781112670898,
"step": 50,
"unclipped_grad_norm": 2.770655393600464
},
{
"bits_per_byte": 5.940455495174687,
"learning_rate": 4.4999999999999996e-05,
"loss_nats": 4.117609977722168,
"step": 60,
"unclipped_grad_norm": 2.5750629901885986
},
{
"bits_per_byte": 5.762018679390299,
"learning_rate": 5.2499999999999995e-05,
"loss_nats": 3.993927001953125,
"step": 70,
"unclipped_grad_norm": 2.5019538402557373
},
{
"bits_per_byte": 5.5944951946990615,
"learning_rate": 5.9999999999999995e-05,
"loss_nats": 3.8778085708618164,
"step": 80,
"unclipped_grad_norm": 2.5689163208007812
},
{
"bits_per_byte": 5.40584020169631,
"learning_rate": 6.749999999999999e-05,
"loss_nats": 3.7470428943634033,
"step": 90,
"unclipped_grad_norm": 2.3143606185913086
},
{
"bits_per_byte": 5.179311877251155,
"learning_rate": 7.5e-05,
"loss_nats": 3.5900254249572754,
"step": 100,
"unclipped_grad_norm": 2.227365255355835
},
{
"bits_per_byte": 4.996660110659421,
"learning_rate": 8.249999999999999e-05,
"loss_nats": 3.463420867919922,
"step": 110,
"unclipped_grad_norm": 2.0371005535125732
},
{
"bits_per_byte": 4.800645980026251,
"learning_rate": 8.999999999999999e-05,
"loss_nats": 3.327554225921631,
"step": 120,
"unclipped_grad_norm": 1.8128844499588013
},
{
"bits_per_byte": 4.634309863556569,
"learning_rate": 9.75e-05,
"loss_nats": 3.212258815765381,
"step": 130,
"unclipped_grad_norm": 1.5838756561279297
},
{
"bits_per_byte": 4.478764626532625,
"learning_rate": 0.00010499999999999999,
"loss_nats": 3.104443073272705,
"step": 140,
"unclipped_grad_norm": 1.4334924221038818
},
{
"bits_per_byte": 4.278149959988874,
"learning_rate": 0.0001125,
"loss_nats": 2.9653875827789307,
"step": 150,
"unclipped_grad_norm": 1.2058515548706055
},
{
"bits_per_byte": 4.130972366860008,
"learning_rate": 0.00011999999999999999,
"loss_nats": 2.8633718490600586,
"step": 160,
"unclipped_grad_norm": 1.057314395904541
},
{
"bits_per_byte": 4.025422830407184,
"learning_rate": 0.00012749999999999998,
"loss_nats": 2.790210485458374,
"step": 170,
"unclipped_grad_norm": 0.9353183507919312
},
{
"bits_per_byte": 3.9235808959416594,
"learning_rate": 0.00013499999999999997,
"loss_nats": 2.719619035720825,
"step": 180,
"unclipped_grad_norm": 0.8658975958824158
},
{
"bits_per_byte": 3.837114554428572,
"learning_rate": 0.0001425,
"loss_nats": 2.6596851348876953,
"step": 190,
"unclipped_grad_norm": 0.8723611235618591
},
{
"bits_per_byte": 3.771249326731737,
"learning_rate": 0.00015,
"loss_nats": 2.6140308380126953,
"step": 200,
"unclipped_grad_norm": 0.9528701305389404
},
{
"bits_per_byte": 3.687100279456375,
"learning_rate": 0.0001575,
"loss_nats": 2.5557031631469727,
"step": 210,
"unclipped_grad_norm": 0.7721163034439087
},
{
"bits_per_byte": 3.664338719579984,
"learning_rate": 0.00016499999999999997,
"loss_nats": 2.539926052093506,
"step": 220,
"unclipped_grad_norm": 1.0300326347351074
},
{
"bits_per_byte": 3.616636923606072,
"learning_rate": 0.0001725,
"loss_nats": 2.506861686706543,
"step": 230,
"unclipped_grad_norm": 1.116415023803711
},
{
"bits_per_byte": 3.58821025521026,
"learning_rate": 0.00017999999999999998,
"loss_nats": 2.4871578216552734,
"step": 240,
"unclipped_grad_norm": 0.7745823860168457
},
{
"bits_per_byte": 3.5119345736913976,
"learning_rate": 0.0001875,
"loss_nats": 2.4342875480651855,
"step": 250,
"unclipped_grad_norm": 0.9986598491668701
},
{
"bits_per_byte": 3.522710662629159,
"learning_rate": 0.000195,
"loss_nats": 2.4417569637298584,
"step": 260,
"unclipped_grad_norm": 0.8885983228683472
},
{
"bits_per_byte": 3.5021329384534243,
"learning_rate": 0.00020249999999999996,
"loss_nats": 2.4274935722351074,
"step": 270,
"unclipped_grad_norm": 0.7707400321960449
},
{
"bits_per_byte": 3.4629779922967905,
"learning_rate": 0.00020999999999999998,
"loss_nats": 2.40035343170166,
"step": 280,
"unclipped_grad_norm": 1.055368423461914
},
{
"bits_per_byte": 3.4474741002868097,
"learning_rate": 0.00021749999999999997,
"loss_nats": 2.3896069526672363,
"step": 290,
"unclipped_grad_norm": 1.053453803062439
},
{
"bits_per_byte": 3.4491093113312137,
"learning_rate": 0.000225,
"loss_nats": 2.390740394592285,
"step": 300,
"unclipped_grad_norm": 1.2985762357711792
},
{
"bits_per_byte": 3.334430591805289,
"learning_rate": 0.00023249999999999996,
"loss_nats": 2.311251163482666,
"step": 310,
"unclipped_grad_norm": 0.8886587619781494
},
{
"bits_per_byte": 3.4024717440008403,
"learning_rate": 0.00023999999999999998,
"loss_nats": 2.3584136962890625,
"step": 320,
"unclipped_grad_norm": 1.0081987380981445
},
{
"bits_per_byte": 3.413793361907139,
"learning_rate": 0.0002475,
"loss_nats": 2.3662612438201904,
"step": 330,
"unclipped_grad_norm": 0.8511857986450195
},
{
"bits_per_byte": 3.3420762525320464,
"learning_rate": 0.00025499999999999996,
"loss_nats": 2.3165507316589355,
"step": 340,
"unclipped_grad_norm": 0.9547706842422485
},
{
"bits_per_byte": 3.3350242759160675,
"learning_rate": 0.0002625,
"loss_nats": 2.3116626739501953,
"step": 350,
"unclipped_grad_norm": 1.1394811868667603
},
{
"bits_per_byte": 3.3268671387856448,
"learning_rate": 0.00026999999999999995,
"loss_nats": 2.3060085773468018,
"step": 360,
"unclipped_grad_norm": 1.0605591535568237
},
{
"bits_per_byte": 3.320889365806955,
"learning_rate": 0.00027749999999999997,
"loss_nats": 2.3018651008605957,
"step": 370,
"unclipped_grad_norm": 0.8489253520965576
},
{
"bits_per_byte": 3.2193453052927254,
"learning_rate": 0.000285,
"loss_nats": 2.231480121612549,
"step": 380,
"unclipped_grad_norm": 0.923150897026062
},
{
"bits_per_byte": 3.297119987596508,
"learning_rate": 0.0002925,
"loss_nats": 2.2853894233703613,
"step": 390,
"unclipped_grad_norm": 0.7830708026885986
},
{
"bits_per_byte": 3.2808776020837227,
"learning_rate": 0.0003,
"loss_nats": 2.2741310596466064,
"step": 400,
"unclipped_grad_norm": 0.7351507544517517
},
{
"bits_per_byte": 3.2985058237971607,
"learning_rate": 0.0002999988466112388,
"loss_nats": 2.2863500118255615,
"step": 410,
"unclipped_grad_norm": 0.8769013285636902
},
{
"bits_per_byte": 3.225643997897925,
"learning_rate": 0.00029999538646466355,
"loss_nats": 2.2358460426330566,
"step": 420,
"unclipped_grad_norm": 0.7473549842834473
},
{
"bits_per_byte": 3.2585050669635836,
"learning_rate": 0.0002999896196193985,
"loss_nats": 2.2586236000061035,
"step": 430,
"unclipped_grad_norm": 0.9111914038658142
},
{
"bits_per_byte": 3.2092602426461303,
"learning_rate": 0.00029998154617398326,
"loss_nats": 2.224489688873291,
"step": 440,
"unclipped_grad_norm": 1.075931191444397
},
{
"bits_per_byte": 3.2640807445045557,
"learning_rate": 0.0002999711662663707,
"loss_nats": 2.26248836517334,
"step": 450,
"unclipped_grad_norm": 0.8512128591537476
},
{
"bits_per_byte": 3.207580316112502,
"learning_rate": 0.0002999584800739247,
"loss_nats": 2.223325252532959,
"step": 460,
"unclipped_grad_norm": 1.2196831703186035
},
{
"bits_per_byte": 3.1660169249131855,
"learning_rate": 0.0002999434878134173,
"loss_nats": 2.1945157051086426,
"step": 470,
"unclipped_grad_norm": 0.7675222158432007
},
{
"bits_per_byte": 3.1673281206433637,
"learning_rate": 0.000299926189741025,
"loss_nats": 2.1954245567321777,
"step": 480,
"unclipped_grad_norm": 0.9160394668579102
},
{
"bits_per_byte": 3.113679164602616,
"learning_rate": 0.000299906586152324,
"loss_nats": 2.158237934112549,
"step": 490,
"unclipped_grad_norm": 0.6955189108848572
},
{
"bits_per_byte": 3.0709621137813112,
"learning_rate": 0.0002998846773822856,
"loss_nats": 2.128628730773926,
"step": 500,
"unclipped_grad_norm": 0.9366468191146851
},
{
"bits_per_byte": 3.1516054667014854,
"learning_rate": 0.0002998604638052702,
"loss_nats": 2.1845264434814453,
"step": 510,
"unclipped_grad_norm": 1.3161629438400269
},
{
"bits_per_byte": 3.151479575401054,
"learning_rate": 0.0002998339458350211,
"loss_nats": 2.184439182281494,
"step": 520,
"unclipped_grad_norm": 0.8699809312820435
},
{
"bits_per_byte": 3.012768688174164,
"learning_rate": 0.0002998051239246572,
"loss_nats": 2.088292121887207,
"step": 530,
"unclipped_grad_norm": 1.1934181451797485
},
{
"bits_per_byte": 3.0200408025832473,
"learning_rate": 0.00029977399856666537,
"loss_nats": 2.0933327674865723,
"step": 540,
"unclipped_grad_norm": 0.8321981430053711
},
{
"bits_per_byte": 3.02149439994834,
"learning_rate": 0.0002997405702928922,
"loss_nats": 2.0943403244018555,
"step": 550,
"unclipped_grad_norm": 0.9657202959060669
},
{
"bits_per_byte": 3.0546815481200342,
"learning_rate": 0.0002997048396745345,
"loss_nats": 2.1173439025878906,
"step": 560,
"unclipped_grad_norm": 1.2880722284317017
},
{
"bits_per_byte": 2.973202359516667,
"learning_rate": 0.0002996668073221302,
"loss_nats": 2.0608668327331543,
"step": 570,
"unclipped_grad_norm": 0.8365735411643982
},
{
"bits_per_byte": 2.986164347946895,
"learning_rate": 0.000299626473885547,
"loss_nats": 2.0698513984680176,
"step": 580,
"unclipped_grad_norm": 0.9132604002952576
},
{
"bits_per_byte": 2.926336399225948,
"learning_rate": 0.00029958384005397226,
"loss_nats": 2.028381824493408,
"step": 590,
"unclipped_grad_norm": 0.7623230814933777
},
{
"bits_per_byte": 2.903745446139213,
"learning_rate": 0.0002995389065559004,
"loss_nats": 2.012722969055176,
"step": 600,
"unclipped_grad_norm": 0.8222759366035461
},
{
"bits_per_byte": 2.915261748406288,
"learning_rate": 0.000299491674159121,
"loss_nats": 2.020705461502075,
"step": 610,
"unclipped_grad_norm": 1.3526110649108887
},
{
"bits_per_byte": 2.8888483089214763,
"learning_rate": 0.00029944214367070535,
"loss_nats": 2.002397060394287,
"step": 620,
"unclipped_grad_norm": 0.9250975847244263
},
{
"bits_per_byte": 2.806001858243749,
"learning_rate": 0.00029939031593699284,
"loss_nats": 1.944972276687622,
"step": 630,
"unclipped_grad_norm": 0.7925068140029907
},
{
"bits_per_byte": 2.807606972324252,
"learning_rate": 0.00029933619184357654,
"loss_nats": 1.9460848569869995,
"step": 640,
"unclipped_grad_norm": 0.8585688471794128
},
{
"bits_per_byte": 2.785753824809718,
"learning_rate": 0.00029927977231528774,
"loss_nats": 1.93093740940094,
"step": 650,
"unclipped_grad_norm": 0.7446964383125305
},
{
"bits_per_byte": 2.8113040833698495,
"learning_rate": 0.00029922105831618057,
"loss_nats": 1.9486474990844727,
"step": 660,
"unclipped_grad_norm": 1.026633381843567
},
{
"bits_per_byte": 2.7895744193986354,
"learning_rate": 0.0002991600508495153,
"loss_nats": 1.9335856437683105,
"step": 670,
"unclipped_grad_norm": 0.8819775581359863
},
{
"bits_per_byte": 2.7608092891459255,
"learning_rate": 0.00029909675095774114,
"loss_nats": 1.913647174835205,
"step": 680,
"unclipped_grad_norm": 0.845567524433136
},
{
"bits_per_byte": 2.7436651985946665,
"learning_rate": 0.0002990311597224787,
"loss_nats": 1.9017637968063354,
"step": 690,
"unclipped_grad_norm": 1.0123262405395508
},
{
"bits_per_byte": 2.7268093509662537,
"learning_rate": 0.00029896327826450103,
"loss_nats": 1.890080213546753,
"step": 700,
"unclipped_grad_norm": 0.8187565803527832
},
{
"bits_per_byte": 2.696082242632501,
"learning_rate": 0.0002988931077437152,
"loss_nats": 1.8687818050384521,
"step": 710,
"unclipped_grad_norm": 0.9888712763786316
},
{
"bits_per_byte": 2.7032793726058073,
"learning_rate": 0.00029882064935914164,
"loss_nats": 1.8737704753875732,
"step": 720,
"unclipped_grad_norm": 0.8442726731300354
},
{
"bits_per_byte": 2.680898066370474,
"learning_rate": 0.0002987459043488943,
"loss_nats": 1.8582569360733032,
"step": 730,
"unclipped_grad_norm": 1.1784851551055908
},
{
"bits_per_byte": 2.619564409541223,
"learning_rate": 0.0002986688739901591,
"loss_nats": 1.8157436847686768,
"step": 740,
"unclipped_grad_norm": 0.8383970260620117
},
{
"bits_per_byte": 2.6352585143630582,
"learning_rate": 0.0002985895595991723,
"loss_nats": 1.8266220092773438,
"step": 750,
"unclipped_grad_norm": 1.062637448310852
},
{
"bits_per_byte": 2.547899926857294,
"learning_rate": 0.00029850796253119824,
"loss_nats": 1.7660696506500244,
"step": 760,
"unclipped_grad_norm": 0.8228002786636353
},
{
"bits_per_byte": 2.573545291823353,
"learning_rate": 0.00029842408418050544,
"loss_nats": 1.7838456630706787,
"step": 770,
"unclipped_grad_norm": 0.8357667326927185
},
{
"bits_per_byte": 2.523504187832405,
"learning_rate": 0.0002983379259803436,
"loss_nats": 1.749159812927246,
"step": 780,
"unclipped_grad_norm": 0.7918128967285156
},
{
"bits_per_byte": 2.5308719245953677,
"learning_rate": 0.0002982494894029186,
"loss_nats": 1.7542667388916016,
"step": 790,
"unclipped_grad_norm": 1.3666422367095947
},
{
"bits_per_byte": 2.443318996694129,
"learning_rate": 0.0002981587759593675,
"loss_nats": 1.6935796737670898,
"step": 800,
"unclipped_grad_norm": 0.7999250888824463
},
{
"bits_per_byte": 2.500851320991359,
"learning_rate": 0.00029806578719973283,
"loss_nats": 1.7334580421447754,
"step": 810,
"unclipped_grad_norm": 1.1067378520965576
},
{
"bits_per_byte": 2.4739762800716316,
"learning_rate": 0.00029797052471293596,
"loss_nats": 1.714829683303833,
"step": 820,
"unclipped_grad_norm": 1.090567708015442
},
{
"bits_per_byte": 2.467352884221603,
"learning_rate": 0.0002978729901267499,
"loss_nats": 1.7102386951446533,
"step": 830,
"unclipped_grad_norm": 0.9818849563598633
},
{
"bits_per_byte": 2.471093850843171,
"learning_rate": 0.0002977731851077716,
"loss_nats": 1.712831735610962,
"step": 840,
"unclipped_grad_norm": 0.9710109829902649
},
{
"bits_per_byte": 2.396449768733044,
"learning_rate": 0.0002976711113613935,
"loss_nats": 1.6610924005508423,
"step": 850,
"unclipped_grad_norm": 1.0234057903289795
},
{
"bits_per_byte": 2.473427999380681,
"learning_rate": 0.0002975667706317743,
"loss_nats": 1.7144496440887451,
"step": 860,
"unclipped_grad_norm": 0.9857868552207947
},
{
"bits_per_byte": 2.465926632097998,
"learning_rate": 0.00029746016470180915,
"loss_nats": 1.7092500925064087,
"step": 870,
"unclipped_grad_norm": 1.1294745206832886
},
{
"bits_per_byte": 2.391602093753174,
"learning_rate": 0.00029735129539309926,
"loss_nats": 1.6577322483062744,
"step": 880,
"unclipped_grad_norm": 0.8081791400909424
},
{
"bits_per_byte": 2.354239550783832,
"learning_rate": 0.0002972401645659207,
"loss_nats": 1.6318345069885254,
"step": 890,
"unclipped_grad_norm": 0.8606147170066833
},
{
"bits_per_byte": 2.395512979233794,
"learning_rate": 0.00029712677411919283,
"loss_nats": 1.6604430675506592,
"step": 900,
"unclipped_grad_norm": 0.8592826128005981
},
{
"bits_per_byte": 2.3758491708647442,
"learning_rate": 0.0002970111259904454,
"loss_nats": 1.646813154220581,
"step": 910,
"unclipped_grad_norm": 0.8169032335281372
},
{
"bits_per_byte": 2.353533905967342,
"learning_rate": 0.0002968932221557859,
"loss_nats": 1.6313453912734985,
"step": 920,
"unclipped_grad_norm": 0.7358964085578918
},
{
"bits_per_byte": 2.2994974730142004,
"learning_rate": 0.0002967730646298655,
"loss_nats": 1.5938901901245117,
"step": 930,
"unclipped_grad_norm": 0.8204946517944336
},
{
"bits_per_byte": 2.3548098452540933,
"learning_rate": 0.0002966506554658449,
"loss_nats": 1.6322298049926758,
"step": 940,
"unclipped_grad_norm": 0.8031407594680786
},
{
"bits_per_byte": 2.362864480724602,
"learning_rate": 0.00029652599675535876,
"loss_nats": 1.637812852859497,
"step": 950,
"unclipped_grad_norm": 0.8955363035202026
},
{
"bits_per_byte": 2.311957100121258,
"learning_rate": 0.0002963990906284808,
"loss_nats": 1.6025265455245972,
"step": 960,
"unclipped_grad_norm": 0.8827840089797974
},
{
"bits_per_byte": 2.2663949403067303,
"learning_rate": 0.0002962699392536863,
"loss_nats": 1.5709452629089355,
"step": 970,
"unclipped_grad_norm": 1.4933069944381714
},
{
"bits_per_byte": 2.401434685868302,
"learning_rate": 0.0002961385448378161,
"loss_nats": 1.6645476818084717,
"step": 980,
"unclipped_grad_norm": 1.0056283473968506
},
{
"bits_per_byte": 2.23815401317385,
"learning_rate": 0.00029600490962603817,
"loss_nats": 1.5513701438903809,
"step": 990,
"unclipped_grad_norm": 0.7972294092178345
},
{
"bits_per_byte": 2.255509814368321,
"learning_rate": 0.00029586903590180956,
"loss_nats": 1.5634002685546875,
"step": 1000,
"unclipped_grad_norm": 0.7948418855667114
},
{
"bits_per_byte": 2.249676162851055,
"learning_rate": 0.00029573092598683735,
"loss_nats": 1.559356689453125,
"step": 1010,
"unclipped_grad_norm": 0.9423647522926331
},
{
"bits_per_byte": 2.2157856056373317,
"learning_rate": 0.00029559058224103895,
"loss_nats": 1.5358655452728271,
"step": 1020,
"unclipped_grad_norm": 1.3369964361190796
},
{
"bits_per_byte": 2.21131629248936,
"learning_rate": 0.0002954480070625017,
"loss_nats": 1.532767653465271,
"step": 1030,
"unclipped_grad_norm": 0.7792882919311523
},
{
"bits_per_byte": 2.300165797595453,
"learning_rate": 0.00029530320288744205,
"loss_nats": 1.594353437423706,
"step": 1040,
"unclipped_grad_norm": 0.9419035911560059
},
{
"bits_per_byte": 2.21666237319143,
"learning_rate": 0.0002951561721901638,
"loss_nats": 1.536473274230957,
"step": 1050,
"unclipped_grad_norm": 0.9680492281913757
},
{
"bits_per_byte": 2.242138163263738,
"learning_rate": 0.00029500691748301594,
"loss_nats": 1.5541317462921143,
"step": 1060,
"unclipped_grad_norm": 0.9887726306915283
},
{
"bits_per_byte": 2.276496685270459,
"learning_rate": 0.00029485544131634963,
"loss_nats": 1.5779472589492798,
"step": 1070,
"unclipped_grad_norm": 0.8514108657836914
},
{
"bits_per_byte": 2.2430374605450996,
"learning_rate": 0.0002947017462784745,
"loss_nats": 1.5547550916671753,
"step": 1080,
"unclipped_grad_norm": 0.800697386264801
},
{
"bits_per_byte": 2.1823284447037823,
"learning_rate": 0.00029454583499561476,
"loss_nats": 1.5126748085021973,
"step": 1090,
"unclipped_grad_norm": 0.9124817848205566
},
{
"bits_per_byte": 2.1899661942286,
"learning_rate": 0.00029438771013186413,
"loss_nats": 1.5179688930511475,
"step": 1100,
"unclipped_grad_norm": 0.8022184371948242
},
{
"bits_per_byte": 2.145753238310091,
"learning_rate": 0.0002942273743891401,
"loss_nats": 1.4873228073120117,
"step": 1110,
"unclipped_grad_norm": 1.139209508895874
},
{
"bits_per_byte": 2.188020210534087,
"learning_rate": 0.00029406483050713824,
"loss_nats": 1.5166200399398804,
"step": 1120,
"unclipped_grad_norm": 0.8927217721939087
},
{
"bits_per_byte": 2.2131649340034842,
"learning_rate": 0.00029390008126328504,
"loss_nats": 1.5340490341186523,
"step": 1130,
"unclipped_grad_norm": 0.8271659016609192
},
{
"bits_per_byte": 2.1401269978697646,
"learning_rate": 0.00029373312947269045,
"loss_nats": 1.4834229946136475,
"step": 1140,
"unclipped_grad_norm": 0.9781836271286011
},
{
"bits_per_byte": 2.1955361963420934,
"learning_rate": 0.00029356397798809995,
"loss_nats": 1.5218297243118286,
"step": 1150,
"unclipped_grad_norm": 0.8088869452476501
},
{
"bits_per_byte": 2.103003854750424,
"learning_rate": 0.0002933926296998457,
"loss_nats": 1.4576911926269531,
"step": 1160,
"unclipped_grad_norm": 0.761218786239624
},
{
"bits_per_byte": 2.164645876472672,
"learning_rate": 0.0002932190875357972,
"loss_nats": 1.5004181861877441,
"step": 1170,
"unclipped_grad_norm": 0.8793766498565674
},
{
"bits_per_byte": 2.159191274717907,
"learning_rate": 0.00029304335446131126,
"loss_nats": 1.4966373443603516,
"step": 1180,
"unclipped_grad_norm": 0.8019644618034363
},
{
"bits_per_byte": 2.134348724766076,
"learning_rate": 0.00029286543347918116,
"loss_nats": 1.4794178009033203,
"step": 1190,
"unclipped_grad_norm": 0.7809260487556458
},
{
"bits_per_byte": 2.1456660431061034,
"learning_rate": 0.0002926853276295856,
"loss_nats": 1.4872623682022095,
"step": 1200,
"unclipped_grad_norm": 0.896268904209137
},
{
"bits_per_byte": 2.1662845271700926,
"learning_rate": 0.0002925030399900368,
"loss_nats": 1.501554012298584,
"step": 1210,
"unclipped_grad_norm": 0.8574862480163574
},
{
"bits_per_byte": 2.1378967268533757,
"learning_rate": 0.00029231857367532726,
"loss_nats": 1.481877088546753,
"step": 1220,
"unclipped_grad_norm": 0.8502950668334961
},
{
"bits_per_byte": 2.128576127089866,
"learning_rate": 0.0002921319318374776,
"loss_nats": 1.4754165410995483,
"step": 1230,
"unclipped_grad_norm": 0.7498336434364319
},
{
"bits_per_byte": 2.0970609942498597,
"learning_rate": 0.00029194311766568174,
"loss_nats": 1.4535719156265259,
"step": 1240,
"unclipped_grad_norm": 0.761203944683075
},
{
"bits_per_byte": 2.0385041693014396,
"learning_rate": 0.000291752134386253,
"loss_nats": 1.4129834175109863,
"step": 1250,
"unclipped_grad_norm": 0.7150603532791138
},
{
"bits_per_byte": 2.0885559362167414,
"learning_rate": 0.00029155898526256856,
"loss_nats": 1.447676658630371,
"step": 1260,
"unclipped_grad_norm": 0.7949564456939697
},
{
"bits_per_byte": 2.1179862953629884,
"learning_rate": 0.0002913636735950141,
"loss_nats": 1.468076229095459,
"step": 1270,
"unclipped_grad_norm": 0.862070620059967
},
{
"bits_per_byte": 2.1475409979658098,
"learning_rate": 0.0002911662027209272,
"loss_nats": 1.488561987876892,
"step": 1280,
"unclipped_grad_norm": 0.7539991140365601
},
{
"bits_per_byte": 2.1029943957046267,
"learning_rate": 0.00029096657601454013,
"loss_nats": 1.4576846361160278,
"step": 1290,
"unclipped_grad_norm": 0.8074342608451843
},
{
"bits_per_byte": 2.049033979083166,
"learning_rate": 0.00029076479688692266,
"loss_nats": 1.4202821254730225,
"step": 1300,
"unclipped_grad_norm": 0.7586180567741394
},
{
"bits_per_byte": 2.076369761524555,
"learning_rate": 0.00029056086878592323,
"loss_nats": 1.4392298460006714,
"step": 1310,
"unclipped_grad_norm": 0.9284558892250061
},
{
"bits_per_byte": 2.051258058724124,
"learning_rate": 0.00029035479519611045,
"loss_nats": 1.4218237400054932,
"step": 1320,
"unclipped_grad_norm": 0.7597900032997131
},
{
"bits_per_byte": 2.060438148661879,
"learning_rate": 0.0002901465796387134,
"loss_nats": 1.4281868934631348,
"step": 1330,
"unclipped_grad_norm": 0.7334625720977783
},
{
"bits_per_byte": 2.04908454198252,
"learning_rate": 0.00028993622567156154,
"loss_nats": 1.4203171730041504,
"step": 1340,
"unclipped_grad_norm": 0.7958364486694336
},
{
"bits_per_byte": 2.0721673654507224,
"learning_rate": 0.0002897237368890237,
"loss_nats": 1.436316967010498,
"step": 1350,
"unclipped_grad_norm": 0.7738369107246399
},
{
"bits_per_byte": 2.034246910761982,
"learning_rate": 0.0002895091169219469,
"loss_nats": 1.4100325107574463,
"step": 1360,
"unclipped_grad_norm": 0.8466383218765259
},
{
"bits_per_byte": 2.0554857362476335,
"learning_rate": 0.00028929236943759425,
"loss_nats": 1.4247541427612305,
"step": 1370,
"unclipped_grad_norm": 0.7919691205024719
},
{
"bits_per_byte": 2.020279167789508,
"learning_rate": 0.00028907349813958225,
"loss_nats": 1.40035080909729,
"step": 1380,
"unclipped_grad_norm": 0.8231279850006104
},
{
"bits_per_byte": 2.070644631059983,
"learning_rate": 0.0002888525067678175,
"loss_nats": 1.4352614879608154,
"step": 1390,
"unclipped_grad_norm": 0.7695969343185425
},
{
"bits_per_byte": 2.174483628067326,
"learning_rate": 0.00028862939909843273,
"loss_nats": 1.507237195968628,
"step": 1400,
"unclipped_grad_norm": 0.8150069713592529
},
{
"bits_per_byte": 2.0594870846026074,
"learning_rate": 0.0002884041789437225,
"loss_nats": 1.427527666091919,
"step": 1410,
"unclipped_grad_norm": 0.7077280879020691
},
{
"bits_per_byte": 2.048123158964197,
"learning_rate": 0.0002881768501520778,
"loss_nats": 1.4196507930755615,
"step": 1420,
"unclipped_grad_norm": 0.7647379040718079
},
{
"bits_per_byte": 2.0019392818668,
"learning_rate": 0.0002879474166079205,
"loss_nats": 1.3876385688781738,
"step": 1430,
"unclipped_grad_norm": 0.6688193082809448
},
{
"bits_per_byte": 2.0332329010524948,
"learning_rate": 0.00028771588223163694,
"loss_nats": 1.4093296527862549,
"step": 1440,
"unclipped_grad_norm": 0.8427934646606445
},
{
"bits_per_byte": 2.0070671165849254,
"learning_rate": 0.00028748225097951054,
"loss_nats": 1.39119291305542,
"step": 1450,
"unclipped_grad_norm": 0.6920660138130188
},
{
"bits_per_byte": 2.1256969655317715,
"learning_rate": 0.000287246526843655,
"loss_nats": 1.4734208583831787,
"step": 1460,
"unclipped_grad_norm": 0.799986720085144
},
{
"bits_per_byte": 2.0930381480635276,
"learning_rate": 0.00028700871385194533,
"loss_nats": 1.4507834911346436,
"step": 1470,
"unclipped_grad_norm": 0.6732411980628967
},
{
"bits_per_byte": 2.0451248134290543,
"learning_rate": 0.00028676881606794945,
"loss_nats": 1.4175724983215332,
"step": 1480,
"unclipped_grad_norm": 0.6982382535934448
},
{
"bits_per_byte": 2.0566264971708064,
"learning_rate": 0.00028652683759085854,
"loss_nats": 1.4255448579788208,
"step": 1490,
"unclipped_grad_norm": 0.7266837358474731
},
{
"bits_per_byte": 1.9341151717766374,
"learning_rate": 0.0002862827825554171,
"loss_nats": 1.3406264781951904,
"step": 1500,
"unclipped_grad_norm": 0.6626343727111816
},
{
"bits_per_byte": 1.963616731740342,
"learning_rate": 0.00028603665513185245,
"loss_nats": 1.3610754013061523,
"step": 1510,
"unclipped_grad_norm": 0.789225697517395
},
{
"bits_per_byte": 2.0151121210269856,
"learning_rate": 0.000285788459525803,
"loss_nats": 1.3967692852020264,
"step": 1520,
"unclipped_grad_norm": 0.7025586366653442
},
{
"bits_per_byte": 2.0371694119480934,
"learning_rate": 0.00028553819997824707,
"loss_nats": 1.4120582342147827,
"step": 1530,
"unclipped_grad_norm": 0.750766396522522
},
{
"bits_per_byte": 2.0209939276864937,
"learning_rate": 0.00028528588076542964,
"loss_nats": 1.400846242904663,
"step": 1540,
"unclipped_grad_norm": 0.7461745738983154
},
{
"bits_per_byte": 2.082936231117148,
"learning_rate": 0.00028503150619878996,
"loss_nats": 1.4437813758850098,
"step": 1550,
"unclipped_grad_norm": 0.765493631362915
},
{
"bits_per_byte": 1.954267066908836,
"learning_rate": 0.0002847750806248876,
"loss_nats": 1.3545947074890137,
"step": 1560,
"unclipped_grad_norm": 0.727699875831604
},
{
"bits_per_byte": 1.9578143810655324,
"learning_rate": 0.000284516608425328,
"loss_nats": 1.357053518295288,
"step": 1570,
"unclipped_grad_norm": 0.7358320951461792
},
{
"bits_per_byte": 1.9655836973182597,
"learning_rate": 0.000284256094016688,
"loss_nats": 1.3624387979507446,
"step": 1580,
"unclipped_grad_norm": 0.6977834701538086
},
{
"bits_per_byte": 1.9752479184181935,
"learning_rate": 0.0002839935418504399,
"loss_nats": 1.3691375255584717,
"step": 1590,
"unclipped_grad_norm": 0.6583591103553772
},
{
"bits_per_byte": 1.9806421542625054,
"learning_rate": 0.000283728956412876,
"loss_nats": 1.372876524925232,
"step": 1600,
"unclipped_grad_norm": 0.727998673915863
},
{
"bits_per_byte": 1.9474555220387637,
"learning_rate": 0.00028346234222503136,
"loss_nats": 1.3498733043670654,
"step": 1610,
"unclipped_grad_norm": 0.7535929083824158
},
{
"bits_per_byte": 1.942356580405983,
"learning_rate": 0.00028319370384260675,
"loss_nats": 1.3463389873504639,
"step": 1620,
"unclipped_grad_norm": 0.6403960585594177
},
{
"bits_per_byte": 1.9826001767425787,
"learning_rate": 0.00028292304585589105,
"loss_nats": 1.3742337226867676,
"step": 1630,
"unclipped_grad_norm": 0.7281255125999451
},
{
"bits_per_byte": 1.9377553565647996,
"learning_rate": 0.0002826503728896824,
"loss_nats": 1.3431496620178223,
"step": 1640,
"unclipped_grad_norm": 0.7114111185073853
},
{
"bits_per_byte": 1.9278453722567828,
"learning_rate": 0.0002823756896032095,
"loss_nats": 1.3362805843353271,
"step": 1650,
"unclipped_grad_norm": 0.6373128890991211
},
{
"bits_per_byte": 1.9498221752972869,
"learning_rate": 0.0002820990006900518,
"loss_nats": 1.3515137434005737,
"step": 1660,
"unclipped_grad_norm": 0.6859769225120544
},
{
"bits_per_byte": 1.9095433225172644,
"learning_rate": 0.00028182031087805947,
"loss_nats": 1.323594570159912,
"step": 1670,
"unclipped_grad_norm": 0.7934040427207947
},
{
"bits_per_byte": 1.938967146322779,
"learning_rate": 0.0002815396249292725,
"loss_nats": 1.343989610671997,
"step": 1680,
"unclipped_grad_norm": 0.7104787230491638
},
{
"bits_per_byte": 1.9591448388526072,
"learning_rate": 0.0002812569476398391,
"loss_nats": 1.357975721359253,
"step": 1690,
"unclipped_grad_norm": 0.6892858743667603
},
{
"bits_per_byte": 1.9072468381802934,
"learning_rate": 0.00028097228383993446,
"loss_nats": 1.3220027685165405,
"step": 1700,
"unclipped_grad_norm": 0.7498061656951904
},
{
"bits_per_byte": 1.9434417909329273,
"learning_rate": 0.0002806856383936774,
"loss_nats": 1.3470911979675293,
"step": 1710,
"unclipped_grad_norm": 0.6478108763694763
},
{
"bits_per_byte": 1.8728540916260505,
"learning_rate": 0.00028039701619904774,
"loss_nats": 1.2981635332107544,
"step": 1720,
"unclipped_grad_norm": 0.7232697010040283
},
{
"bits_per_byte": 1.9589699324966796,
"learning_rate": 0.0002801064221878024,
"loss_nats": 1.3578544855117798,
"step": 1730,
"unclipped_grad_norm": 0.7008518576622009
},
{
"bits_per_byte": 1.9590435410712488,
"learning_rate": 0.00027981386132539134,
"loss_nats": 1.3579055070877075,
"step": 1740,
"unclipped_grad_norm": 0.6955974698066711
},
{
"bits_per_byte": 1.919930042733473,
"learning_rate": 0.0002795193386108724,
"loss_nats": 1.330794095993042,
"step": 1750,
"unclipped_grad_norm": 0.6556993126869202
},
{
"bits_per_byte": 1.9569465566092785,
"learning_rate": 0.00027922285907682625,
"loss_nats": 1.3564519882202148,
"step": 1760,
"unclipped_grad_norm": 0.698861837387085
},
{
"bits_per_byte": 1.9512123110642108,
"learning_rate": 0.00027892442778927005,
"loss_nats": 1.3524773120880127,
"step": 1770,
"unclipped_grad_norm": 0.6838504672050476
},
{
"bits_per_byte": 2.0216474617597724,
"learning_rate": 0.00027862404984757116,
"loss_nats": 1.401299238204956,
"step": 1780,
"unclipped_grad_norm": 0.7214158773422241
},
{
"bits_per_byte": 1.8826208143858987,
"learning_rate": 0.0002783217303843597,
"loss_nats": 1.3049333095550537,
"step": 1790,
"unclipped_grad_norm": 0.7236432433128357
},
{
"bits_per_byte": 2.013814339943575,
"learning_rate": 0.00027801747456544134,
"loss_nats": 1.3958697319030762,
"step": 1800,
"unclipped_grad_norm": 0.7153979539871216
},
{
"bits_per_byte": 1.8985195785622602,
"learning_rate": 0.00027771128758970843,
"loss_nats": 1.3159534931182861,
"step": 1810,
"unclipped_grad_norm": 0.7165744304656982
},
{
"bits_per_byte": 1.956814645916067,
"learning_rate": 0.00027740317468905165,
"loss_nats": 1.3563605546951294,
"step": 1820,
"unclipped_grad_norm": 0.7207518815994263
},
{
"bits_per_byte": 1.973846947744265,
"learning_rate": 0.0002770931411282704,
"loss_nats": 1.368166446685791,
"step": 1830,
"unclipped_grad_norm": 0.6764873266220093
},
{
"bits_per_byte": 1.953527541510126,
"learning_rate": 0.0002767811922049827,
"loss_nats": 1.3540821075439453,
"step": 1840,
"unclipped_grad_norm": 0.6510032415390015
},
{
"bits_per_byte": 1.8838981295338566,
"learning_rate": 0.00027646733324953503,
"loss_nats": 1.3058186769485474,
"step": 1850,
"unclipped_grad_norm": 0.6467540860176086
},
{
"bits_per_byte": 1.9116946534969073,
"learning_rate": 0.00027615156962491097,
"loss_nats": 1.3250857591629028,
"step": 1860,
"unclipped_grad_norm": 0.6336910128593445
},
{
"bits_per_byte": 1.996913260877985,
"learning_rate": 0.0002758339067266396,
"loss_nats": 1.3841547966003418,
"step": 1870,
"unclipped_grad_norm": 0.7375073432922363
},
{
"bits_per_byte": 1.8879835774051028,
"learning_rate": 0.0002755143499827035,
"loss_nats": 1.3086504936218262,
"step": 1880,
"unclipped_grad_norm": 0.6048603057861328
},
{
"bits_per_byte": 1.9029378128629257,
"learning_rate": 0.0002751929048534456,
"loss_nats": 1.3190159797668457,
"step": 1890,
"unclipped_grad_norm": 0.6331550478935242
},
{
"bits_per_byte": 1.9156196415548983,
"learning_rate": 0.0002748695768314763,
"loss_nats": 1.3278063535690308,
"step": 1900,
"unclipped_grad_norm": 0.7952298521995544
},
{
"bits_per_byte": 1.9644835243006904,
"learning_rate": 0.00027454437144157947,
"loss_nats": 1.3616762161254883,
"step": 1910,
"unclipped_grad_norm": 0.6600897312164307
},
{
"bits_per_byte": 1.8776484519841532,
"learning_rate": 0.00027421729424061787,
"loss_nats": 1.3014867305755615,
"step": 1920,
"unclipped_grad_norm": 0.645132303237915
},
{
"bits_per_byte": 1.9490762865404947,
"learning_rate": 0.0002738883508174385,
"loss_nats": 1.350996732711792,
"step": 1930,
"unclipped_grad_norm": 0.6212714314460754
},
{
"bits_per_byte": 1.942025857768374,
"learning_rate": 0.0002735575467927768,
"loss_nats": 1.3461097478866577,
"step": 1940,
"unclipped_grad_norm": 0.6636208295822144
},
{
"bits_per_byte": 1.8837596834999122,
"learning_rate": 0.0002732248878191609,
"loss_nats": 1.305722713470459,
"step": 1950,
"unclipped_grad_norm": 0.7458274960517883
},
{
"bits_per_byte": 1.8770238109962194,
"learning_rate": 0.0002728903795808148,
"loss_nats": 1.301053762435913,
"step": 1960,
"unclipped_grad_norm": 0.6796664595603943
},
{
"bits_per_byte": 1.873630765277347,
"learning_rate": 0.0002725540277935614,
"loss_nats": 1.2987018823623657,
"step": 1970,
"unclipped_grad_norm": 0.7089046239852905
},
{
"bits_per_byte": 1.8950018454215123,
"learning_rate": 0.00027221583820472456,
"loss_nats": 1.3135151863098145,
"step": 1980,
"unclipped_grad_norm": 0.6426982283592224
},
{
"bits_per_byte": 1.8681299001895,
"learning_rate": 0.0002718758165930314,
"loss_nats": 1.294888973236084,
"step": 1990,
"unclipped_grad_norm": 0.7099292278289795
},
{
"bits_per_byte": 1.964639340582372,
"learning_rate": 0.0002715339687685131,
"loss_nats": 1.3617842197418213,
"step": 2000,
"unclipped_grad_norm": 0.7598643898963928
},
{
"bits_per_byte": 1.939805561745735,
"learning_rate": 0.00027119030057240576,
"loss_nats": 1.3445707559585571,
"step": 2010,
"unclipped_grad_norm": 0.6349508166313171
},
{
"bits_per_byte": 1.851296754271001,
"learning_rate": 0.0002708448178770506,
"loss_nats": 1.2832211256027222,
"step": 2020,
"unclipped_grad_norm": 0.6008884310722351
},
{
"bits_per_byte": 1.890781391169339,
"learning_rate": 0.00027049752658579376,
"loss_nats": 1.3105897903442383,
"step": 2030,
"unclipped_grad_norm": 0.637248158454895
},
{
"bits_per_byte": 1.882627693691933,
"learning_rate": 0.00027014843263288514,
"loss_nats": 1.3049380779266357,
"step": 2040,
"unclipped_grad_norm": 0.6375738382339478
},
{
"bits_per_byte": 1.848167701921203,
"learning_rate": 0.00026979754198337724,
"loss_nats": 1.2810522317886353,
"step": 2050,
"unclipped_grad_norm": 0.6515361070632935
},
{
"bits_per_byte": 1.833815233758749,
"learning_rate": 0.00026944486063302305,
"loss_nats": 1.271103858947754,
"step": 2060,
"unclipped_grad_norm": 0.6752470135688782
},
{
"bits_per_byte": 1.960125139962525,
"learning_rate": 0.0002690903946081738,
"loss_nats": 1.3586552143096924,
"step": 2070,
"unclipped_grad_norm": 0.7093353867530823
},
{
"bits_per_byte": 1.93064937739645,
"learning_rate": 0.00026873414996567596,
"loss_nats": 1.338224172592163,
"step": 2080,
"unclipped_grad_norm": 0.7233569622039795
},
{
"bits_per_byte": 1.8411410067549012,
"learning_rate": 0.00026837613279276736,
"loss_nats": 1.276181697845459,
"step": 2090,
"unclipped_grad_norm": 0.6208991408348083
},
{
"bits_per_byte": 1.7919973082361553,
"learning_rate": 0.0002680163492069739,
"loss_nats": 1.2421178817749023,
"step": 2100,
"unclipped_grad_norm": 0.6259884834289551
},
{
"bits_per_byte": 1.830389339353562,
"learning_rate": 0.00026765480535600414,
"loss_nats": 1.2687292098999023,
"step": 2110,
"unclipped_grad_norm": 0.633368968963623
},
{
"bits_per_byte": 1.8446745622995284,
"learning_rate": 0.0002672915074176452,
"loss_nats": 1.2786309719085693,
"step": 2120,
"unclipped_grad_norm": 0.6770470142364502
},
{
"bits_per_byte": 1.8875441617321482,
"learning_rate": 0.00026692646159965624,
"loss_nats": 1.308345913887024,
"step": 2130,
"unclipped_grad_norm": 0.7079503536224365
},
{
"bits_per_byte": 1.912643825747019,
"learning_rate": 0.0002665596741396633,
"loss_nats": 1.3257436752319336,
"step": 2140,
"unclipped_grad_norm": 0.6220520734786987
},
{
"bits_per_byte": 1.8089860984536854,
"learning_rate": 0.000266191151305052,
"loss_nats": 1.2538936138153076,
"step": 2150,
"unclipped_grad_norm": 0.6506339907646179
},
{
"bits_per_byte": 1.9026210208200363,
"learning_rate": 0.00026582089939286075,
"loss_nats": 1.3187963962554932,
"step": 2160,
"unclipped_grad_norm": 0.6963920593261719
},
{
"bits_per_byte": 1.8293292382936435,
"learning_rate": 0.0002654489247296732,
"loss_nats": 1.2679944038391113,
"step": 2170,
"unclipped_grad_norm": 0.6530651450157166
},
{
"bits_per_byte": 1.90049290749826,
"learning_rate": 0.00026507523367151007,
"loss_nats": 1.3173213005065918,
"step": 2180,
"unclipped_grad_norm": 0.6189562678337097
},
{
"bits_per_byte": 1.8805790363548553,
"learning_rate": 0.00026469983260372037,
"loss_nats": 1.3035180568695068,
"step": 2190,
"unclipped_grad_norm": 0.6570361852645874
},
{
"bits_per_byte": 1.8344993807438814,
"learning_rate": 0.0002643227279408727,
"loss_nats": 1.271578073501587,
"step": 2200,
"unclipped_grad_norm": 0.5929612517356873
},
{
"bits_per_byte": 1.837753980428809,
"learning_rate": 0.0002639439261266453,
"loss_nats": 1.273833990097046,
"step": 2210,
"unclipped_grad_norm": 0.6836420893669128
},
{
"bits_per_byte": 1.8346952689832141,
"learning_rate": 0.000263563433633716,
"loss_nats": 1.2717138528823853,
"step": 2220,
"unclipped_grad_norm": 0.6392546892166138
},
{
"bits_per_byte": 1.7850062134785822,
"learning_rate": 0.00026318125696365156,
"loss_nats": 1.237272024154663,
"step": 2230,
"unclipped_grad_norm": 0.5976205468177795
},
{
"bits_per_byte": 1.8807291772090586,
"learning_rate": 0.000262797402646797,
"loss_nats": 1.3036221265792847,
"step": 2240,
"unclipped_grad_norm": 0.6432849168777466
},
{
"bits_per_byte": 1.861241479074492,
"learning_rate": 0.00026241187724216334,
"loss_nats": 1.2901142835617065,
"step": 2250,
"unclipped_grad_norm": 0.6619172096252441
},
{
"bits_per_byte": 1.9088667427687702,
"learning_rate": 0.0002620246873373162,
"loss_nats": 1.3231256008148193,
"step": 2260,
"unclipped_grad_norm": 0.6604200601577759
},
{
"bits_per_byte": 1.8085772956925845,
"learning_rate": 0.00026163583954826264,
"loss_nats": 1.2536102533340454,
"step": 2270,
"unclipped_grad_norm": 0.6277530193328857
},
{
"bits_per_byte": 1.8640159031982109,
"learning_rate": 0.0002612453405193386,
"loss_nats": 1.2920373678207397,
"step": 2280,
"unclipped_grad_norm": 0.6375154256820679
},
{
"bits_per_byte": 1.8154523021608253,
"learning_rate": 0.000260853196923095,
"loss_nats": 1.258375644683838,
"step": 2290,
"unclipped_grad_norm": 0.6188551187515259
},
{
"bits_per_byte": 1.8127673090155552,
"learning_rate": 0.0002604594154601839,
"loss_nats": 1.256514549255371,
"step": 2300,
"unclipped_grad_norm": 0.6347740292549133
},
{
"bits_per_byte": 1.811916166876435,
"learning_rate": 0.0002600640028592441,
"loss_nats": 1.2559245824813843,
"step": 2310,
"unclipped_grad_norm": 0.621824324131012
},
{
"bits_per_byte": 1.866308603916863,
"learning_rate": 0.0002596669658767859,
"loss_nats": 1.2936265468597412,
"step": 2320,
"unclipped_grad_norm": 0.6393704414367676
},
{
"bits_per_byte": 1.8101679632304146,
"learning_rate": 0.000259268311297076,
"loss_nats": 1.2547128200531006,
"step": 2330,
"unclipped_grad_norm": 0.6721166372299194
},
{
"bits_per_byte": 1.82719372971788,
"learning_rate": 0.00025886804593202114,
"loss_nats": 1.2665141820907593,
"step": 2340,
"unclipped_grad_norm": 0.646355152130127
},
{
"bits_per_byte": 1.8870428323048831,
"learning_rate": 0.00025846617662105193,
"loss_nats": 1.3079984188079834,
"step": 2350,
"unclipped_grad_norm": 0.6942237019538879
},
{
"bits_per_byte": 1.804568896048925,
"learning_rate": 0.00025806271023100637,
"loss_nats": 1.2508318424224854,
"step": 2360,
"unclipped_grad_norm": 0.6387670040130615
},
{
"bits_per_byte": 1.739788190948431,
"learning_rate": 0.00025765765365601165,
"loss_nats": 1.2059292793273926,
"step": 2370,
"unclipped_grad_norm": 0.6083642840385437
},
{
"bits_per_byte": 1.8484354789085964,
"learning_rate": 0.00025725101381736707,
"loss_nats": 1.2812378406524658,
"step": 2380,
"unclipped_grad_norm": 0.6751434803009033
},
{
"bits_per_byte": 1.8404750899307605,
"learning_rate": 0.00025684279766342567,
"loss_nats": 1.2757201194763184,
"step": 2390,
"unclipped_grad_norm": 0.6037119626998901
},
{
"bits_per_byte": 1.8757652419572053,
"learning_rate": 0.00025643301216947504,
"loss_nats": 1.3001813888549805,
"step": 2400,
"unclipped_grad_norm": 0.6957288980484009
},
{
"bits_per_byte": 1.801620425482533,
"learning_rate": 0.00025602166433761866,
"loss_nats": 1.2487881183624268,
"step": 2410,
"unclipped_grad_norm": 0.626511812210083
},
{
"bits_per_byte": 1.8236299052267009,
"learning_rate": 0.0002556087611966561,
"loss_nats": 1.264043927192688,
"step": 2420,
"unclipped_grad_norm": 0.6481806635856628
},
{
"bits_per_byte": 1.8322460640522769,
"learning_rate": 0.00025519430980196277,
"loss_nats": 1.2700161933898926,
"step": 2430,
"unclipped_grad_norm": 0.634190559387207
},
{
"bits_per_byte": 1.778641823500753,
"learning_rate": 0.0002547783172353695,
"loss_nats": 1.2328605651855469,
"step": 2440,
"unclipped_grad_norm": 0.6495575308799744
},
{
"bits_per_byte": 1.9154751761281736,
"learning_rate": 0.0002543607906050415,
"loss_nats": 1.327706217765808,
"step": 2450,
"unclipped_grad_norm": 0.682089626789093
},
{
"bits_per_byte": 1.7848875454494868,
"learning_rate": 0.0002539417370453568,
"loss_nats": 1.237189769744873,
"step": 2460,
"unclipped_grad_norm": 0.6699327230453491
},
{
"bits_per_byte": 1.784353367335907,
"learning_rate": 0.00025352116371678464,
"loss_nats": 1.2368195056915283,
"step": 2470,
"unclipped_grad_norm": 0.6535210609436035
},
{
"bits_per_byte": 1.865447314801342,
"learning_rate": 0.0002530990778057627,
"loss_nats": 1.293029546737671,
"step": 2480,
"unclipped_grad_norm": 0.6640622019767761
},
{
"bits_per_byte": 1.857090849778569,
"learning_rate": 0.00025267548652457475,
"loss_nats": 1.287237286567688,
"step": 2490,
"unclipped_grad_norm": 0.6223002076148987
},
{
"bits_per_byte": 1.8122928088818246,
"learning_rate": 0.00025225039711122696,
"loss_nats": 1.2561856508255005,
"step": 2500,
"unclipped_grad_norm": 0.6052747368812561
},
{
"bits_per_byte": 1.76881026328153,
"learning_rate": 0.00025182381682932464,
"loss_nats": 1.226045846939087,
"step": 2510,
"unclipped_grad_norm": 0.6596174836158752
},
{
"bits_per_byte": 1.8006248179166882,
"learning_rate": 0.0002513957529679477,
"loss_nats": 1.2480980157852173,
"step": 2520,
"unclipped_grad_norm": 0.6490770578384399
},
{
"bits_per_byte": 1.8232907554391995,
"learning_rate": 0.00025096621284152656,
"loss_nats": 1.2638088464736938,
"step": 2530,
"unclipped_grad_norm": 0.6303134560585022
},
{
"bits_per_byte": 1.8125131186575798,
"learning_rate": 0.0002505352037897168,
"loss_nats": 1.256338357925415,
"step": 2540,
"unclipped_grad_norm": 0.6458812952041626
},
{
"bits_per_byte": 1.7674137641565237,
"learning_rate": 0.0002501027331772739,
"loss_nats": 1.2250778675079346,
"step": 2550,
"unclipped_grad_norm": 0.6546228528022766
},
{
"bits_per_byte": 1.7342415784754543,
"learning_rate": 0.00024966880839392746,
"loss_nats": 1.2020846605300903,
"step": 2560,
"unclipped_grad_norm": 0.602087676525116
},
{
"bits_per_byte": 1.774496353684356,
"learning_rate": 0.0002492334368542548,
"loss_nats": 1.2299871444702148,
"step": 2570,
"unclipped_grad_norm": 0.6244005560874939
},
{
"bits_per_byte": 1.762712446412538,
"learning_rate": 0.00024879662599755416,
"loss_nats": 1.2218191623687744,
"step": 2580,
"unclipped_grad_norm": 0.6090607643127441
},
{
"bits_per_byte": 1.7589357073995908,
"learning_rate": 0.00024835838328771803,
"loss_nats": 1.2192013263702393,
"step": 2590,
"unclipped_grad_norm": 0.6631695032119751
},
{
"bits_per_byte": 1.7746397872151756,
"learning_rate": 0.0002479187162131051,
"loss_nats": 1.2300865650177002,
"step": 2600,
"unclipped_grad_norm": 0.6302726864814758
},
{
"bits_per_byte": 1.8022297600145398,
"learning_rate": 0.00024747763228641264,
"loss_nats": 1.2492104768753052,
"step": 2610,
"unclipped_grad_norm": 0.6297239065170288
},
{
"bits_per_byte": 1.7449476704743152,
"learning_rate": 0.00024703513904454795,
"loss_nats": 1.209505558013916,
"step": 2620,
"unclipped_grad_norm": 0.617965042591095
},
{
"bits_per_byte": 1.738682342503383,
"learning_rate": 0.00024659124404849956,
"loss_nats": 1.205162763595581,
"step": 2630,
"unclipped_grad_norm": 0.6048376560211182
},
{
"bits_per_byte": 1.7097366304672672,
"learning_rate": 0.00024614595488320833,
"loss_nats": 1.1850991249084473,
"step": 2640,
"unclipped_grad_norm": 0.5914375185966492
},
{
"bits_per_byte": 1.790221415383346,
"learning_rate": 0.00024569927915743745,
"loss_nats": 1.240886926651001,
"step": 2650,
"unclipped_grad_norm": 0.6316050887107849
},
{
"bits_per_byte": 1.7080907564985102,
"learning_rate": 0.0002452512245036427,
"loss_nats": 1.1839582920074463,
"step": 2660,
"unclipped_grad_norm": 0.6427422761917114
},
{
"bits_per_byte": 1.776176280217984,
"learning_rate": 0.00024480179857784177,
"loss_nats": 1.2311515808105469,
"step": 2670,
"unclipped_grad_norm": 0.695833146572113
},
{
"bits_per_byte": 1.8751694940546166,
"learning_rate": 0.00024435100905948387,
"loss_nats": 1.2997684478759766,
"step": 2680,
"unclipped_grad_norm": 0.7215304374694824
},
{
"bits_per_byte": 1.7935356930481232,
"learning_rate": 0.00024389886365131793,
"loss_nats": 1.243184208869934,
"step": 2690,
"unclipped_grad_norm": 0.6506277322769165
},
{
"bits_per_byte": 1.7071520751901008,
"learning_rate": 0.00024344537007926163,
"loss_nats": 1.1833076477050781,
"step": 2700,
"unclipped_grad_norm": 0.6119946241378784
},
{
"bits_per_byte": 1.7912357690581349,
"learning_rate": 0.00024299053609226878,
"loss_nats": 1.2415900230407715,
"step": 2710,
"unclipped_grad_norm": 0.6407736539840698
},
{
"bits_per_byte": 1.8449512823847667,
"learning_rate": 0.00024253436946219733,
"loss_nats": 1.2788227796554565,
"step": 2720,
"unclipped_grad_norm": 0.699180543422699
},
{
"bits_per_byte": 1.7188499911364874,
"learning_rate": 0.0002420768779836764,
"loss_nats": 1.1914160251617432,
"step": 2730,
"unclipped_grad_norm": 0.6055090427398682
},
{
"bits_per_byte": 1.7885469922945458,
"learning_rate": 0.00024161806947397305,
"loss_nats": 1.2397263050079346,
"step": 2740,
"unclipped_grad_norm": 0.6273350715637207
},
{
"bits_per_byte": 1.7444217475279766,
"learning_rate": 0.00024115795177285886,
"loss_nats": 1.2091410160064697,
"step": 2750,
"unclipped_grad_norm": 0.623899519443512
},
{
"bits_per_byte": 1.855314440977807,
"learning_rate": 0.0002406965327424758,
"loss_nats": 1.286005973815918,
"step": 2760,
"unclipped_grad_norm": 0.6717104315757751
},
{
"bits_per_byte": 1.79349716893433,
"learning_rate": 0.00024023382026720197,
"loss_nats": 1.2431575059890747,
"step": 2770,
"unclipped_grad_norm": 0.6304712891578674
},
{
"bits_per_byte": 1.8011758503300526,
"learning_rate": 0.00023976982225351698,
"loss_nats": 1.248479962348938,
"step": 2780,
"unclipped_grad_norm": 0.6547600626945496
},
{
"bits_per_byte": 1.7772979510669111,
"learning_rate": 0.0002393045466298666,
"loss_nats": 1.231929063796997,
"step": 2790,
"unclipped_grad_norm": 0.6679553389549255
},
{
"bits_per_byte": 1.784543924113063,
"learning_rate": 0.00023883800134652763,
"loss_nats": 1.2369515895843506,
"step": 2800,
"unclipped_grad_norm": 0.6488308310508728
},
{
"bits_per_byte": 1.723108969502454,
"learning_rate": 0.0002383701943754715,
"loss_nats": 1.1943681240081787,
"step": 2810,
"unclipped_grad_norm": 0.5890340209007263
},
{
"bits_per_byte": 1.7962786443467342,
"learning_rate": 0.00023790113371022884,
"loss_nats": 1.2450854778289795,
"step": 2820,
"unclipped_grad_norm": 0.678447961807251
},
{
"bits_per_byte": 1.7361824026904413,
"learning_rate": 0.0002374308273657521,
"loss_nats": 1.203429937362671,
"step": 2830,
"unclipped_grad_norm": 0.5870603322982788
},
{
"bits_per_byte": 1.7695838412451108,
"learning_rate": 0.00023695928337827911,
"loss_nats": 1.2265820503234863,
"step": 2840,
"unclipped_grad_norm": 0.6424716711044312
},
{
"bits_per_byte": 1.7215556221998611,
"learning_rate": 0.00023648650980519564,
"loss_nats": 1.193291425704956,
"step": 2850,
"unclipped_grad_norm": 0.621953010559082
},
{
"bits_per_byte": 1.750462466156882,
"learning_rate": 0.00023601251472489757,
"loss_nats": 1.2133281230926514,
"step": 2860,
"unclipped_grad_norm": 0.6744791269302368
},
{
"bits_per_byte": 1.7458160108785215,
"learning_rate": 0.00023553730623665305,
"loss_nats": 1.210107445716858,
"step": 2870,
"unclipped_grad_norm": 0.6091592311859131
},
{
"bits_per_byte": 1.7563069525811528,
"learning_rate": 0.00023506089246046403,
"loss_nats": 1.2173792123794556,
"step": 2880,
"unclipped_grad_norm": 0.6055625081062317
},
{
"bits_per_byte": 1.7055577960166026,
"learning_rate": 0.00023458328153692736,
"loss_nats": 1.1822025775909424,
"step": 2890,
"unclipped_grad_norm": 0.5587822794914246
},
{
"bits_per_byte": 1.7494334939567697,
"learning_rate": 0.0002341044816270961,
"loss_nats": 1.212614893913269,
"step": 2900,
"unclipped_grad_norm": 0.7476353049278259
},
{
"bits_per_byte": 1.7069099236176861,
"learning_rate": 0.00023362450091233953,
"loss_nats": 1.1831398010253906,
"step": 2910,
"unclipped_grad_norm": 0.6083102226257324
},
{
"bits_per_byte": 1.7573092674703812,
"learning_rate": 0.00023314334759420388,
"loss_nats": 1.2180739641189575,
"step": 2920,
"unclipped_grad_norm": 0.594536542892456
},
{
"bits_per_byte": 1.7148656690639488,
"learning_rate": 0.0002326610298942717,
"loss_nats": 1.1886543035507202,
"step": 2930,
"unclipped_grad_norm": 0.6017132997512817
},
{
"bits_per_byte": 1.7206826382640814,
"learning_rate": 0.00023217755605402175,
"loss_nats": 1.1926863193511963,
"step": 2940,
"unclipped_grad_norm": 0.6191802620887756
},
{
"bits_per_byte": 1.7777722792179909,
"learning_rate": 0.00023169293433468802,
"loss_nats": 1.2322578430175781,
"step": 2950,
"unclipped_grad_norm": 0.6343991756439209
},
{
"bits_per_byte": 1.7965679191654853,
"learning_rate": 0.00023120717301711857,
"loss_nats": 1.245285987854004,
"step": 2960,
"unclipped_grad_norm": 0.6511562466621399
},
{
"bits_per_byte": 1.750740734085978,
"learning_rate": 0.00023072028040163408,
"loss_nats": 1.2135210037231445,
"step": 2970,
"unclipped_grad_norm": 0.6213991641998291
},
{
"bits_per_byte": 1.7817877301503355,
"learning_rate": 0.00023023226480788593,
"loss_nats": 1.2350411415100098,
"step": 2980,
"unclipped_grad_norm": 0.6211501359939575
},
{
"bits_per_byte": 1.724867836072828,
"learning_rate": 0.00022974313457471416,
"loss_nats": 1.1955872774124146,
"step": 2990,
"unclipped_grad_norm": 0.642371416091919
},
{
"bits_per_byte": 1.795713509355999,
"learning_rate": 0.0002292528980600049,
"loss_nats": 1.2446937561035156,
"step": 3000,
"unclipped_grad_norm": 0.6221276521682739
},
{
"bits_per_byte": 1.6848799779380654,
"learning_rate": 0.00022876156364054758,
"loss_nats": 1.1678698062896729,
"step": 3010,
"unclipped_grad_norm": 0.6145718693733215
},
{
"bits_per_byte": 1.752584732068529,
"learning_rate": 0.0002282691397118918,
"loss_nats": 1.214799165725708,
"step": 3020,
"unclipped_grad_norm": 0.6433340907096863
},
{
"bits_per_byte": 1.6945199494842276,
"learning_rate": 0.00022777563468820376,
"loss_nats": 1.1745517253875732,
"step": 3030,
"unclipped_grad_norm": 0.6049435138702393
},
{
"bits_per_byte": 1.6808318503020565,
"learning_rate": 0.00022728105700212282,
"loss_nats": 1.1650638580322266,
"step": 3040,
"unclipped_grad_norm": 0.5896543860435486
},
{
"bits_per_byte": 1.8340489581812716,
"learning_rate": 0.00022678541510461698,
"loss_nats": 1.2712658643722534,
"step": 3050,
"unclipped_grad_norm": 0.6642063856124878
},
{
"bits_per_byte": 1.7718811854953362,
"learning_rate": 0.00022628871746483876,
"loss_nats": 1.2281744480133057,
"step": 3060,
"unclipped_grad_norm": 0.6303152441978455
},
{
"bits_per_byte": 1.6793462641639036,
"learning_rate": 0.0002257909725699804,
"loss_nats": 1.164034128189087,
"step": 3070,
"unclipped_grad_norm": 0.6531770825386047
},
{
"bits_per_byte": 1.767122941493915,
"learning_rate": 0.00022529218892512886,
"loss_nats": 1.2248762845993042,
"step": 3080,
"unclipped_grad_norm": 0.6228736042976379
},
{
"bits_per_byte": 1.7547854220689696,
"learning_rate": 0.00022479237505312034,
"loss_nats": 1.2163245677947998,
"step": 3090,
"unclipped_grad_norm": 0.6808046102523804
},
{
"bits_per_byte": 1.7115178547822532,
"learning_rate": 0.00022429153949439496,
"loss_nats": 1.1863337755203247,
"step": 3100,
"unclipped_grad_norm": 0.6307847499847412
},
{
"bits_per_byte": 1.8621731090942157,
"learning_rate": 0.00022378969080685053,
"loss_nats": 1.2907600402832031,
"step": 3110,
"unclipped_grad_norm": 0.652417004108429
},
{
"bits_per_byte": 1.7445247651358435,
"learning_rate": 0.0002232868375656965,
"loss_nats": 1.2092124223709106,
"step": 3120,
"unclipped_grad_norm": 0.6338313221931458
},
{
"bits_per_byte": 1.6854237870800937,
"learning_rate": 0.00022278298836330733,
"loss_nats": 1.1682467460632324,
"step": 3130,
"unclipped_grad_norm": 0.6034438610076904
},
{
"bits_per_byte": 1.717318657613205,
"learning_rate": 0.00022227815180907572,
"loss_nats": 1.190354585647583,
"step": 3140,
"unclipped_grad_norm": 0.6048492193222046
},
{
"bits_per_byte": 1.7027365926118492,
"learning_rate": 0.00022177233652926546,
"loss_nats": 1.1802470684051514,
"step": 3150,
"unclipped_grad_norm": 0.5864679217338562
},
{
"bits_per_byte": 1.705547477057551,
"learning_rate": 0.00022126555116686416,
"loss_nats": 1.1821954250335693,
"step": 3160,
"unclipped_grad_norm": 0.6051758527755737
},
{
"bits_per_byte": 1.7744621291368343,
"learning_rate": 0.00022075780438143532,
"loss_nats": 1.2299634218215942,
"step": 3170,
"unclipped_grad_norm": 0.6531821489334106
},
{
"bits_per_byte": 1.7458908233316468,
"learning_rate": 0.00022024910484897065,
"loss_nats": 1.2101593017578125,
"step": 3180,
"unclipped_grad_norm": 0.6303388476371765
},
{
"bits_per_byte": 1.7361681281304195,
"learning_rate": 0.00021973946126174158,
"loss_nats": 1.2034200429916382,
"step": 3190,
"unclipped_grad_norm": 0.678484320640564
},
{
"bits_per_byte": 1.7289813171161645,
"learning_rate": 0.00021922888232815086,
"loss_nats": 1.1984385251998901,
"step": 3200,
"unclipped_grad_norm": 0.6228032112121582
},
{
"bits_per_byte": 1.6986354943193747,
"learning_rate": 0.0002187173767725837,
"loss_nats": 1.1774044036865234,
"step": 3210,
"unclipped_grad_norm": 0.6199705004692078
},
{
"bits_per_byte": 1.6704626723162361,
"learning_rate": 0.0002182049533352588,
"loss_nats": 1.1578764915466309,
"step": 3220,
"unclipped_grad_norm": 0.6396132111549377
},
{
"bits_per_byte": 1.7446800654695727,
"learning_rate": 0.00021769162077207884,
"loss_nats": 1.209320068359375,
"step": 3230,
"unclipped_grad_norm": 0.6783336400985718
},
{
"bits_per_byte": 1.7503792265538642,
"learning_rate": 0.00021717738785448102,
"loss_nats": 1.2132704257965088,
"step": 3240,
"unclipped_grad_norm": 0.6652441620826721
},
{
"bits_per_byte": 1.7183588086856232,
"learning_rate": 0.00021666226336928708,
"loss_nats": 1.1910755634307861,
"step": 3250,
"unclipped_grad_norm": 0.63749098777771
},
{
"bits_per_byte": 1.7194732562632142,
"learning_rate": 0.0002161462561185532,
"loss_nats": 1.1918480396270752,
"step": 3260,
"unclipped_grad_norm": 0.6323399543762207
},
{
"bits_per_byte": 1.76014147776479,
"learning_rate": 0.00021562937491941957,
"loss_nats": 1.2200371026992798,
"step": 3270,
"unclipped_grad_norm": 0.668753445148468
},
{
"bits_per_byte": 1.7466587258677493,
"learning_rate": 0.00021511162860395975,
"loss_nats": 1.2106915712356567,
"step": 3280,
"unclipped_grad_norm": 0.5981830954551697
},
{
"bits_per_byte": 1.7077491989538964,
"learning_rate": 0.0002145930260190298,
"loss_nats": 1.1837215423583984,
"step": 3290,
"unclipped_grad_norm": 0.6258406639099121
},
{
"bits_per_byte": 1.7122833495612435,
"learning_rate": 0.00021407357602611698,
"loss_nats": 1.1868643760681152,
"step": 3300,
"unclipped_grad_norm": 0.6118162870407104
},
{
"bits_per_byte": 1.6971689982554674,
"learning_rate": 0.0002135532875011885,
"loss_nats": 1.176387906074524,
"step": 3310,
"unclipped_grad_norm": 0.614425003528595
},
{
"bits_per_byte": 1.7078183359795434,
"learning_rate": 0.0002130321693345397,
"loss_nats": 1.1837694644927979,
"step": 3320,
"unclipped_grad_norm": 0.6319419741630554
},
{
"bits_per_byte": 1.7065344854908524,
"learning_rate": 0.00021251023043064224,
"loss_nats": 1.1828795671463013,
"step": 3330,
"unclipped_grad_norm": 0.6474409699440002
},
{
"bits_per_byte": 1.6476438422171078,
"learning_rate": 0.00021198747970799188,
"loss_nats": 1.1420596837997437,
"step": 3340,
"unclipped_grad_norm": 0.6128159165382385
},
{
"bits_per_byte": 1.693542744062025,
"learning_rate": 0.00021146392609895618,
"loss_nats": 1.1738743782043457,
"step": 3350,
"unclipped_grad_norm": 0.6108464002609253
},
{
"bits_per_byte": 1.7204735073606323,
"learning_rate": 0.00021093957854962178,
"loss_nats": 1.1925413608551025,
"step": 3360,
"unclipped_grad_norm": 0.6041978597640991
},
{
"bits_per_byte": 1.6622183399818258,
"learning_rate": 0.00021041444601964157,
"loss_nats": 1.152161955833435,
"step": 3370,
"unclipped_grad_norm": 0.5887104272842407
},
{
"bits_per_byte": 1.7058931621857851,
"learning_rate": 0.00020988853748208164,
"loss_nats": 1.1824350357055664,
"step": 3380,
"unclipped_grad_norm": 0.6352498531341553
},
{
"bits_per_byte": 1.707757282138487,
"learning_rate": 0.00020936186192326782,
"loss_nats": 1.1837271451950073,
"step": 3390,
"unclipped_grad_norm": 0.6421219706535339
},
{
"bits_per_byte": 1.6979798964542856,
"learning_rate": 0.00020883442834263225,
"loss_nats": 1.1769499778747559,
"step": 3400,
"unclipped_grad_norm": 0.6395401954650879
},
{
"bits_per_byte": 1.6959995162296002,
"learning_rate": 0.0002083062457525596,
"loss_nats": 1.1755772829055786,
"step": 3410,
"unclipped_grad_norm": 0.6283812522888184
},
{
"bits_per_byte": 1.6799083034669233,
"learning_rate": 0.00020777732317823306,
"loss_nats": 1.1644237041473389,
"step": 3420,
"unclipped_grad_norm": 0.6197249293327332
},
{
"bits_per_byte": 1.71363410130112,
"learning_rate": 0.00020724766965748005,
"loss_nats": 1.187800645828247,
"step": 3430,
"unclipped_grad_norm": 0.6102274060249329
},
{
"bits_per_byte": 1.7091876618457131,
"learning_rate": 0.00020671729424061788,
"loss_nats": 1.1847186088562012,
"step": 3440,
"unclipped_grad_norm": 0.6624795198440552
},
{
"bits_per_byte": 1.7982115573597812,
"learning_rate": 0.00020618620599029906,
"loss_nats": 1.2464252710342407,
"step": 3450,
"unclipped_grad_norm": 0.6986591815948486
},
{
"bits_per_byte": 1.7759950105039746,
"learning_rate": 0.00020565441398135649,
"loss_nats": 1.2310259342193604,
"step": 3460,
"unclipped_grad_norm": 0.6196012496948242
},
{
"bits_per_byte": 1.7475674821949083,
"learning_rate": 0.00020512192730064827,
"loss_nats": 1.211321473121643,
"step": 3470,
"unclipped_grad_norm": 0.6346544623374939
},
{
"bits_per_byte": 1.7169992858305527,
"learning_rate": 0.00020458875504690273,
"loss_nats": 1.1901332139968872,
"step": 3480,
"unclipped_grad_norm": 0.6380975246429443
},
{
"bits_per_byte": 1.6316707815355735,
"learning_rate": 0.00020405490633056245,
"loss_nats": 1.1309880018234253,
"step": 3490,
"unclipped_grad_norm": 0.6554663181304932
},
{
"bits_per_byte": 1.641168867394774,
"learning_rate": 0.00020352039027362917,
"loss_nats": 1.1375715732574463,
"step": 3500,
"unclipped_grad_norm": 0.6088140606880188
},
{
"bits_per_byte": 1.6880614849963764,
"learning_rate": 0.0002029852160095075,
"loss_nats": 1.1700750589370728,
"step": 3510,
"unclipped_grad_norm": 0.6358106732368469
},
{
"bits_per_byte": 1.6616972325497115,
"learning_rate": 0.000202449392682849,
"loss_nats": 1.1518007516860962,
"step": 3520,
"unclipped_grad_norm": 0.6153261661529541
},
{
"bits_per_byte": 1.6712895649015846,
"learning_rate": 0.0002019129294493959,
"loss_nats": 1.158449649810791,
"step": 3530,
"unclipped_grad_norm": 0.6411516666412354
},
{
"bits_per_byte": 1.6785265948498915,
"learning_rate": 0.00020137583547582478,
"loss_nats": 1.163465976715088,
"step": 3540,
"unclipped_grad_norm": 0.6034340858459473
},
{
"bits_per_byte": 1.7376134703282706,
"learning_rate": 0.00020083811993958973,
"loss_nats": 1.204421877861023,
"step": 3550,
"unclipped_grad_norm": 0.6545450687408447
},
{
"bits_per_byte": 1.6707187544833708,
"learning_rate": 0.00020029979202876563,
"loss_nats": 1.158053994178772,
"step": 3560,
"unclipped_grad_norm": 0.6340785026550293
},
{
"bits_per_byte": 1.7138805524398064,
"learning_rate": 0.00019976086094189118,
"loss_nats": 1.1879714727401733,
"step": 3570,
"unclipped_grad_norm": 0.6533586382865906
},
{
"bits_per_byte": 1.7826314770354685,
"learning_rate": 0.00019922133588781175,
"loss_nats": 1.235625982284546,
"step": 3580,
"unclipped_grad_norm": 0.7123501896858215
},
{
"bits_per_byte": 1.7131767994324758,
"learning_rate": 0.0001986812260855219,
"loss_nats": 1.1874836683273315,
"step": 3590,
"unclipped_grad_norm": 0.6637008786201477
},
{
"bits_per_byte": 1.6959864455481348,
"learning_rate": 0.00019814054076400787,
"loss_nats": 1.1755682229995728,
"step": 3600,
"unclipped_grad_norm": 0.6217906475067139
},
{
"bits_per_byte": 1.6716725702650561,
"learning_rate": 0.00019759928916209004,
"loss_nats": 1.1587151288986206,
"step": 3610,
"unclipped_grad_norm": 0.6237639784812927
},
{
"bits_per_byte": 1.6746752153664697,
"learning_rate": 0.00019705748052826493,
"loss_nats": 1.1607964038848877,
"step": 3620,
"unclipped_grad_norm": 0.6187261343002319
},
{
"bits_per_byte": 1.6883108598401275,
"learning_rate": 0.00019651512412054723,
"loss_nats": 1.1702479124069214,
"step": 3630,
"unclipped_grad_norm": 0.6150439381599426
},
{
"bits_per_byte": 1.6436062055228018,
"learning_rate": 0.00019597222920631144,
"loss_nats": 1.13926100730896,
"step": 3640,
"unclipped_grad_norm": 0.6298535466194153
},
{
"bits_per_byte": 1.7182858880416574,
"learning_rate": 0.0001954288050621338,
"loss_nats": 1.1910250186920166,
"step": 3650,
"unclipped_grad_norm": 0.6331585049629211
},
{
"bits_per_byte": 1.6224437403341332,
"learning_rate": 0.00019488486097363363,
"loss_nats": 1.1245923042297363,
"step": 3660,
"unclipped_grad_norm": 0.6383513808250427
},
{
"bits_per_byte": 1.644355533932611,
"learning_rate": 0.00019434040623531452,
"loss_nats": 1.1397804021835327,
"step": 3670,
"unclipped_grad_norm": 0.6061356067657471
},
{
"bits_per_byte": 1.7035681287287707,
"learning_rate": 0.0001937954501504058,
"loss_nats": 1.1808234453201294,
"step": 3680,
"unclipped_grad_norm": 0.6479222774505615
},
{
"bits_per_byte": 1.6706625161565387,
"learning_rate": 0.00019325000203070333,
"loss_nats": 1.1580150127410889,
"step": 3690,
"unclipped_grad_norm": 0.6192170977592468
},
{
"bits_per_byte": 1.6109532354646872,
"learning_rate": 0.0001927040711964106,
"loss_nats": 1.1166276931762695,
"step": 3700,
"unclipped_grad_norm": 0.6093165278434753
},
{
"bits_per_byte": 1.6773731072105547,
"learning_rate": 0.00019215766697597923,
"loss_nats": 1.1626664400100708,
"step": 3710,
"unclipped_grad_norm": 0.6192633509635925
},
{
"bits_per_byte": 1.6614944650043442,
"learning_rate": 0.00019161079870594978,
"loss_nats": 1.1516602039337158,
"step": 3720,
"unclipped_grad_norm": 0.6275249719619751
},
{
"bits_per_byte": 1.7267034069054867,
"learning_rate": 0.0001910634757307922,
"loss_nats": 1.19685959815979,
"step": 3730,
"unclipped_grad_norm": 0.6384285092353821
},
{
"bits_per_byte": 1.6506055554476162,
"learning_rate": 0.0001905157074027459,
"loss_nats": 1.1441125869750977,
"step": 3740,
"unclipped_grad_norm": 0.6508066654205322
},
{
"bits_per_byte": 1.6398485565841001,
"learning_rate": 0.00018996750308166037,
"loss_nats": 1.136656403541565,
"step": 3750,
"unclipped_grad_norm": 0.6484293937683105
},
{
"bits_per_byte": 1.6392961483095287,
"learning_rate": 0.00018941887213483482,
"loss_nats": 1.1362735033035278,
"step": 3760,
"unclipped_grad_norm": 0.6600143909454346
},
{
"bits_per_byte": 1.686776946577082,
"learning_rate": 0.00018886982393685845,
"loss_nats": 1.169184684753418,
"step": 3770,
"unclipped_grad_norm": 0.6462109684944153
},
{
"bits_per_byte": 1.6300704829692951,
"learning_rate": 0.00018832036786945008,
"loss_nats": 1.1298787593841553,
"step": 3780,
"unclipped_grad_norm": 0.6114524006843567
},
{
"bits_per_byte": 1.675651388892767,
"learning_rate": 0.0001877705133212979,
"loss_nats": 1.161473035812378,
"step": 3790,
"unclipped_grad_norm": 0.6200737953186035
},
{
"bits_per_byte": 1.617114685914498,
"learning_rate": 0.00018722026968789906,
"loss_nats": 1.1208984851837158,
"step": 3800,
"unclipped_grad_norm": 0.6216682195663452
},
{
"bits_per_byte": 1.6234193979124778,
"learning_rate": 0.00018666964637139907,
"loss_nats": 1.125268578529358,
"step": 3810,
"unclipped_grad_norm": 0.6173998117446899
},
{
"bits_per_byte": 1.7138243141129743,
"learning_rate": 0.00018611865278043115,
"loss_nats": 1.1879324913024902,
"step": 3820,
"unclipped_grad_norm": 0.6088318228721619
},
{
"bits_per_byte": 1.6341469877426964,
"learning_rate": 0.00018556729832995555,
"loss_nats": 1.1327043771743774,
"step": 3830,
"unclipped_grad_norm": 0.6438922882080078
},
{
"bits_per_byte": 1.655027229401299,
"learning_rate": 0.00018501559244109864,
"loss_nats": 1.1471774578094482,
"step": 3840,
"unclipped_grad_norm": 0.669468879699707
},
{
"bits_per_byte": 1.6818487837166085,
"learning_rate": 0.00018446354454099178,
"loss_nats": 1.1657687425613403,
"step": 3850,
"unclipped_grad_norm": 0.6244412660598755
},
{
"bits_per_byte": 1.6164324307385254,
"learning_rate": 0.00018391116406261048,
"loss_nats": 1.1204255819320679,
"step": 3860,
"unclipped_grad_norm": 0.6409146785736084
},
{
"bits_per_byte": 1.6824030838003392,
"learning_rate": 0.00018335846044461305,
"loss_nats": 1.1661529541015625,
"step": 3870,
"unclipped_grad_norm": 0.6731106042861938
},
{
"bits_per_byte": 1.6375393455309653,
"learning_rate": 0.0001828054431311793,
"loss_nats": 1.1350557804107666,
"step": 3880,
"unclipped_grad_norm": 0.6218310594558716
},
{
"bits_per_byte": 1.6796400105315774,
"learning_rate": 0.00018225212157184934,
"loss_nats": 1.1642377376556396,
"step": 3890,
"unclipped_grad_norm": 0.635161280632019
},
{
"bits_per_byte": 1.666564169586478,
"learning_rate": 0.00018169850522136192,
"loss_nats": 1.1551742553710938,
"step": 3900,
"unclipped_grad_norm": 0.7091010808944702
},
{
"bits_per_byte": 1.7026119051899735,
"learning_rate": 0.000181144603539493,
"loss_nats": 1.180160641670227,
"step": 3910,
"unclipped_grad_norm": 0.6815565824508667
},
{
"bits_per_byte": 1.6485720325838145,
"learning_rate": 0.00018059042599089412,
"loss_nats": 1.1427030563354492,
"step": 3920,
"unclipped_grad_norm": 0.628564178943634
},
{
"bits_per_byte": 1.6283515163739215,
"learning_rate": 0.00018003598204493048,
"loss_nats": 1.1286872625350952,
"step": 3930,
"unclipped_grad_norm": 0.6302286982536316
},
{
"bits_per_byte": 1.6363571367889342,
"learning_rate": 0.00017948128117551936,
"loss_nats": 1.1342363357543945,
"step": 3940,
"unclipped_grad_norm": 0.607494056224823
},
{
"bits_per_byte": 1.7146744243561893,
"learning_rate": 0.00017892633286096816,
"loss_nats": 1.1885217428207397,
"step": 3950,
"unclipped_grad_norm": 0.70033860206604
},
{
"bits_per_byte": 1.7144123227962744,
"learning_rate": 0.00017837114658381247,
"loss_nats": 1.1883400678634644,
"step": 3960,
"unclipped_grad_norm": 0.645106315612793
},
{
"bits_per_byte": 1.6837865122438798,
"learning_rate": 0.00017781573183065386,
"loss_nats": 1.167111873626709,
"step": 3970,
"unclipped_grad_norm": 0.6373703479766846
},
{
"bits_per_byte": 1.6677649524548022,
"learning_rate": 0.0001772600980919981,
"loss_nats": 1.1560065746307373,
"step": 3980,
"unclipped_grad_norm": 0.6565650701522827
},
{
"bits_per_byte": 1.7059107044161732,
"learning_rate": 0.0001767042548620927,
"loss_nats": 1.1824471950531006,
"step": 3990,
"unclipped_grad_norm": 0.6542640924453735
},
{
"bits_per_byte": 1.7121254694877515,
"learning_rate": 0.00017614821163876486,
"loss_nats": 1.1867549419403076,
"step": 4000,
"unclipped_grad_norm": 0.6575196385383606
},
{
"bits_per_byte": 1.7220670986035271,
"learning_rate": 0.00017559197792325906,
"loss_nats": 1.19364595413208,
"step": 4010,
"unclipped_grad_norm": 0.6806800961494446
},
{
"bits_per_byte": 1.5958445595863893,
"learning_rate": 0.0001750355632200748,
"loss_nats": 1.1061551570892334,
"step": 4020,
"unclipped_grad_norm": 0.6171497106552124
},
{
"bits_per_byte": 1.6345342926724393,
"learning_rate": 0.00017447897703680418,
"loss_nats": 1.1329728364944458,
"step": 4030,
"unclipped_grad_norm": 0.6173087954521179
},
{
"bits_per_byte": 1.6363991005557448,
"learning_rate": 0.0001739222288839694,
"loss_nats": 1.134265422821045,
"step": 4040,
"unclipped_grad_norm": 0.634988009929657
},
{
"bits_per_byte": 1.6319382145576653,
"learning_rate": 0.00017336532827486026,
"loss_nats": 1.1311733722686768,
"step": 4050,
"unclipped_grad_norm": 0.6698096990585327
},
{
"bits_per_byte": 1.6486795217406036,
"learning_rate": 0.00017280828472537163,
"loss_nats": 1.1427775621414185,
"step": 4060,
"unclipped_grad_norm": 0.6841239929199219
},
{
"bits_per_byte": 1.6129987971140494,
"learning_rate": 0.0001722511077538408,
"loss_nats": 1.1180455684661865,
"step": 4070,
"unclipped_grad_norm": 0.6276201605796814
},
{
"bits_per_byte": 1.6508999897458934,
"learning_rate": 0.00017169380688088497,
"loss_nats": 1.1443166732788086,
"step": 4080,
"unclipped_grad_norm": 0.6372854113578796
},
{
"bits_per_byte": 1.6061886281051838,
"learning_rate": 0.0001711363916292384,
"loss_nats": 1.1133251190185547,
"step": 4090,
"unclipped_grad_norm": 0.6243290305137634
},
{
"bits_per_byte": 1.578650766049031,
"learning_rate": 0.00017057887152358974,
"loss_nats": 1.0942373275756836,
"step": 4100,
"unclipped_grad_norm": 0.5791712403297424
},
{
"bits_per_byte": 1.6840439702722212,
"learning_rate": 0.0001700212560904194,
"loss_nats": 1.1672903299331665,
"step": 4110,
"unclipped_grad_norm": 0.6848294734954834
},
{
"bits_per_byte": 1.6312047085517354,
"learning_rate": 0.00016946355485783656,
"loss_nats": 1.1306649446487427,
"step": 4120,
"unclipped_grad_norm": 0.6449601054191589
},
{
"bits_per_byte": 1.594994965291127,
"learning_rate": 0.0001689057773554166,
"loss_nats": 1.1055662631988525,
"step": 4130,
"unclipped_grad_norm": 0.6161995530128479
},
{
"bits_per_byte": 1.6232881751498696,
"learning_rate": 0.00016834793311403806,
"loss_nats": 1.1251776218414307,
"step": 4140,
"unclipped_grad_norm": 0.6562530398368835
},
{
"bits_per_byte": 1.67032388231699,
"learning_rate": 0.00016779003166571981,
"loss_nats": 1.1577802896499634,
"step": 4150,
"unclipped_grad_norm": 0.672451376914978
},
{
"bits_per_byte": 1.6420112384187,
"learning_rate": 0.00016723208254345838,
"loss_nats": 1.138155460357666,
"step": 4160,
"unclipped_grad_norm": 0.6645690202713013
},
{
"bits_per_byte": 1.597898376402993,
"learning_rate": 0.00016667409528106481,
"loss_nats": 1.1075787544250488,
"step": 4170,
"unclipped_grad_norm": 0.6766495108604431
},
{
"bits_per_byte": 1.5688259131531912,
"learning_rate": 0.00016611607941300184,
"loss_nats": 1.0874272584915161,
"step": 4180,
"unclipped_grad_norm": 0.6362206339836121
},
{
"bits_per_byte": 1.7152548658028512,
"learning_rate": 0.0001655580444742211,
"loss_nats": 1.1889240741729736,
"step": 4190,
"unclipped_grad_norm": 0.6535307765007019
},
{
"bits_per_byte": 1.6086450563074575,
"learning_rate": 0.00016499999999999997,
"loss_nats": 1.1150277853012085,
"step": 4200,
"unclipped_grad_norm": 0.6474415063858032
},
{
"bits_per_byte": 1.6237748860518113,
"learning_rate": 0.0001644419555257789,
"loss_nats": 1.1255149841308594,
"step": 4210,
"unclipped_grad_norm": 0.6242862343788147
},
{
"bits_per_byte": 1.6672617312183775,
"learning_rate": 0.0001638839205869981,
"loss_nats": 1.1556577682495117,
"step": 4220,
"unclipped_grad_norm": 0.6519601345062256
},
{
"bits_per_byte": 1.635159105643024,
"learning_rate": 0.0001633259047189352,
"loss_nats": 1.1334059238433838,
"step": 4230,
"unclipped_grad_norm": 0.6224661469459534
},
{
"bits_per_byte": 1.6806368219759782,
"learning_rate": 0.0001627679174565416,
"loss_nats": 1.164928674697876,
"step": 4240,
"unclipped_grad_norm": 0.6462982296943665
},
{
"bits_per_byte": 1.7100040634893587,
"learning_rate": 0.00016220996833428016,
"loss_nats": 1.1852844953536987,
"step": 4250,
"unclipped_grad_norm": 0.6697248220443726
},
{
"bits_per_byte": 1.6765044228410466,
"learning_rate": 0.00016165206688596191,
"loss_nats": 1.1620643138885498,
"step": 4260,
"unclipped_grad_norm": 0.6539841294288635
},
{
"bits_per_byte": 1.6521546031839376,
"learning_rate": 0.00016109422264458335,
"loss_nats": 1.1451863050460815,
"step": 4270,
"unclipped_grad_norm": 0.6337559819221497
},
{
"bits_per_byte": 1.688240002987972,
"learning_rate": 0.00016053644514216336,
"loss_nats": 1.1701987981796265,
"step": 4280,
"unclipped_grad_norm": 0.6939600706100464
},
{
"bits_per_byte": 1.645464994013327,
"learning_rate": 0.00015997874390958058,
"loss_nats": 1.1405494213104248,
"step": 4290,
"unclipped_grad_norm": 0.6979376673698425
},
{
"bits_per_byte": 1.5687516166480184,
"learning_rate": 0.00015942112847641018,
"loss_nats": 1.0873757600784302,
"step": 4300,
"unclipped_grad_norm": 0.6490140557289124
},
{
"bits_per_byte": 1.720270739815265,
"learning_rate": 0.00015886360837076156,
"loss_nats": 1.1924008131027222,
"step": 4310,
"unclipped_grad_norm": 0.7267009615898132
},
{
"bits_per_byte": 1.5912837516681586,
"learning_rate": 0.00015830619311911495,
"loss_nats": 1.1029938459396362,
"step": 4320,
"unclipped_grad_norm": 0.6906614899635315
},
{
"bits_per_byte": 1.5899403951822693,
"learning_rate": 0.00015774889224615915,
"loss_nats": 1.102062702178955,
"step": 4330,
"unclipped_grad_norm": 0.6376214027404785
},
{
"bits_per_byte": 1.5963857889886546,
"learning_rate": 0.00015719171527462834,
"loss_nats": 1.1065303087234497,
"step": 4340,
"unclipped_grad_norm": 0.6380553841590881
},
{
"bits_per_byte": 1.5301035033634773,
"learning_rate": 0.0001566346717251397,
"loss_nats": 1.060586929321289,
"step": 4350,
"unclipped_grad_norm": 0.6225847601890564
},
{
"bits_per_byte": 1.67761336697381,
"learning_rate": 0.00015607777111603056,
"loss_nats": 1.1628329753875732,
"step": 4360,
"unclipped_grad_norm": 0.6995339393615723
},
{
"bits_per_byte": 1.6233086410853221,
"learning_rate": 0.00015552102296319576,
"loss_nats": 1.1251918077468872,
"step": 4370,
"unclipped_grad_norm": 0.6690109372138977
},
{
"bits_per_byte": 1.5971554113512656,
"learning_rate": 0.00015496443677992516,
"loss_nats": 1.1070637702941895,
"step": 4380,
"unclipped_grad_norm": 0.7015417814254761
},
{
"bits_per_byte": 1.6390723988807563,
"learning_rate": 0.0001544080220767409,
"loss_nats": 1.1361184120178223,
"step": 4390,
"unclipped_grad_norm": 0.6816950440406799
},
{
"bits_per_byte": 1.5932933689434905,
"learning_rate": 0.00015385178836123511,
"loss_nats": 1.104386806488037,
"step": 4400,
"unclipped_grad_norm": 0.667495846748352
},
{
"bits_per_byte": 1.5991400911422224,
"learning_rate": 0.00015329574513790726,
"loss_nats": 1.1084394454956055,
"step": 4410,
"unclipped_grad_norm": 0.6873564124107361
},
{
"bits_per_byte": 1.58255941575519,
"learning_rate": 0.00015273990190800186,
"loss_nats": 1.0969465970993042,
"step": 4420,
"unclipped_grad_norm": 0.6864304542541504
},
{
"bits_per_byte": 1.5926611607189187,
"learning_rate": 0.0001521842681693461,
"loss_nats": 1.1039485931396484,
"step": 4430,
"unclipped_grad_norm": 0.6576697826385498
},
{
"bits_per_byte": 1.6095080652494869,
"learning_rate": 0.00015162885341618752,
"loss_nats": 1.1156259775161743,
"step": 4440,
"unclipped_grad_norm": 0.6515920162200928
},
{
"bits_per_byte": 1.5838350110766395,
"learning_rate": 0.00015107366713903178,
"loss_nats": 1.0978307723999023,
"step": 4450,
"unclipped_grad_norm": 0.6458138227462769
},
{
"bits_per_byte": 1.5629618207067222,
"learning_rate": 0.0001505187188244806,
"loss_nats": 1.0833625793457031,
"step": 4460,
"unclipped_grad_norm": 0.6665330529212952
},
{
"bits_per_byte": 1.6502230660320973,
"learning_rate": 0.00014996401795506952,
"loss_nats": 1.1438474655151367,
"step": 4470,
"unclipped_grad_norm": 0.6849002838134766
},
{
"bits_per_byte": 1.5654218605446637,
"learning_rate": 0.00014940957400910585,
"loss_nats": 1.0850677490234375,
"step": 4480,
"unclipped_grad_norm": 0.630775511264801
},
{
"bits_per_byte": 1.533044750658533,
"learning_rate": 0.00014885539646050699,
"loss_nats": 1.0626256465911865,
"step": 4490,
"unclipped_grad_norm": 0.6286258697509766
},
{
"bits_per_byte": 1.6266820807819962,
"learning_rate": 0.00014830149477863805,
"loss_nats": 1.1275300979614258,
"step": 4500,
"unclipped_grad_norm": 0.6595945954322815
},
{
"bits_per_byte": 1.5885456158837719,
"learning_rate": 0.00014774787842815066,
"loss_nats": 1.1010959148406982,
"step": 4510,
"unclipped_grad_norm": 0.6231974363327026
},
{
"bits_per_byte": 1.5897587815029581,
"learning_rate": 0.00014719455686882066,
"loss_nats": 1.1019368171691895,
"step": 4520,
"unclipped_grad_norm": 0.6408888697624207
},
{
"bits_per_byte": 1.6075569220754484,
"learning_rate": 0.00014664153955538695,
"loss_nats": 1.1142735481262207,
"step": 4530,
"unclipped_grad_norm": 0.6721575260162354
},
{
"bits_per_byte": 1.5571181941957055,
"learning_rate": 0.00014608883593738946,
"loss_nats": 1.0793120861053467,
"step": 4540,
"unclipped_grad_norm": 0.6309816837310791
},
{
"bits_per_byte": 1.6337729254770696,
"learning_rate": 0.00014553645545900817,
"loss_nats": 1.1324450969696045,
"step": 4550,
"unclipped_grad_norm": 0.6641245484352112
},
{
"bits_per_byte": 1.6370763682348426,
"learning_rate": 0.0001449844075589013,
"loss_nats": 1.134734869003296,
"step": 4560,
"unclipped_grad_norm": 0.6858609914779663
},
{
"bits_per_byte": 1.5844336826842929,
"learning_rate": 0.0001444327016700444,
"loss_nats": 1.0982457399368286,
"step": 4570,
"unclipped_grad_norm": 0.6500639319419861
},
{
"bits_per_byte": 1.5406408803818417,
"learning_rate": 0.0001438813472195688,
"loss_nats": 1.0678908824920654,
"step": 4580,
"unclipped_grad_norm": 0.636735200881958
},
{
"bits_per_byte": 1.6489479866586005,
"learning_rate": 0.00014333035362860093,
"loss_nats": 1.1429636478424072,
"step": 4590,
"unclipped_grad_norm": 0.6933116316795349
},
{
"bits_per_byte": 1.6320697812855753,
"learning_rate": 0.00014277973031210088,
"loss_nats": 1.131264567375183,
"step": 4600,
"unclipped_grad_norm": 0.6834596395492554
},
{
"bits_per_byte": 1.589729888417613,
"learning_rate": 0.00014222948667870207,
"loss_nats": 1.101916790008545,
"step": 4610,
"unclipped_grad_norm": 0.6632579565048218
},
{
"bits_per_byte": 1.5335797886853673,
"learning_rate": 0.00014167963213054991,
"loss_nats": 1.062996506690979,
"step": 4620,
"unclipped_grad_norm": 0.6395919322967529
},
{
"bits_per_byte": 1.6054043872172494,
"learning_rate": 0.00014113017606314152,
"loss_nats": 1.1127815246582031,
"step": 4630,
"unclipped_grad_norm": 0.6490065455436707
},
{
"bits_per_byte": 1.5840957367753474,
"learning_rate": 0.00014058112786516518,
"loss_nats": 1.0980114936828613,
"step": 4640,
"unclipped_grad_norm": 0.6492030024528503
},
{
"bits_per_byte": 1.5576186637097162,
"learning_rate": 0.0001400324969183396,
"loss_nats": 1.0796589851379395,
"step": 4650,
"unclipped_grad_norm": 0.6488379836082458
},
{
"bits_per_byte": 1.593877078060519,
"learning_rate": 0.00013948429259725406,
"loss_nats": 1.1047914028167725,
"step": 4660,
"unclipped_grad_norm": 0.6992486119270325
},
{
"bits_per_byte": 1.6105064245377456,
"learning_rate": 0.00013893652426920776,
"loss_nats": 1.1163179874420166,
"step": 4670,
"unclipped_grad_norm": 0.6828540563583374
},
{
"bits_per_byte": 1.5810454524796447,
"learning_rate": 0.00013838920129405016,
"loss_nats": 1.0958971977233887,
"step": 4680,
"unclipped_grad_norm": 0.6880136728286743
},
{
"bits_per_byte": 1.5551754781715592,
"learning_rate": 0.0001378423330240207,
"loss_nats": 1.077965497970581,
"step": 4690,
"unclipped_grad_norm": 0.6890891194343567
},
{
"bits_per_byte": 1.6623815515174944,
"learning_rate": 0.00013729592880358936,
"loss_nats": 1.1522750854492188,
"step": 4700,
"unclipped_grad_norm": 0.6787605285644531
},
{
"bits_per_byte": 1.574140349047503,
"learning_rate": 0.00013674999796929659,
"loss_nats": 1.0911109447479248,
"step": 4710,
"unclipped_grad_norm": 0.6684292554855347
},
{
"bits_per_byte": 1.5498508953008467,
"learning_rate": 0.00013620454984959417,
"loss_nats": 1.0742747783660889,
"step": 4720,
"unclipped_grad_norm": 0.6828400492668152
},
{
"bits_per_byte": 1.6704473658603094,
"learning_rate": 0.00013565959376468542,
"loss_nats": 1.1578658819198608,
"step": 4730,
"unclipped_grad_norm": 0.6912305355072021
},
{
"bits_per_byte": 1.6014281483293014,
"learning_rate": 0.00013511513902636634,
"loss_nats": 1.110025405883789,
"step": 4740,
"unclipped_grad_norm": 0.6723529100418091
},
{
"bits_per_byte": 1.639597977861793,
"learning_rate": 0.0001345711949378662,
"loss_nats": 1.1364827156066895,
"step": 4750,
"unclipped_grad_norm": 0.7447375059127808
},
{
"bits_per_byte": 1.523300557625948,
"learning_rate": 0.00013402777079368853,
"loss_nats": 1.0558714866638184,
"step": 4760,
"unclipped_grad_norm": 0.6460341811180115
},
{
"bits_per_byte": 1.6314774730360038,
"learning_rate": 0.0001334848758794528,
"loss_nats": 1.1308540105819702,
"step": 4770,
"unclipped_grad_norm": 0.6882368922233582
},
{
"bits_per_byte": 1.5726310293035308,
"learning_rate": 0.00013294251947173504,
"loss_nats": 1.0900647640228271,
"step": 4780,
"unclipped_grad_norm": 0.6284239888191223
},
{
"bits_per_byte": 1.60157536547844,
"learning_rate": 0.00013240071083790996,
"loss_nats": 1.1101274490356445,
"step": 4790,
"unclipped_grad_norm": 0.6737696528434753
},
{
"bits_per_byte": 1.577140070443852,
"learning_rate": 0.0001318594592359921,
"loss_nats": 1.0931901931762695,
"step": 4800,
"unclipped_grad_norm": 0.6438981890678406
},
{
"bits_per_byte": 1.5444138357764698,
"learning_rate": 0.0001313187739144781,
"loss_nats": 1.0705060958862305,
"step": 4810,
"unclipped_grad_norm": 0.690973699092865
},
{
"bits_per_byte": 1.6096637095485178,
"learning_rate": 0.0001307786641121882,
"loss_nats": 1.1157338619232178,
"step": 4820,
"unclipped_grad_norm": 0.6775097250938416
},
{
"bits_per_byte": 1.6126245628657718,
"learning_rate": 0.00013023913905810876,
"loss_nats": 1.117786169052124,
"step": 4830,
"unclipped_grad_norm": 0.7250232696533203
},
{
"bits_per_byte": 1.6110075819823597,
"learning_rate": 0.00012970020797123432,
"loss_nats": 1.1166653633117676,
"step": 4840,
"unclipped_grad_norm": 0.6946505308151245
},
{
"bits_per_byte": 1.560937584906067,
"learning_rate": 0.00012916188006041027,
"loss_nats": 1.0819594860076904,
"step": 4850,
"unclipped_grad_norm": 0.7250073552131653
},
{
"bits_per_byte": 1.5705242418304615,
"learning_rate": 0.00012862416452417516,
"loss_nats": 1.08860445022583,
"step": 4860,
"unclipped_grad_norm": 0.6968505382537842
},
{
"bits_per_byte": 1.5589560008028256,
"learning_rate": 0.00012808707055060405,
"loss_nats": 1.0805859565734863,
"step": 4870,
"unclipped_grad_norm": 0.659229040145874
},
{
"bits_per_byte": 1.7343542271117696,
"learning_rate": 0.000127550607317151,
"loss_nats": 1.202162742614746,
"step": 4880,
"unclipped_grad_norm": 0.7630480527877808
},
{
"bits_per_byte": 1.5109135071975544,
"learning_rate": 0.00012701478399049248,
"loss_nats": 1.0472854375839233,
"step": 4890,
"unclipped_grad_norm": 0.6373679041862488
},
{
"bits_per_byte": 1.6030399697331876,
"learning_rate": 0.0001264796097263708,
"loss_nats": 1.111142635345459,
"step": 4900,
"unclipped_grad_norm": 0.6882696151733398
},
{
"bits_per_byte": 1.6017267102111992,
"learning_rate": 0.0001259450936694375,
"loss_nats": 1.1102323532104492,
"step": 4910,
"unclipped_grad_norm": 0.7222312688827515
},
{
"bits_per_byte": 1.533604038239139,
"learning_rate": 0.00012541124495309727,
"loss_nats": 1.0630133152008057,
"step": 4920,
"unclipped_grad_norm": 0.6319286227226257
},
{
"bits_per_byte": 1.5357895937663035,
"learning_rate": 0.00012487807269935167,
"loss_nats": 1.064528226852417,
"step": 4930,
"unclipped_grad_norm": 0.6290988922119141
},
{
"bits_per_byte": 1.5644580697692285,
"learning_rate": 0.00012434558601864349,
"loss_nats": 1.084399700164795,
"step": 4940,
"unclipped_grad_norm": 0.6635708808898926
},
{
"bits_per_byte": 1.596738869370876,
"learning_rate": 0.0001238137940097009,
"loss_nats": 1.1067750453948975,
"step": 4950,
"unclipped_grad_norm": 0.7171201109886169
},
{
"bits_per_byte": 1.6163304450265636,
"learning_rate": 0.0001232827057593821,
"loss_nats": 1.1203548908233643,
"step": 4960,
"unclipped_grad_norm": 0.739273726940155
},
{
"bits_per_byte": 1.566945798813959,
"learning_rate": 0.00012275233034251992,
"loss_nats": 1.086124062538147,
"step": 4970,
"unclipped_grad_norm": 0.6495019197463989
},
{
"bits_per_byte": 1.623716067985216,
"learning_rate": 0.0001222226768217669,
"loss_nats": 1.125474214553833,
"step": 4980,
"unclipped_grad_norm": 0.7210032939910889
},
{
"bits_per_byte": 1.574822088275523,
"learning_rate": 0.00012169375424744034,
"loss_nats": 1.091583490371704,
"step": 4990,
"unclipped_grad_norm": 0.6862797737121582
},
{
"bits_per_byte": 1.5840169687212522,
"learning_rate": 0.00012116557165736772,
"loss_nats": 1.097956895828247,
"step": 5000,
"unclipped_grad_norm": 0.6914816498756409
},
{
"bits_per_byte": 1.6270380848692823,
"learning_rate": 0.00012063813807673214,
"loss_nats": 1.127776861190796,
"step": 5010,
"unclipped_grad_norm": 0.7159916758537292
},
{
"bits_per_byte": 1.5917599716283977,
"learning_rate": 0.00012011146251791832,
"loss_nats": 1.1033239364624023,
"step": 5020,
"unclipped_grad_norm": 0.7277734875679016
},
{
"bits_per_byte": 1.5241281381418998,
"learning_rate": 0.0001195855539803584,
"loss_nats": 1.0564451217651367,
"step": 5030,
"unclipped_grad_norm": 0.6716130375862122
},
{
"bits_per_byte": 1.5837488477685573,
"learning_rate": 0.00011906042145037817,
"loss_nats": 1.0977710485458374,
"step": 5040,
"unclipped_grad_norm": 0.6623762845993042
},
{
"bits_per_byte": 1.554003760371231,
"learning_rate": 0.0001185360739010438,
"loss_nats": 1.0771533250808716,
"step": 5050,
"unclipped_grad_norm": 0.7006968855857849
},
{
"bits_per_byte": 1.621560093474,
"learning_rate": 0.00011801252029200808,
"loss_nats": 1.1239798069000244,
"step": 5060,
"unclipped_grad_norm": 0.6823760271072388
},
{
"bits_per_byte": 1.5589410383122004,
"learning_rate": 0.00011748976956935777,
"loss_nats": 1.0805755853652954,
"step": 5070,
"unclipped_grad_norm": 0.685271680355072
},
{
"bits_per_byte": 1.6273230601217619,
"learning_rate": 0.00011696783066546024,
"loss_nats": 1.1279743909835815,
"step": 5080,
"unclipped_grad_norm": 0.725353479385376
},
{
"bits_per_byte": 1.5689380458415538,
"learning_rate": 0.00011644671249881147,
"loss_nats": 1.0875049829483032,
"step": 5090,
"unclipped_grad_norm": 0.6860542297363281
},
{
"bits_per_byte": 1.5571642855461367,
"learning_rate": 0.00011592642397388293,
"loss_nats": 1.0793440341949463,
"step": 5100,
"unclipped_grad_norm": 0.7099642753601074
},
{
"bits_per_byte": 1.593192587110085,
"learning_rate": 0.00011540697398097015,
"loss_nats": 1.1043169498443604,
"step": 5110,
"unclipped_grad_norm": 0.683228075504303
},
{
"bits_per_byte": 1.5025353723607726,
"learning_rate": 0.00011488837139604016,
"loss_nats": 1.041478157043457,
"step": 5120,
"unclipped_grad_norm": 0.6536243557929993
},
{
"bits_per_byte": 1.6259317204762818,
"learning_rate": 0.00011437062508058039,
"loss_nats": 1.1270099878311157,
"step": 5130,
"unclipped_grad_norm": 0.7276868224143982
},
{
"bits_per_byte": 1.5937286570328242,
"learning_rate": 0.00011385374388144673,
"loss_nats": 1.1046885251998901,
"step": 5140,
"unclipped_grad_norm": 0.7137117981910706
},
{
"bits_per_byte": 1.5363523209999266,
"learning_rate": 0.00011333773663071286,
"loss_nats": 1.0649182796478271,
"step": 5150,
"unclipped_grad_norm": 0.6770874857902527
},
{
"bits_per_byte": 1.5548322008004372,
"learning_rate": 0.00011282261214551897,
"loss_nats": 1.0777275562286377,
"step": 5160,
"unclipped_grad_norm": 0.6993823051452637
},
{
"bits_per_byte": 1.6209285731800318,
"learning_rate": 0.00011230837922792112,
"loss_nats": 1.123542070388794,
"step": 5170,
"unclipped_grad_norm": 0.7498713135719299
},
{
"bits_per_byte": 1.5887963665887297,
"learning_rate": 0.0001117950466647412,
"loss_nats": 1.1012697219848633,
"step": 5180,
"unclipped_grad_norm": 0.7006720304489136
},
{
"bits_per_byte": 1.4814539110006604,
"learning_rate": 0.00011128262322741627,
"loss_nats": 1.0268656015396118,
"step": 5190,
"unclipped_grad_norm": 0.6951001286506653
},
{
"bits_per_byte": 1.5501159205658264,
"learning_rate": 0.00011077111767184913,
"loss_nats": 1.0744584798812866,
"step": 5200,
"unclipped_grad_norm": 0.696289598941803
},
{
"bits_per_byte": 1.519997458833477,
"learning_rate": 0.00011026053873825837,
"loss_nats": 1.053581953048706,
"step": 5210,
"unclipped_grad_norm": 0.6860307455062866
},
{
"bits_per_byte": 1.5330389032484038,
"learning_rate": 0.0001097508951510293,
"loss_nats": 1.0626215934753418,
"step": 5220,
"unclipped_grad_norm": 0.6861689686775208
},
{
"bits_per_byte": 1.6136053799236425,
"learning_rate": 0.0001092421956185646,
"loss_nats": 1.1184660196304321,
"step": 5230,
"unclipped_grad_norm": 0.6980286836624146
},
{
"bits_per_byte": 1.5805682006235005,
"learning_rate": 0.00010873444883313581,
"loss_nats": 1.0955663919448853,
"step": 5240,
"unclipped_grad_norm": 0.7267829775810242
},
{
"bits_per_byte": 1.5736680846882336,
"learning_rate": 0.00010822766347073447,
"loss_nats": 1.0907835960388184,
"step": 5250,
"unclipped_grad_norm": 0.6905968189239502
},
{
"bits_per_byte": 1.5160989561037188,
"learning_rate": 0.00010772184819092424,
"loss_nats": 1.050879716873169,
"step": 5260,
"unclipped_grad_norm": 0.7048349380493164
},
{
"bits_per_byte": 1.5706724908755052,
"learning_rate": 0.0001072170116366926,
"loss_nats": 1.0887072086334229,
"step": 5270,
"unclipped_grad_norm": 0.7007996439933777
},
{
"bits_per_byte": 1.4955876172312168,
"learning_rate": 0.00010671316243430345,
"loss_nats": 1.0366623401641846,
"step": 5280,
"unclipped_grad_norm": 0.7013132572174072
},
{
"bits_per_byte": 1.524417756925953,
"learning_rate": 0.00010621030919314944,
"loss_nats": 1.0566458702087402,
"step": 5290,
"unclipped_grad_norm": 0.6891999840736389
},
{
"bits_per_byte": 1.4950008124264729,
"learning_rate": 0.000105708460505605,
"loss_nats": 1.0362555980682373,
"step": 5300,
"unclipped_grad_norm": 0.6865445971488953
},
{
"bits_per_byte": 1.4986692023693766,
"learning_rate": 0.00010520762494687964,
"loss_nats": 1.0387983322143555,
"step": 5310,
"unclipped_grad_norm": 0.6979682445526123
},
{
"bits_per_byte": 1.5140539104023092,
"learning_rate": 0.00010470781107487111,
"loss_nats": 1.0494621992111206,
"step": 5320,
"unclipped_grad_norm": 0.7112694978713989
},
{
"bits_per_byte": 1.5563401446832021,
"learning_rate": 0.00010420902743001955,
"loss_nats": 1.078772783279419,
"step": 5330,
"unclipped_grad_norm": 0.7123240828514099
},
{
"bits_per_byte": 1.620746271570117,
"learning_rate": 0.00010371128253516117,
"loss_nats": 1.1234157085418701,
"step": 5340,
"unclipped_grad_norm": 0.7256788611412048
},
{
"bits_per_byte": 1.4811187168141287,
"learning_rate": 0.000103214584895383,
"loss_nats": 1.0266332626342773,
"step": 5350,
"unclipped_grad_norm": 0.6512540578842163
},
{
"bits_per_byte": 1.5334795227999143,
"learning_rate": 0.00010271894299787715,
"loss_nats": 1.062927007675171,
"step": 5360,
"unclipped_grad_norm": 0.7001609206199646
},
{
"bits_per_byte": 1.6031871868823262,
"learning_rate": 0.00010222436531179622,
"loss_nats": 1.1112446784973145,
"step": 5370,
"unclipped_grad_norm": 0.7270175814628601
},
{
"bits_per_byte": 1.5079560935333174,
"learning_rate": 0.0001017308602881082,
"loss_nats": 1.045235514640808,
"step": 5380,
"unclipped_grad_norm": 0.6764644980430603
},
{
"bits_per_byte": 1.5113635857948622,
"learning_rate": 0.00010123843635945242,
"loss_nats": 1.0475974082946777,
"step": 5390,
"unclipped_grad_norm": 0.7015690803527832
},
{
"bits_per_byte": 1.567986121869028,
"learning_rate": 0.00010074710193999505,
"loss_nats": 1.0868451595306396,
"step": 5400,
"unclipped_grad_norm": 0.7621549963951111
},
{
"bits_per_byte": 1.456367317667255,
"learning_rate": 0.0001002568654252858,
"loss_nats": 1.009476900100708,
"step": 5410,
"unclipped_grad_norm": 0.7006528377532959
},
{
"bits_per_byte": 1.6203708334432836,
"learning_rate": 9.976773519211404e-05,
"loss_nats": 1.1231554746627808,
"step": 5420,
"unclipped_grad_norm": 0.7240427136421204
},
{
"bits_per_byte": 1.543870886547696,
"learning_rate": 9.92797195983659e-05,
"loss_nats": 1.0701297521591187,
"step": 5430,
"unclipped_grad_norm": 0.7349725961685181
},
{
"bits_per_byte": 1.5391332804643783,
"learning_rate": 9.879282698288142e-05,
"loss_nats": 1.0668458938598633,
"step": 5440,
"unclipped_grad_norm": 0.6845248937606812
},
{
"bits_per_byte": 1.5639070373558641,
"learning_rate": 9.830706566531197e-05,
"loss_nats": 1.0840177536010742,
"step": 5450,
"unclipped_grad_norm": 0.7343961596488953
},
{
"bits_per_byte": 1.5791133993798518,
"learning_rate": 9.782244394597826e-05,
"loss_nats": 1.0945580005645752,
"step": 5460,
"unclipped_grad_norm": 0.6990759968757629
},
{
"bits_per_byte": 1.5450971228483479,
"learning_rate": 9.733897010572829e-05,
"loss_nats": 1.0709797143936157,
"step": 5470,
"unclipped_grad_norm": 0.73023521900177
},
{
"bits_per_byte": 1.4896457886265577,
"learning_rate": 9.685665240579612e-05,
"loss_nats": 1.0325437784194946,
"step": 5480,
"unclipped_grad_norm": 0.7134303450584412
},
{
"bits_per_byte": 1.5553481487530256,
"learning_rate": 9.637549908766041e-05,
"loss_nats": 1.07808518409729,
"step": 5490,
"unclipped_grad_norm": 0.713401734828949
},
{
"bits_per_byte": 1.5796455137016214,
"learning_rate": 9.589551837290389e-05,
"loss_nats": 1.0949268341064453,
"step": 5500,
"unclipped_grad_norm": 0.7012893557548523
},
{
"bits_per_byte": 1.56096613402611,
"learning_rate": 9.541671846307257e-05,
"loss_nats": 1.0819792747497559,
"step": 5510,
"unclipped_grad_norm": 0.7354693412780762
},
{
"bits_per_byte": 1.558417523122974,
"learning_rate": 9.493910753953597e-05,
"loss_nats": 1.0802127122879028,
"step": 5520,
"unclipped_grad_norm": 0.8108682036399841
},
{
"bits_per_byte": 1.572987033390817,
"learning_rate": 9.446269376334689e-05,
"loss_nats": 1.0903115272521973,
"step": 5530,
"unclipped_grad_norm": 0.813023567199707
},
{
"bits_per_byte": 1.603627462468535,
"learning_rate": 9.39874852751024e-05,
"loss_nats": 1.1115498542785645,
"step": 5540,
"unclipped_grad_norm": 0.7556710243225098
},
{
"bits_per_byte": 1.5354511319094055,
"learning_rate": 9.351349019480433e-05,
"loss_nats": 1.064293622970581,
"step": 5550,
"unclipped_grad_norm": 0.6996996998786926
},
{
"bits_per_byte": 1.531031521747533,
"learning_rate": 9.304071662172086e-05,
"loss_nats": 1.061230182647705,
"step": 5560,
"unclipped_grad_norm": 0.7085531949996948
},
{
"bits_per_byte": 1.5727634559446952,
"learning_rate": 9.256917263424787e-05,
"loss_nats": 1.0901565551757812,
"step": 5570,
"unclipped_grad_norm": 0.7508514523506165
},
{
"bits_per_byte": 1.4695356852785186,
"learning_rate": 9.209886628977114e-05,
"loss_nats": 1.0186045169830322,
"step": 5580,
"unclipped_grad_norm": 0.6981281042098999
},
{
"bits_per_byte": 1.5444360215384312,
"learning_rate": 9.162980562452843e-05,
"loss_nats": 1.0705214738845825,
"step": 5590,
"unclipped_grad_norm": 0.7099512815475464
},
{
"bits_per_byte": 1.5326933901028204,
"learning_rate": 9.116199865347238e-05,
"loss_nats": 1.0623821020126343,
"step": 5600,
"unclipped_grad_norm": 0.7117766737937927
},
{
"bits_per_byte": 1.5419961036706404,
"learning_rate": 9.069545337013333e-05,
"loss_nats": 1.0688302516937256,
"step": 5610,
"unclipped_grad_norm": 0.7040424942970276
},
{
"bits_per_byte": 1.4775775940328635,
"learning_rate": 9.0230177746483e-05,
"loss_nats": 1.0241787433624268,
"step": 5620,
"unclipped_grad_norm": 0.7147260308265686
},
{
"bits_per_byte": 1.4921469323180554,
"learning_rate": 8.976617973279797e-05,
"loss_nats": 1.0342774391174316,
"step": 5630,
"unclipped_grad_norm": 0.7185394763946533
},
{
"bits_per_byte": 1.4712231790487844,
"learning_rate": 8.930346725752419e-05,
"loss_nats": 1.0197741985321045,
"step": 5640,
"unclipped_grad_norm": 0.7179238796234131
},
{
"bits_per_byte": 1.5902296700010203,
"learning_rate": 8.88420482271411e-05,
"loss_nats": 1.1022632122039795,
"step": 5650,
"unclipped_grad_norm": 0.7511428594589233
},
{
"bits_per_byte": 1.541337238135185,
"learning_rate": 8.838193052602691e-05,
"loss_nats": 1.0683735609054565,
"step": 5660,
"unclipped_grad_norm": 0.7174038887023926
},
{
"bits_per_byte": 1.5341244577406499,
"learning_rate": 8.792312201632352e-05,
"loss_nats": 1.0633740425109863,
"step": 5670,
"unclipped_grad_norm": 0.7094559669494629
},
{
"bits_per_byte": 1.5432753106277581,
"learning_rate": 8.746563053780261e-05,
"loss_nats": 1.0697169303894043,
"step": 5680,
"unclipped_grad_norm": 0.7527599334716797
},
{
"bits_per_byte": 1.5847210656938846,
"learning_rate": 8.700946390773114e-05,
"loss_nats": 1.098444938659668,
"step": 5690,
"unclipped_grad_norm": 0.7476640939712524
},
{
"bits_per_byte": 1.5107903676195364,
"learning_rate": 8.655462992073833e-05,
"loss_nats": 1.047200083732605,
"step": 5700,
"unclipped_grad_norm": 0.7240637540817261
},
{
"bits_per_byte": 1.5529789157547396,
"learning_rate": 8.610113634868199e-05,
"loss_nats": 1.0764429569244385,
"step": 5710,
"unclipped_grad_norm": 0.7409384250640869
},
{
"bits_per_byte": 1.5779322225337262,
"learning_rate": 8.564899094051613e-05,
"loss_nats": 1.0937392711639404,
"step": 5720,
"unclipped_grad_norm": 0.7523439526557922
},
{
"bits_per_byte": 1.4812139952027068,
"learning_rate": 8.519820142215816e-05,
"loss_nats": 1.0266993045806885,
"step": 5730,
"unclipped_grad_norm": 0.6902204155921936
},
{
"bits_per_byte": 1.5538140635073292,
"learning_rate": 8.474877549635728e-05,
"loss_nats": 1.077021837234497,
"step": 5740,
"unclipped_grad_norm": 0.7577133178710938
},
{
"bits_per_byte": 1.5073047952345,
"learning_rate": 8.430072084256247e-05,
"loss_nats": 1.0447840690612793,
"step": 5750,
"unclipped_grad_norm": 0.7320472002029419
},
{
"bits_per_byte": 1.5094478710469015,
"learning_rate": 8.385404511679161e-05,
"loss_nats": 1.0462695360183716,
"step": 5760,
"unclipped_grad_norm": 0.7613655924797058
},
{
"bits_per_byte": 1.4626881960344158,
"learning_rate": 8.340875595150034e-05,
"loss_nats": 1.0138581991195679,
"step": 5770,
"unclipped_grad_norm": 0.7212797403335571
},
{
"bits_per_byte": 1.5541401426133652,
"learning_rate": 8.296486095545203e-05,
"loss_nats": 1.0772478580474854,
"step": 5780,
"unclipped_grad_norm": 0.7401144504547119
},
{
"bits_per_byte": 1.4976115090665703,
"learning_rate": 8.252236771358729e-05,
"loss_nats": 1.0380651950836182,
"step": 5790,
"unclipped_grad_norm": 0.6966454386711121
},
{
"bits_per_byte": 1.508392929466509,
"learning_rate": 8.208128378689484e-05,
"loss_nats": 1.045538306236267,
"step": 5800,
"unclipped_grad_norm": 0.7519288063049316
},
{
"bits_per_byte": 1.536275616737642,
"learning_rate": 8.16416167122819e-05,
"loss_nats": 1.0648651123046875,
"step": 5810,
"unclipped_grad_norm": 0.7322137355804443
},
{
"bits_per_byte": 1.5318357126229676,
"learning_rate": 8.120337400244578e-05,
"loss_nats": 1.0617876052856445,
"step": 5820,
"unclipped_grad_norm": 0.7490501999855042
},
{
"bits_per_byte": 1.5069907549140245,
"learning_rate": 8.076656314574521e-05,
"loss_nats": 1.0445663928985596,
"step": 5830,
"unclipped_grad_norm": 0.7249871492385864
},
{
"bits_per_byte": 1.4694756633333674,
"learning_rate": 8.03311916060725e-05,
"loss_nats": 1.018562912940979,
"step": 5840,
"unclipped_grad_norm": 0.7599647641181946
},
{
"bits_per_byte": 1.4762989030236986,
"learning_rate": 7.989726682272606e-05,
"loss_nats": 1.0232924222946167,
"step": 5850,
"unclipped_grad_norm": 0.7097748517990112
},
{
"bits_per_byte": 1.502543627528014,
"learning_rate": 7.946479621028318e-05,
"loss_nats": 1.0414838790893555,
"step": 5860,
"unclipped_grad_norm": 0.7764433026313782
},
{
"bits_per_byte": 1.5481047554466367,
"learning_rate": 7.903378715847343e-05,
"loss_nats": 1.0730644464492798,
"step": 5870,
"unclipped_grad_norm": 0.770169734954834
},
{
"bits_per_byte": 1.5544658777540994,
"learning_rate": 7.860424703205224e-05,
"loss_nats": 1.0774736404418945,
"step": 5880,
"unclipped_grad_norm": 0.731796145439148
},
{
"bits_per_byte": 1.528056737835559,
"learning_rate": 7.817618317067534e-05,
"loss_nats": 1.0591682195663452,
"step": 5890,
"unclipped_grad_norm": 0.7195150852203369
},
{
"bits_per_byte": 1.5162458292875556,
"learning_rate": 7.774960288877295e-05,
"loss_nats": 1.0509815216064453,
"step": 5900,
"unclipped_grad_norm": 0.8138624429702759
},
{
"bits_per_byte": 1.5072279189895643,
"learning_rate": 7.732451347542522e-05,
"loss_nats": 1.04473078250885,
"step": 5910,
"unclipped_grad_norm": 0.7780150771141052
},
{
"bits_per_byte": 1.514708304388842,
"learning_rate": 7.690092219423722e-05,
"loss_nats": 1.0499157905578613,
"step": 5920,
"unclipped_grad_norm": 0.7354658246040344
},
{
"bits_per_byte": 1.5573187259666117,
"learning_rate": 7.647883628321536e-05,
"loss_nats": 1.079451084136963,
"step": 5930,
"unclipped_grad_norm": 0.7978370785713196
},
{
"bits_per_byte": 1.5533937379086207,
"learning_rate": 7.605826295464316e-05,
"loss_nats": 1.076730489730835,
"step": 5940,
"unclipped_grad_norm": 0.8458540439605713
},
{
"bits_per_byte": 1.5137727187681485,
"learning_rate": 7.563920939495851e-05,
"loss_nats": 1.049267292022705,
"step": 5950,
"unclipped_grad_norm": 0.7532646059989929
},
{
"bits_per_byte": 1.5086940710881698,
"learning_rate": 7.522168276463047e-05,
"loss_nats": 1.0457470417022705,
"step": 5960,
"unclipped_grad_norm": 0.7333019375801086
},
{
"bits_per_byte": 1.5303000795334134,
"learning_rate": 7.480569019803719e-05,
"loss_nats": 1.0607231855392456,
"step": 5970,
"unclipped_grad_norm": 0.8049731254577637
},
{
"bits_per_byte": 1.4828514420215715,
"learning_rate": 7.439123880334388e-05,
"loss_nats": 1.0278342962265015,
"step": 5980,
"unclipped_grad_norm": 0.7353474497795105
},
{
"bits_per_byte": 1.4545697550004368,
"learning_rate": 7.39783356623813e-05,
"loss_nats": 1.0082309246063232,
"step": 5990,
"unclipped_grad_norm": 0.737419068813324
},
{
"bits_per_byte": 1.5419821730759207,
"learning_rate": 7.356698783052497e-05,
"loss_nats": 1.068820595741272,
"step": 6000,
"unclipped_grad_norm": 0.7629180550575256
},
{
"bits_per_byte": 1.5277340983825405,
"learning_rate": 7.315720233657431e-05,
"loss_nats": 1.058944582939148,
"step": 6010,
"unclipped_grad_norm": 0.7620755434036255
},
{
"bits_per_byte": 1.4955477172562166,
"learning_rate": 7.274898618263286e-05,
"loss_nats": 1.0366346836090088,
"step": 6020,
"unclipped_grad_norm": 0.7292536497116089
},
{
"bits_per_byte": 1.5533901262729524,
"learning_rate": 7.234234634398831e-05,
"loss_nats": 1.0767279863357544,
"step": 6030,
"unclipped_grad_norm": 0.767517626285553
},
{
"bits_per_byte": 1.495873108431649,
"learning_rate": 7.193728976899362e-05,
"loss_nats": 1.0368602275848389,
"step": 6040,
"unclipped_grad_norm": 0.7536795735359192
},
{
"bits_per_byte": 1.5050292927809341,
"learning_rate": 7.153382337894797e-05,
"loss_nats": 1.043206810951233,
"step": 6050,
"unclipped_grad_norm": 0.7510635256767273
},
{
"bits_per_byte": 1.556982155918873,
"learning_rate": 7.113195406797887e-05,
"loss_nats": 1.079217791557312,
"step": 6060,
"unclipped_grad_norm": 0.7961422801017761
},
{
"bits_per_byte": 1.5009287104364122,
"learning_rate": 7.073168870292397e-05,
"loss_nats": 1.0403645038604736,
"step": 6070,
"unclipped_grad_norm": 0.7310256958007812
},
{
"bits_per_byte": 1.5023740526342633,
"learning_rate": 7.033303412321404e-05,
"loss_nats": 1.0413663387298584,
"step": 6080,
"unclipped_grad_norm": 0.7653129696846008
},
{
"bits_per_byte": 1.5019286175685285,
"learning_rate": 6.993599714075586e-05,
"loss_nats": 1.0410575866699219,
"step": 6090,
"unclipped_grad_norm": 0.7687012553215027
},
{
"bits_per_byte": 1.5089062976793344,
"learning_rate": 6.954058453981609e-05,
"loss_nats": 1.0458941459655762,
"step": 6100,
"unclipped_grad_norm": 0.7417535185813904
},
{
"bits_per_byte": 1.475459455704837,
"learning_rate": 6.914680307690504e-05,
"loss_nats": 1.0227105617523193,
"step": 6110,
"unclipped_grad_norm": 0.7481279969215393
},
{
"bits_per_byte": 1.4820735644917191,
"learning_rate": 6.875465948066139e-05,
"loss_nats": 1.0272951126098633,
"step": 6120,
"unclipped_grad_norm": 0.7211357951164246
},
{
"bits_per_byte": 1.5036838725032344,
"learning_rate": 6.836416045173733e-05,
"loss_nats": 1.0422742366790771,
"step": 6130,
"unclipped_grad_norm": 0.7536711096763611
},
{
"bits_per_byte": 1.5397933498783898,
"learning_rate": 6.797531266268377e-05,
"loss_nats": 1.0673034191131592,
"step": 6140,
"unclipped_grad_norm": 0.7828236818313599
},
{
"bits_per_byte": 1.4767353949915882,
"learning_rate": 6.758812275783663e-05,
"loss_nats": 1.0235949754714966,
"step": 6150,
"unclipped_grad_norm": 0.7367910742759705
},
{
"bits_per_byte": 1.563501158299828,
"learning_rate": 6.720259735320298e-05,
"loss_nats": 1.0837364196777344,
"step": 6160,
"unclipped_grad_norm": 0.7933995723724365
},
{
"bits_per_byte": 1.433251129564785,
"learning_rate": 6.681874303634842e-05,
"loss_nats": 0.9934539794921875,
"step": 6170,
"unclipped_grad_norm": 0.728959321975708
},
{
"bits_per_byte": 1.4945863342378936,
"learning_rate": 6.643656636628402e-05,
"loss_nats": 1.03596830368042,
"step": 6180,
"unclipped_grad_norm": 0.7535345554351807
},
{
"bits_per_byte": 1.5074559679846085,
"learning_rate": 6.605607387335466e-05,
"loss_nats": 1.0448888540267944,
"step": 6190,
"unclipped_grad_norm": 0.7919867038726807
},
{
"bits_per_byte": 1.4628087558726706,
"learning_rate": 6.567727205912724e-05,
"loss_nats": 1.013941764831543,
"step": 6200,
"unclipped_grad_norm": 0.7595178484916687
},
{
"bits_per_byte": 1.4945321597028718,
"learning_rate": 6.530016739627959e-05,
"loss_nats": 1.0359307527542114,
"step": 6210,
"unclipped_grad_norm": 0.7536692023277283
},
{
"bits_per_byte": 1.4821234394604692,
"learning_rate": 6.492476632848993e-05,
"loss_nats": 1.027329683303833,
"step": 6220,
"unclipped_grad_norm": 0.7611709833145142
},
{
"bits_per_byte": 1.4502368240945993,
"learning_rate": 6.455107527032678e-05,
"loss_nats": 1.0052275657653809,
"step": 6230,
"unclipped_grad_norm": 0.7465225458145142
},
{
"bits_per_byte": 1.4792031740488187,
"learning_rate": 6.417910060713926e-05,
"loss_nats": 1.0253055095672607,
"step": 6240,
"unclipped_grad_norm": 0.8008144497871399
},
{
"bits_per_byte": 1.4709335602647315,
"learning_rate": 6.380884869494799e-05,
"loss_nats": 1.019573450088501,
"step": 6250,
"unclipped_grad_norm": 0.7485248446464539
},
{
"bits_per_byte": 1.5311833824282448,
"learning_rate": 6.34403258603367e-05,
"loss_nats": 1.0613354444503784,
"step": 6260,
"unclipped_grad_norm": 0.7729804515838623
},
{
"bits_per_byte": 1.494225858601685,
"learning_rate": 6.30735384003437e-05,
"loss_nats": 1.0357184410095215,
"step": 6270,
"unclipped_grad_norm": 0.7719584703445435
},
{
"bits_per_byte": 1.606436799070379,
"learning_rate": 6.270849258235484e-05,
"loss_nats": 1.1134971380233765,
"step": 6280,
"unclipped_grad_norm": 0.814758837223053
},
{
"bits_per_byte": 1.469587624039079,
"learning_rate": 6.234519464399581e-05,
"loss_nats": 1.0186405181884766,
"step": 6290,
"unclipped_grad_norm": 0.7629152536392212
},
{
"bits_per_byte": 1.4932348945674137,
"learning_rate": 6.198365079302614e-05,
"loss_nats": 1.0350315570831299,
"step": 6300,
"unclipped_grad_norm": 0.7575296759605408
},
{
"bits_per_byte": 1.5423225267419782,
"learning_rate": 6.16238672072326e-05,
"loss_nats": 1.069056510925293,
"step": 6310,
"unclipped_grad_norm": 0.7936079502105713
},
{
"bits_per_byte": 1.497591903044372,
"learning_rate": 6.126585003432406e-05,
"loss_nats": 1.0380516052246094,
"step": 6320,
"unclipped_grad_norm": 0.7546678781509399
},
{
"bits_per_byte": 1.438224007914483,
"learning_rate": 6.090960539182614e-05,
"loss_nats": 0.9969009160995483,
"step": 6330,
"unclipped_grad_norm": 0.7539783716201782
},
{
"bits_per_byte": 1.5101449167308483,
"learning_rate": 6.0555139366976935e-05,
"loss_nats": 1.046752691268921,
"step": 6340,
"unclipped_grad_norm": 0.7760982513427734
},
{
"bits_per_byte": 1.5028686747381446,
"learning_rate": 6.020245801662274e-05,
"loss_nats": 1.0417091846466064,
"step": 6350,
"unclipped_grad_norm": 0.8006681203842163
},
{
"bits_per_byte": 1.4519638738745637,
"learning_rate": 5.985156736711483e-05,
"loss_nats": 1.0064246654510498,
"step": 6360,
"unclipped_grad_norm": 0.7430040240287781
},
{
"bits_per_byte": 1.5063248380898837,
"learning_rate": 5.950247341420618e-05,
"loss_nats": 1.044104814529419,
"step": 6370,
"unclipped_grad_norm": 0.8066008687019348
},
{
"bits_per_byte": 1.461864399136783,
"learning_rate": 5.915518212294935e-05,
"loss_nats": 1.0132871866226196,
"step": 6380,
"unclipped_grad_norm": 0.739668071269989
},
{
"bits_per_byte": 1.518872692296834,
"learning_rate": 5.880969942759425e-05,
"loss_nats": 1.052802324295044,
"step": 6390,
"unclipped_grad_norm": 0.8261381983757019
},
{
"bits_per_byte": 1.4734527621345699,
"learning_rate": 5.846603123148688e-05,
"loss_nats": 1.0213196277618408,
"step": 6400,
"unclipped_grad_norm": 0.7401824593544006
},
{
"bits_per_byte": 1.4482483606853236,
"learning_rate": 5.812418340696858e-05,
"loss_nats": 1.0038492679595947,
"step": 6410,
"unclipped_grad_norm": 0.7752094268798828
},
{
"bits_per_byte": 1.4492296936911466,
"learning_rate": 5.778416179527541e-05,
"loss_nats": 1.0045294761657715,
"step": 6420,
"unclipped_grad_norm": 0.7460422515869141
},
{
"bits_per_byte": 1.4798391658917094,
"learning_rate": 5.7445972206438635e-05,
"loss_nats": 1.0257463455200195,
"step": 6430,
"unclipped_grad_norm": 0.7759443521499634
},
{
"bits_per_byte": 1.4925646781770012,
"learning_rate": 5.710962041918516e-05,
"loss_nats": 1.0345669984817505,
"step": 6440,
"unclipped_grad_norm": 0.7866507768630981
},
{
"bits_per_byte": 1.473837315341899,
"learning_rate": 5.677511218083908e-05,
"loss_nats": 1.0215861797332764,
"step": 6450,
"unclipped_grad_norm": 0.7885896563529968
},
{
"bits_per_byte": 1.452434934355277,
"learning_rate": 5.644245320722316e-05,
"loss_nats": 1.0067511796951294,
"step": 6460,
"unclipped_grad_norm": 0.7844802737236023
},
{
"bits_per_byte": 1.4767341911130323,
"learning_rate": 5.61116491825615e-05,
"loss_nats": 1.0235941410064697,
"step": 6470,
"unclipped_grad_norm": 0.7592610716819763
},
{
"bits_per_byte": 1.5086992305676956,
"learning_rate": 5.5782705759382104e-05,
"loss_nats": 1.045750617980957,
"step": 6480,
"unclipped_grad_norm": 0.8035558462142944
},
{
"bits_per_byte": 1.4908353926225757,
"learning_rate": 5.5455628558420546e-05,
"loss_nats": 1.0333683490753174,
"step": 6490,
"unclipped_grad_norm": 0.8066015243530273
},
{
"bits_per_byte": 1.4884610001447638,
"learning_rate": 5.513042316852368e-05,
"loss_nats": 1.0317225456237793,
"step": 6500,
"unclipped_grad_norm": 0.7627172470092773
},
{
"bits_per_byte": 1.4363769142442164,
"learning_rate": 5.480709514655443e-05,
"loss_nats": 0.995620608329773,
"step": 6510,
"unclipped_grad_norm": 0.7699123024940491
},
{
"bits_per_byte": 1.4159479550614777,
"learning_rate": 5.448565001729653e-05,
"loss_nats": 0.9814603328704834,
"step": 6520,
"unclipped_grad_norm": 0.7812637090682983
},
{
"bits_per_byte": 1.498887964301274,
"learning_rate": 5.41660932733604e-05,
"loss_nats": 1.038949966430664,
"step": 6530,
"unclipped_grad_norm": 0.8116927742958069
},
{
"bits_per_byte": 1.4673561491441343,
"learning_rate": 5.384843037508902e-05,
"loss_nats": 1.0170937776565552,
"step": 6540,
"unclipped_grad_norm": 0.7371414303779602
},
{
"bits_per_byte": 1.5117546743429242,
"learning_rate": 5.353266675046498e-05,
"loss_nats": 1.0478684902191162,
"step": 6550,
"unclipped_grad_norm": 0.8187429308891296
},
{
"bits_per_byte": 1.4773364743563537,
"learning_rate": 5.321880779501729e-05,
"loss_nats": 1.0240116119384766,
"step": 6560,
"unclipped_grad_norm": 0.7700304388999939
},
{
"bits_per_byte": 1.4043874532531306,
"learning_rate": 5.290685887172962e-05,
"loss_nats": 0.9734472036361694,
"step": 6570,
"unclipped_grad_norm": 0.7884607315063477
},
{
"bits_per_byte": 1.4975829599465271,
"learning_rate": 5.2596825310948295e-05,
"loss_nats": 1.0380454063415527,
"step": 6580,
"unclipped_grad_norm": 0.778003454208374
},
{
"bits_per_byte": 1.4674713775202122,
"learning_rate": 5.228871241029154e-05,
"loss_nats": 1.0171736478805542,
"step": 6590,
"unclipped_grad_norm": 0.7916907668113708
},
{
"bits_per_byte": 1.3998400599816672,
"learning_rate": 5.198252543455865e-05,
"loss_nats": 0.9702951908111572,
"step": 6600,
"unclipped_grad_norm": 0.7509376406669617
},
{
"bits_per_byte": 1.5187203156681697,
"learning_rate": 5.167826961564027e-05,
"loss_nats": 1.052696704864502,
"step": 6610,
"unclipped_grad_norm": 0.7783279418945312
},
{
"bits_per_byte": 1.511926828976438,
"learning_rate": 5.137595015242884e-05,
"loss_nats": 1.0479878187179565,
"step": 6620,
"unclipped_grad_norm": 0.8419784903526306
},
{
"bits_per_byte": 1.5028970518755371,
"learning_rate": 5.1075572210729925e-05,
"loss_nats": 1.0417288541793823,
"step": 6630,
"unclipped_grad_norm": 0.8222247958183289
},
{
"bits_per_byte": 1.4773347545298452,
"learning_rate": 5.07771409231737e-05,
"loss_nats": 1.024010419845581,
"step": 6640,
"unclipped_grad_norm": 0.8660594820976257
},
{
"bits_per_byte": 1.4614148364874275,
"learning_rate": 5.048066138912756e-05,
"loss_nats": 1.0129755735397339,
"step": 6650,
"unclipped_grad_norm": 0.7894479036331177
},
{
"bits_per_byte": 1.5265555012761776,
"learning_rate": 5.018613867460863e-05,
"loss_nats": 1.0581276416778564,
"step": 6660,
"unclipped_grad_norm": 0.810829758644104
},
{
"bits_per_byte": 1.4984096805492246,
"learning_rate": 4.989357781219755e-05,
"loss_nats": 1.0386184453964233,
"step": 6670,
"unclipped_grad_norm": 0.8341873288154602
},
{
"bits_per_byte": 1.5059076081788905,
"learning_rate": 4.960298380095222e-05,
"loss_nats": 1.0438156127929688,
"step": 6680,
"unclipped_grad_norm": 0.822468638420105
},
{
"bits_per_byte": 1.4780378196065724,
"learning_rate": 4.931436160632256e-05,
"loss_nats": 1.0244977474212646,
"step": 6690,
"unclipped_grad_norm": 0.810896635055542
},
{
"bits_per_byte": 1.524204498438883,
"learning_rate": 4.9027716160065485e-05,
"loss_nats": 1.0564980506896973,
"step": 6700,
"unclipped_grad_norm": 0.8146765828132629
},
{
"bits_per_byte": 1.451385668202363,
"learning_rate": 4.874305236016084e-05,
"loss_nats": 1.00602388381958,
"step": 6710,
"unclipped_grad_norm": 0.8321484327316284
},
{
"bits_per_byte": 1.4556473982907434,
"learning_rate": 4.846037507072752e-05,
"loss_nats": 1.0089778900146484,
"step": 6720,
"unclipped_grad_norm": 0.781524658203125
},
{
"bits_per_byte": 1.448410712307738,
"learning_rate": 4.817968912194049e-05,
"loss_nats": 1.0039618015289307,
"step": 6730,
"unclipped_grad_norm": 0.8034297227859497
},
{
"bits_per_byte": 1.4779710903380376,
"learning_rate": 4.790099930994813e-05,
"loss_nats": 1.024451494216919,
"step": 6740,
"unclipped_grad_norm": 0.8218453526496887
},
{
"bits_per_byte": 1.4435792036970492,
"learning_rate": 4.762431039679047e-05,
"loss_nats": 1.0006128549575806,
"step": 6750,
"unclipped_grad_norm": 0.7989961504936218
},
{
"bits_per_byte": 1.4296412137231747,
"learning_rate": 4.734962711031755e-05,
"loss_nats": 0.9909517765045166,
"step": 6760,
"unclipped_grad_norm": 0.7599804997444153
},
{
"bits_per_byte": 1.4469161830717403,
"learning_rate": 4.707695414410892e-05,
"loss_nats": 1.0029258728027344,
"step": 6770,
"unclipped_grad_norm": 0.7921444773674011
},
{
"bits_per_byte": 1.438630832875099,
"learning_rate": 4.680629615739322e-05,
"loss_nats": 0.9971829056739807,
"step": 6780,
"unclipped_grad_norm": 0.7756091356277466
},
{
"bits_per_byte": 1.5384264317693321,
"learning_rate": 4.653765777496863e-05,
"loss_nats": 1.0663559436798096,
"step": 6790,
"unclipped_grad_norm": 0.8340134024620056
},
{
"bits_per_byte": 1.5144365718004789,
"learning_rate": 4.6271043587123986e-05,
"loss_nats": 1.049727439880371,
"step": 6800,
"unclipped_grad_norm": 0.8667078614234924
},
{
"bits_per_byte": 1.4971179188585941,
"learning_rate": 4.600645814956006e-05,
"loss_nats": 1.0377230644226074,
"step": 6810,
"unclipped_grad_norm": 0.8064154386520386
},
{
"bits_per_byte": 1.4405763866129846,
"learning_rate": 4.574390598331204e-05,
"loss_nats": 0.9985314607620239,
"step": 6820,
"unclipped_grad_norm": 0.7713363170623779
},
{
"bits_per_byte": 1.4385394240961653,
"learning_rate": 4.548339157467199e-05,
"loss_nats": 0.9971195459365845,
"step": 6830,
"unclipped_grad_norm": 0.8126972317695618
},
{
"bits_per_byte": 1.4959006256557872,
"learning_rate": 4.52249193751124e-05,
"loss_nats": 1.036879301071167,
"step": 6840,
"unclipped_grad_norm": 0.7896748185157776
},
{
"bits_per_byte": 1.465242826330332,
"learning_rate": 4.4968493801209985e-05,
"loss_nats": 1.0156289339065552,
"step": 6850,
"unclipped_grad_norm": 0.8369152545928955
},
{
"bits_per_byte": 1.4691382333723746,
"learning_rate": 4.471411923457034e-05,
"loss_nats": 1.0183290243148804,
"step": 6860,
"unclipped_grad_norm": 0.8024821877479553
},
{
"bits_per_byte": 1.4717907217966317,
"learning_rate": 4.446180002175291e-05,
"loss_nats": 1.020167589187622,
"step": 6870,
"unclipped_grad_norm": 0.808424174785614
},
{
"bits_per_byte": 1.5114406340224489,
"learning_rate": 4.421154047419693e-05,
"loss_nats": 1.0476508140563965,
"step": 6880,
"unclipped_grad_norm": 0.8209573030471802
},
{
"bits_per_byte": 1.4045318326885299,
"learning_rate": 4.396334486814753e-05,
"loss_nats": 0.9735472798347473,
"step": 6890,
"unclipped_grad_norm": 0.7931460738182068
},
{
"bits_per_byte": 1.436284731543354,
"learning_rate": 4.371721744458285e-05,
"loss_nats": 0.9955567121505737,
"step": 6900,
"unclipped_grad_norm": 0.7861348390579224
},
{
"bits_per_byte": 1.6150507221214936,
"learning_rate": 4.347316240914145e-05,
"loss_nats": 1.119467854499817,
"step": 6910,
"unclipped_grad_norm": 0.8759272694587708
},
{
"bits_per_byte": 1.4220056999728183,
"learning_rate": 4.3231183932050546e-05,
"loss_nats": 0.9856592416763306,
"step": 6920,
"unclipped_grad_norm": 0.7773904204368591
},
{
"bits_per_byte": 1.4332720254568647,
"learning_rate": 4.299128614805464e-05,
"loss_nats": 0.9934684634208679,
"step": 6930,
"unclipped_grad_norm": 0.8008426427841187
},
{
"bits_per_byte": 1.46887819560427,
"learning_rate": 4.2753473156344965e-05,
"loss_nats": 1.0181487798690796,
"step": 6940,
"unclipped_grad_norm": 0.8342399597167969
},
{
"bits_per_byte": 1.4565233059315876,
"learning_rate": 4.251774902048942e-05,
"loss_nats": 1.0095850229263306,
"step": 6950,
"unclipped_grad_norm": 0.8141571283340454
},
{
"bits_per_byte": 1.5009811651449254,
"learning_rate": 4.228411776836306e-05,
"loss_nats": 1.0404008626937866,
"step": 6960,
"unclipped_grad_norm": 0.8327651619911194
},
{
"bits_per_byte": 1.4767801104808127,
"learning_rate": 4.2052583392079427e-05,
"loss_nats": 1.0236259698867798,
"step": 6970,
"unclipped_grad_norm": 0.801481306552887
},
{
"bits_per_byte": 1.517935386849632,
"learning_rate": 4.182314984792214e-05,
"loss_nats": 1.0521526336669922,
"step": 6980,
"unclipped_grad_norm": 0.8559384942054749
},
{
"bits_per_byte": 1.462364696668143,
"learning_rate": 4.159582105627748e-05,
"loss_nats": 1.0136339664459229,
"step": 6990,
"unclipped_grad_norm": 0.8250771164894104
},
{
"bits_per_byte": 1.4113358963132898,
"learning_rate": 4.137060090156724e-05,
"loss_nats": 0.9782634973526001,
"step": 7000,
"unclipped_grad_norm": 0.8160101771354675
},
{
"bits_per_byte": 1.4572850170922589,
"learning_rate": 4.114749323218249e-05,
"loss_nats": 1.010113000869751,
"step": 7010,
"unclipped_grad_norm": 0.8388007283210754
},
{
"bits_per_byte": 1.4311967968002286,
"learning_rate": 4.0926501860417706e-05,
"loss_nats": 0.9920300245285034,
"step": 7020,
"unclipped_grad_norm": 0.8063672780990601
},
{
"bits_per_byte": 1.4271649215247264,
"learning_rate": 4.070763056240571e-05,
"loss_nats": 0.9892353415489197,
"step": 7030,
"unclipped_grad_norm": 0.7880839109420776
},
{
"bits_per_byte": 1.5230513547648479,
"learning_rate": 4.049088307805306e-05,
"loss_nats": 1.0556987524032593,
"step": 7040,
"unclipped_grad_norm": 0.8723810911178589
},
{
"bits_per_byte": 1.48768088684045,
"learning_rate": 4.027626311097629e-05,
"loss_nats": 1.031181812286377,
"step": 7050,
"unclipped_grad_norm": 0.8570515513420105
},
{
"bits_per_byte": 1.4927676177050193,
"learning_rate": 4.006377432843846e-05,
"loss_nats": 1.0347076654434204,
"step": 7060,
"unclipped_grad_norm": 0.8982753157615662
},
{
"bits_per_byte": 1.4459658069430723,
"learning_rate": 3.985342036128656e-05,
"loss_nats": 1.0022671222686768,
"step": 7070,
"unclipped_grad_norm": 0.8039788603782654
},
{
"bits_per_byte": 1.364939964607379,
"learning_rate": 3.964520480388956e-05,
"loss_nats": 0.9461042881011963,
"step": 7080,
"unclipped_grad_norm": 0.7935104966163635
},
{
"bits_per_byte": 1.4353330655448047,
"learning_rate": 3.943913121407678e-05,
"loss_nats": 0.9948970675468445,
"step": 7090,
"unclipped_grad_norm": 0.810722291469574
},
{
"bits_per_byte": 1.4301969756594377,
"learning_rate": 3.923520311307735e-05,
"loss_nats": 0.9913370013237,
"step": 7100,
"unclipped_grad_norm": 0.7980725169181824
},
{
"bits_per_byte": 1.4812796925753362,
"learning_rate": 3.9033423985459816e-05,
"loss_nats": 1.0267448425292969,
"step": 7110,
"unclipped_grad_norm": 0.8589518666267395
},
{
"bits_per_byte": 1.491397775890897,
"learning_rate": 3.88337972790728e-05,
"loss_nats": 1.0337581634521484,
"step": 7120,
"unclipped_grad_norm": 0.881410539150238
},
{
"bits_per_byte": 1.4146181852050066,
"learning_rate": 3.8636326404985845e-05,
"loss_nats": 0.9805386066436768,
"step": 7130,
"unclipped_grad_norm": 0.8223039507865906
},
{
"bits_per_byte": 1.522814190689308,
"learning_rate": 3.844101473743143e-05,
"loss_nats": 1.0555343627929688,
"step": 7140,
"unclipped_grad_norm": 0.8451608419418335
},
{
"bits_per_byte": 1.5467134158011566,
"learning_rate": 3.824786561374701e-05,
"loss_nats": 1.072100043296814,
"step": 7150,
"unclipped_grad_norm": 0.8498587012290955
},
{
"bits_per_byte": 1.4723551688567635,
"learning_rate": 3.805688233431824e-05,
"loss_nats": 1.0205588340759277,
"step": 7160,
"unclipped_grad_norm": 0.8513677716255188
},
{
"bits_per_byte": 1.4676500674944588,
"learning_rate": 3.786806816252236e-05,
"loss_nats": 1.0172975063323975,
"step": 7170,
"unclipped_grad_norm": 0.8321399092674255
},
{
"bits_per_byte": 1.3859237398218012,
"learning_rate": 3.768142632467269e-05,
"loss_nats": 0.9606491327285767,
"step": 7180,
"unclipped_grad_norm": 0.7659841179847717
},
{
"bits_per_byte": 1.3902817801946648,
"learning_rate": 3.7496960009963236e-05,
"loss_nats": 0.9636698961257935,
"step": 7190,
"unclipped_grad_norm": 0.7895154356956482
},
{
"bits_per_byte": 1.4776350362382518,
"learning_rate": 3.7314672370414325e-05,
"loss_nats": 1.0242185592651367,
"step": 7200,
"unclipped_grad_norm": 0.8478621244430542
},
{
"bits_per_byte": 1.4778720283311406,
"learning_rate": 3.713456652081882e-05,
"loss_nats": 1.0243828296661377,
"step": 7210,
"unclipped_grad_norm": 0.8023228049278259
},
{
"bits_per_byte": 1.4423936412933265,
"learning_rate": 3.695664553868873e-05,
"loss_nats": 0.9997910857200623,
"step": 7220,
"unclipped_grad_norm": 0.8181896805763245
},
{
"bits_per_byte": 1.4464050506333759,
"learning_rate": 3.678091246420277e-05,
"loss_nats": 1.0025715827941895,
"step": 7230,
"unclipped_grad_norm": 0.8425759077072144
},
{
"bits_per_byte": 1.4082518174266925,
"learning_rate": 3.660737030015427e-05,
"loss_nats": 0.9761257767677307,
"step": 7240,
"unclipped_grad_norm": 0.7972860336303711
},
{
"bits_per_byte": 1.4478307868090285,
"learning_rate": 3.6436022011900056e-05,
"loss_nats": 1.0035598278045654,
"step": 7250,
"unclipped_grad_norm": 0.8181965947151184
},
{
"bits_per_byte": 1.4351106059859136,
"learning_rate": 3.626687052730954e-05,
"loss_nats": 0.9947428703308105,
"step": 7260,
"unclipped_grad_norm": 0.8426879048347473
},
{
"bits_per_byte": 1.5111732010003571,
"learning_rate": 3.6099918736714955e-05,
"loss_nats": 1.047465443611145,
"step": 7270,
"unclipped_grad_norm": 0.862413763999939
},
{
"bits_per_byte": 1.457399041589781,
"learning_rate": 3.5935169492861716e-05,
"loss_nats": 1.0101920366287231,
"step": 7280,
"unclipped_grad_norm": 0.8488914370536804
},
{
"bits_per_byte": 1.4536087159474156,
"learning_rate": 3.5772625610859886e-05,
"loss_nats": 1.0075647830963135,
"step": 7290,
"unclipped_grad_norm": 0.8383923172950745
},
{
"bits_per_byte": 1.4305855704590622,
"learning_rate": 3.5612289868135887e-05,
"loss_nats": 0.9916063547134399,
"step": 7300,
"unclipped_grad_norm": 0.8480672240257263
},
{
"bits_per_byte": 1.4800558640317965,
"learning_rate": 3.545416500438522e-05,
"loss_nats": 1.0258965492248535,
"step": 7310,
"unclipped_grad_norm": 0.8378974795341492
},
{
"bits_per_byte": 1.4690943777964045,
"learning_rate": 3.52982537215255e-05,
"loss_nats": 1.018298625946045,
"step": 7320,
"unclipped_grad_norm": 0.8481025695800781
},
{
"bits_per_byte": 1.4814797083982898,
"learning_rate": 3.5144558683650396e-05,
"loss_nats": 1.0268834829330444,
"step": 7330,
"unclipped_grad_norm": 0.9106674194335938
},
{
"bits_per_byte": 1.449738418372399,
"learning_rate": 3.499308251698404e-05,
"loss_nats": 1.0048820972442627,
"step": 7340,
"unclipped_grad_norm": 0.8405259847640991
},
{
"bits_per_byte": 1.486937577823421,
"learning_rate": 3.484382780983616e-05,
"loss_nats": 1.0306665897369385,
"step": 7350,
"unclipped_grad_norm": 0.8228537440299988
},
{
"bits_per_byte": 1.4739011209053692,
"learning_rate": 3.469679711255795e-05,
"loss_nats": 1.0216304063796997,
"step": 7360,
"unclipped_grad_norm": 0.8316923975944519
},
{
"bits_per_byte": 1.4767537971352307,
"learning_rate": 3.4551992937498284e-05,
"loss_nats": 1.0236077308654785,
"step": 7370,
"unclipped_grad_norm": 0.8346049785614014
},
{
"bits_per_byte": 1.5318123229824503,
"learning_rate": 3.440941775896104e-05,
"loss_nats": 1.0617713928222656,
"step": 7380,
"unclipped_grad_norm": 0.9013696908950806
},
{
"bits_per_byte": 1.5039485538029123,
"learning_rate": 3.4269074013162596e-05,
"loss_nats": 1.0424576997756958,
"step": 7390,
"unclipped_grad_norm": 0.8856703639030457
},
{
"bits_per_byte": 1.4314098833046476,
"learning_rate": 3.41309640981904e-05,
"loss_nats": 0.9921777248382568,
"step": 7400,
"unclipped_grad_norm": 0.8249487280845642
},
{
"bits_per_byte": 1.4394619390353935,
"learning_rate": 3.3995090373961795e-05,
"loss_nats": 0.9977589845657349,
"step": 7410,
"unclipped_grad_norm": 0.8362482786178589
},
{
"bits_per_byte": 1.518494330464936,
"learning_rate": 3.3861455162183876e-05,
"loss_nats": 1.0525400638580322,
"step": 7420,
"unclipped_grad_norm": 0.8421095609664917
},
{
"bits_per_byte": 1.4326227909498577,
"learning_rate": 3.3730060746313664e-05,
"loss_nats": 0.9930184483528137,
"step": 7430,
"unclipped_grad_norm": 0.8718878626823425
},
{
"bits_per_byte": 1.4438445729273304,
"learning_rate": 3.3600909371519225e-05,
"loss_nats": 1.0007967948913574,
"step": 7440,
"unclipped_grad_norm": 0.8476481437683105
},
{
"bits_per_byte": 1.3938656406646694,
"learning_rate": 3.3474003244641194e-05,
"loss_nats": 0.9661540389060974,
"step": 7450,
"unclipped_grad_norm": 0.8507251739501953
},
{
"bits_per_byte": 1.420028243453198,
"learning_rate": 3.334934453415514e-05,
"loss_nats": 0.9842885732650757,
"step": 7460,
"unclipped_grad_norm": 0.8349717259407043
},
{
"bits_per_byte": 1.4775118966602339,
"learning_rate": 3.3226935370134486e-05,
"loss_nats": 1.0241332054138184,
"step": 7470,
"unclipped_grad_norm": 0.8582044243812561
},
{
"bits_per_byte": 1.3999588999934132,
"learning_rate": 3.31067778442141e-05,
"loss_nats": 0.9703775644302368,
"step": 7480,
"unclipped_grad_norm": 0.84687340259552
},
{
"bits_per_byte": 1.4226986180731447,
"learning_rate": 3.298887400955457e-05,
"loss_nats": 0.9861395359039307,
"step": 7490,
"unclipped_grad_norm": 0.831066370010376
},
{
"bits_per_byte": 1.4892197876003705,
"learning_rate": 3.287322588080714e-05,
"loss_nats": 1.0322484970092773,
"step": 7500,
"unclipped_grad_norm": 0.8232710361480713
},
{
"bits_per_byte": 1.461985990870943,
"learning_rate": 3.275983543407924e-05,
"loss_nats": 1.013371467590332,
"step": 7510,
"unclipped_grad_norm": 0.8441055417060852
},
{
"bits_per_byte": 1.4446456681150495,
"learning_rate": 3.2648704606900735e-05,
"loss_nats": 1.001352071762085,
"step": 7520,
"unclipped_grad_norm": 0.8653393387794495
},
{
"bits_per_byte": 1.4775328785436392,
"learning_rate": 3.253983529819086e-05,
"loss_nats": 1.0241477489471436,
"step": 7530,
"unclipped_grad_norm": 0.8701198697090149
},
{
"bits_per_byte": 1.5027517265355579,
"learning_rate": 3.243322936822571e-05,
"loss_nats": 1.041628122329712,
"step": 7540,
"unclipped_grad_norm": 0.8485988974571228
},
{
"bits_per_byte": 1.4225563884208812,
"learning_rate": 3.232888863860648e-05,
"loss_nats": 0.9860409498214722,
"step": 7550,
"unclipped_grad_norm": 0.8343487977981567
},
{
"bits_per_byte": 1.4745732291049412,
"learning_rate": 3.222681489222838e-05,
"loss_nats": 1.0220962762832642,
"step": 7560,
"unclipped_grad_norm": 0.8472090363502502
},
{
"bits_per_byte": 1.480253472097638,
"learning_rate": 3.212700987325009e-05,
"loss_nats": 1.0260335206985474,
"step": 7570,
"unclipped_grad_norm": 0.8703269958496094
},
{
"bits_per_byte": 1.4382653697420156,
"learning_rate": 3.202947528706403e-05,
"loss_nats": 0.9969295859336853,
"step": 7580,
"unclipped_grad_norm": 0.847757875919342
},
{
"bits_per_byte": 1.4346089325933469,
"learning_rate": 3.1934212800267116e-05,
"loss_nats": 0.9943951368331909,
"step": 7590,
"unclipped_grad_norm": 0.845406711101532
},
{
"bits_per_byte": 1.414465206637064,
"learning_rate": 3.1841224040632484e-05,
"loss_nats": 0.9804325699806213,
"step": 7600,
"unclipped_grad_norm": 0.8379301428794861
},
{
"bits_per_byte": 1.3641118681434745,
"learning_rate": 3.175051059708143e-05,
"loss_nats": 0.9455302953720093,
"step": 7610,
"unclipped_grad_norm": 0.7969970107078552
},
{
"bits_per_byte": 1.4648639485504813,
"learning_rate": 3.1662074019656414e-05,
"loss_nats": 1.0153663158416748,
"step": 7620,
"unclipped_grad_norm": 0.8826617002487183
},
{
"bits_per_byte": 1.4355323934371547,
"learning_rate": 3.157591581949455e-05,
"loss_nats": 0.9950352311134338,
"step": 7630,
"unclipped_grad_norm": 0.8636281490325928
},
{
"bits_per_byte": 1.4410591419139565,
"learning_rate": 3.149203746880175e-05,
"loss_nats": 0.998866081237793,
"step": 7640,
"unclipped_grad_norm": 0.8592743277549744
},
{
"bits_per_byte": 1.5176450801349755,
"learning_rate": 3.141044040082763e-05,
"loss_nats": 1.0519514083862305,
"step": 7650,
"unclipped_grad_norm": 0.9256166219711304
},
{
"bits_per_byte": 1.4474489853241133,
"learning_rate": 3.133112600984091e-05,
"loss_nats": 1.0032951831817627,
"step": 7660,
"unclipped_grad_norm": 0.8606430292129517
},
{
"bits_per_byte": 1.4195236463555665,
"learning_rate": 3.125409565110571e-05,
"loss_nats": 0.9839388132095337,
"step": 7670,
"unclipped_grad_norm": 0.8635953664779663
},
{
"bits_per_byte": 1.3989710316468573,
"learning_rate": 3.117935064085835e-05,
"loss_nats": 0.9696928262710571,
"step": 7680,
"unclipped_grad_norm": 0.8459085822105408
},
{
"bits_per_byte": 1.446349844202449,
"learning_rate": 3.110689225628479e-05,
"loss_nats": 1.0025333166122437,
"step": 7690,
"unclipped_grad_norm": 0.8500027656555176
},
{
"bits_per_byte": 1.456846289349908,
"learning_rate": 3.1036721735498926e-05,
"loss_nats": 1.009808897972107,
"step": 7700,
"unclipped_grad_norm": 0.896350085735321
},
{
"bits_per_byte": 1.3794107568336267,
"learning_rate": 3.096884027752132e-05,
"loss_nats": 0.9561346769332886,
"step": 7710,
"unclipped_grad_norm": 0.8459534049034119
},
{
"bits_per_byte": 1.3797531742914946,
"learning_rate": 3.090324904225885e-05,
"loss_nats": 0.9563720226287842,
"step": 7720,
"unclipped_grad_norm": 0.8360687494277954
},
{
"bits_per_byte": 1.4376540574095236,
"learning_rate": 3.083994915048472e-05,
"loss_nats": 0.996505856513977,
"step": 7730,
"unclipped_grad_norm": 0.82816481590271
},
{
"bits_per_byte": 1.4181697129366488,
"learning_rate": 3.077894168381942e-05,
"loss_nats": 0.9830003380775452,
"step": 7740,
"unclipped_grad_norm": 0.83390212059021
},
{
"bits_per_byte": 1.4407892151434272,
"learning_rate": 3.0720227684712254e-05,
"loss_nats": 0.998678982257843,
"step": 7750,
"unclipped_grad_norm": 0.8495392203330994
},
{
"bits_per_byte": 1.4190407190719436,
"learning_rate": 3.0663808156423456e-05,
"loss_nats": 0.9836040735244751,
"step": 7760,
"unclipped_grad_norm": 0.850292444229126
},
{
"bits_per_byte": 1.4212888762840223,
"learning_rate": 3.060968406300711e-05,
"loss_nats": 0.9851623773574829,
"step": 7770,
"unclipped_grad_norm": 0.888531506061554
},
{
"bits_per_byte": 1.5506865590013892,
"learning_rate": 3.055785632929463e-05,
"loss_nats": 1.0748540163040161,
"step": 7780,
"unclipped_grad_norm": 0.9023171067237854
},
{
"bits_per_byte": 1.4346873566821403,
"learning_rate": 3.0508325840878984e-05,
"loss_nats": 0.9944494962692261,
"step": 7790,
"unclipped_grad_norm": 0.8470334410667419
},
{
"bits_per_byte": 1.4727918328073042,
"learning_rate": 3.046109344409957e-05,
"loss_nats": 1.0208615064620972,
"step": 7800,
"unclipped_grad_norm": 0.9052314758300781
},
{
"bits_per_byte": 1.3950921349392975,
"learning_rate": 3.0416159946027727e-05,
"loss_nats": 0.9670041799545288,
"step": 7810,
"unclipped_grad_norm": 0.841830849647522
},
{
"bits_per_byte": 1.4118079886899082,
"learning_rate": 3.0373526114452946e-05,
"loss_nats": 0.978590726852417,
"step": 7820,
"unclipped_grad_norm": 0.8733639121055603
},
{
"bits_per_byte": 1.4649716096899212,
"learning_rate": 3.0333192677869803e-05,
"loss_nats": 1.0154409408569336,
"step": 7830,
"unclipped_grad_norm": 0.851719081401825
},
{
"bits_per_byte": 1.4172854641372379,
"learning_rate": 3.029516032546544e-05,
"loss_nats": 0.9823874235153198,
"step": 7840,
"unclipped_grad_norm": 0.8457031846046448
},
{
"bits_per_byte": 1.3998510668713222,
"learning_rate": 3.02594297071078e-05,
"loss_nats": 0.9703028202056885,
"step": 7850,
"unclipped_grad_norm": 0.8359724283218384
},
{
"bits_per_byte": 1.413704183406996,
"learning_rate": 3.0226001433334603e-05,
"loss_nats": 0.9799050688743591,
"step": 7860,
"unclipped_grad_norm": 0.8466342091560364
},
{
"bits_per_byte": 1.4588555626599382,
"learning_rate": 3.019487607534278e-05,
"loss_nats": 1.0112016201019287,
"step": 7870,
"unclipped_grad_norm": 0.8837466239929199
},
{
"bits_per_byte": 1.4292317230314702,
"learning_rate": 3.016605416497886e-05,
"loss_nats": 0.9906679391860962,
"step": 7880,
"unclipped_grad_norm": 0.8551597595214844
},
{
"bits_per_byte": 1.5078522160121963,
"learning_rate": 3.0139536194729734e-05,
"loss_nats": 1.0451635122299194,
"step": 7890,
"unclipped_grad_norm": 0.8927572965621948
},
{
"bits_per_byte": 1.4757206973514978,
"learning_rate": 3.0115322617714362e-05,
"loss_nats": 1.022891640663147,
"step": 7900,
"unclipped_grad_norm": 0.8807728290557861
},
{
"bits_per_byte": 1.4780766876856672,
"learning_rate": 3.0093413847675965e-05,
"loss_nats": 1.0245246887207031,
"step": 7910,
"unclipped_grad_norm": 0.9293215870857239
},
{
"bits_per_byte": 1.3870213330996077,
"learning_rate": 3.0073810258974985e-05,
"loss_nats": 0.9614099264144897,
"step": 7920,
"unclipped_grad_norm": 0.8720042109489441
},
{
"bits_per_byte": 1.3890644869918578,
"learning_rate": 3.0056512186582658e-05,
"loss_nats": 0.962826132774353,
"step": 7930,
"unclipped_grad_norm": 0.8701853156089783
},
{
"bits_per_byte": 1.4495222361802644,
"learning_rate": 3.0041519926075316e-05,
"loss_nats": 1.0047322511672974,
"step": 7940,
"unclipped_grad_norm": 0.8989008069038391
},
{
"bits_per_byte": 1.4133007980993975,
"learning_rate": 3.0028833733629313e-05,
"loss_nats": 0.9796254634857178,
"step": 7950,
"unclipped_grad_norm": 0.839148759841919
},
{
"bits_per_byte": 1.4208300265715204,
"learning_rate": 3.0018453826016686e-05,
"loss_nats": 0.9848443269729614,
"step": 7960,
"unclipped_grad_norm": 0.8453875184059143
},
{
"bits_per_byte": 1.41087549875693,
"learning_rate": 3.001038038060143e-05,
"loss_nats": 0.9779443740844727,
"step": 7970,
"unclipped_grad_norm": 0.8587523102760315
},
{
"bits_per_byte": 1.446222061092858,
"learning_rate": 3.0004613535336433e-05,
"loss_nats": 1.0024447441101074,
"step": 7980,
"unclipped_grad_norm": 0.8559814691543579
},
{
"bits_per_byte": 1.4566966644436572,
"learning_rate": 3.000115338876117e-05,
"loss_nats": 1.0097051858901978,
"step": 7990,
"unclipped_grad_norm": 0.860873818397522
},
{
"bits_per_byte": 1.3805011268400968,
"learning_rate": 3e-05,
"loss_nats": 0.9568904638290405,
"step": 8000,
"unclipped_grad_norm": 0.8578172326087952
}
]
}
Reference in New Issue View Git Blame Copy Permalink